diff --git a/packages/app/src/blossom.ts b/packages/app/src/blossom.ts index 335de12..7d0a107 100644 --- a/packages/app/src/blossom.ts +++ b/packages/app/src/blossom.ts @@ -1,22 +1,24 @@ import {BLOSSOM_SERVERS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {makeOutboxLoader} from "./relaySelections.js" -export const blossomServers = deriveEventsMapped(repository, { - filters: [{kinds: [BLOSSOM_SERVERS]}], - itemToEvent: item => item.event, +export const blossomServersByPubkey = deriveItemsByKey({ + repository, eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)), + filters: [{kinds: [BLOSSOM_SERVERS]}], + getKey: blossomServers => blossomServers.event.pubkey, }) -export const { - indexStore: blossomServersByPubkey, - deriveItem: deriveBlossomServers, - loadItem: loadBlossomServers, -} = collection({ - name: "blossomServers", - store: blossomServers, - getKey: blossomServers => blossomServers.event.pubkey, - load: makeOutboxLoader(BLOSSOM_SERVERS), -}) +export const blossomServers = deriveItems(blossomServersByPubkey) + +export const getBlossomServersByPubkey = getter(blossomServersByPubkey) + +export const getBlossomServers = (pubkey: string) => getBlossomServersByPubkey().get(pubkey) + +export const forceLoadBlossomServers = makeForceLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServers) + +export const loadBlossomServers = makeLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServers) + +export const deriveBlossomServers = makeDeriveItem(blossomServersByPubkey, loadBlossomServers) diff --git a/packages/app/src/follows.ts b/packages/app/src/follows.ts index 220fc44..815c11e 100644 --- a/packages/app/src/follows.ts +++ b/packages/app/src/follows.ts @@ -1,22 +1,24 @@ import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {makeOutboxLoader} from "./relaySelections.js" -export const follows = deriveEventsMapped(repository, { - filters: [{kinds: [FOLLOWS]}], - itemToEvent: item => item.event, +export const followsByPubkey = deriveItemsByKey({ + repository, eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)), + filters: [{kinds: [FOLLOWS]}], + getKey: follows => follows.event.pubkey, }) -export const { - indexStore: followsByPubkey, - deriveItem: deriveFollows, - loadItem: loadFollows, -} = collection({ - name: "follows", - store: follows, - getKey: follows => follows.event.pubkey, - load: makeOutboxLoader(FOLLOWS), -}) +export const follows = deriveItems(followsByPubkey) + +export const getFollowsByPubkey = getter(followsByPubkey) + +export const getFollows = (pubkey: string) => getFollowsByPubkey().get(pubkey) + +export const forceLoadFollows = makeForceLoadItem(makeOutboxLoader(FOLLOWS), getFollows) + +export const loadFollows = makeLoadItem(makeOutboxLoader(FOLLOWS), getFollows) + +export const deriveFollows = makeDeriveItem(followsByPubkey, loadFollows) diff --git a/packages/app/src/inboxRelaySelections.ts b/packages/app/src/inboxRelaySelections.ts index 666545d..b66346c 100644 --- a/packages/app/src/inboxRelaySelections.ts +++ b/packages/app/src/inboxRelaySelections.ts @@ -1,22 +1,24 @@ import {INBOX_RELAYS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {makeOutboxLoader} from "./relaySelections.js" -export const inboxRelaySelections = deriveEventsMapped(repository, { - filters: [{kinds: [INBOX_RELAYS]}], - itemToEvent: item => item.event, +export const inboxRelaySelectionsByPubkey = deriveItemsByKey({ + repository, eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)), + filters: [{kinds: [INBOX_RELAYS]}], + getKey: inboxRelaySelections => inboxRelaySelections.event.pubkey, }) -export const { - indexStore: inboxRelaySelectionsByPubkey, - deriveItem: deriveInboxRelaySelections, - loadItem: loadInboxRelaySelections, -} = collection({ - name: "inboxRelaySelections", - store: inboxRelaySelections, - getKey: inboxRelaySelections => inboxRelaySelections.event.pubkey, - load: makeOutboxLoader(INBOX_RELAYS), -}) +export const inboxRelaySelections = deriveItems(inboxRelaySelectionsByPubkey) + +export const getInboxRelaySelectionsByPubkey = getter(inboxRelaySelectionsByPubkey) + +export const getInboxRelaySelections = (pubkey: string) => getInboxRelaySelectionsByPubkey().get(pubkey) + +export const forceLoadInboxRelaySelections = makeForceLoadItem(makeOutboxLoader(INBOX_RELAYS), getInboxRelaySelections) + +export const loadInboxRelaySelections = makeLoadItem(makeOutboxLoader(INBOX_RELAYS), getInboxRelaySelections) + +export const deriveInboxRelaySelections = makeDeriveItem(inboxRelaySelectionsByPubkey, loadInboxRelaySelections) diff --git a/packages/app/src/mutes.ts b/packages/app/src/mutes.ts index ae62629..9bf67b3 100644 --- a/packages/app/src/mutes.ts +++ b/packages/app/src/mutes.ts @@ -1,28 +1,30 @@ import {MUTES, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {ensurePlaintext} from "./plaintext.js" import {makeOutboxLoader} from "./relaySelections.js" -export const mutes = deriveEventsMapped(repository, { - filters: [{kinds: [MUTES]}], - itemToEvent: item => item.event, +export const mutesByPubkey = deriveItemsByKey({ + repository, eventToItem: async (event: TrustedEvent) => readList( asDecryptedEvent(event, { content: await ensurePlaintext(event), }), ), + filters: [{kinds: [MUTES]}], + getKey: mute => mute.event.pubkey, }) -export const { - indexStore: mutesByPubkey, - deriveItem: deriveMutes, - loadItem: loadMutes, -} = collection({ - name: "mutes", - store: mutes, - getKey: mute => mute.event.pubkey, - load: makeOutboxLoader(MUTES), -}) +export const mutes = deriveItems(mutesByPubkey) + +export const getMutesByPubkey = getter(mutesByPubkey) + +export const getMutes = (pubkey: string) => getMutesByPubkey().get(pubkey) + +export const forceLoadMutes = makeForceLoadItem(makeOutboxLoader(MUTES), getMutes) + +export const loadMutes = makeLoadItem(makeOutboxLoader(MUTES), getMutes) + +export const deriveMutes = makeDeriveItem(mutesByPubkey, loadMutes) diff --git a/packages/app/src/pins.ts b/packages/app/src/pins.ts index af10fe5..387e588 100644 --- a/packages/app/src/pins.ts +++ b/packages/app/src/pins.ts @@ -1,22 +1,24 @@ import {PINS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {makeOutboxLoader} from "./relaySelections.js" -export const pins = deriveEventsMapped(repository, { - filters: [{kinds: [PINS]}], - itemToEvent: item => item.event, +export const pinsByPubkey = deriveItemsByKey({ + repository, eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)), + filters: [{kinds: [PINS]}], + getKey: pins => pins.event.pubkey, }) -export const { - indexStore: pinsByPubkey, - deriveItem: derivePins, - loadItem: loadPins, -} = collection({ - name: "pins", - store: pins, - getKey: pins => pins.event.pubkey, - load: makeOutboxLoader(PINS), -}) +export const pins = deriveItems(pinsByPubkey) + +export const getPinsByPubkey = getter(pinsByPubkey) + +export const getPins = (pubkey: string) => getPinsByPubkey().get(pubkey) + +export const forceLoadPins = makeForceLoadItem(makeOutboxLoader(PINS), getPins) + +export const loadPins = makeLoadItem(makeOutboxLoader(PINS), getPins) + +export const derivePins = makeDeriveItem(pinsByPubkey, loadPins) diff --git a/packages/app/src/profiles.ts b/packages/app/src/profiles.ts index ebc5093..83b6505 100644 --- a/packages/app/src/profiles.ts +++ b/packages/app/src/profiles.ts @@ -1,6 +1,6 @@ import {derived, readable} from "svelte/store" import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util" -import {deriveItemsByKey, deriveItems, makeDeriveItem, getter} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {repository} from "./core.js" import {makeOutboxLoaderWithIndexers} from "./relaySelections.js" @@ -13,14 +13,16 @@ export const profilesByPubkey = deriveItemsByKey({ export const profiles = deriveItems(profilesByPubkey) -export const loadProfile = makeOutboxLoaderWithIndexers(PROFILE) - -export const deriveProfile = makeDeriveItem(profilesByPubkey, loadProfile) - export const getProfilesByPubkey = getter(profilesByPubkey) export const getProfile = (pubkey: string) => getProfilesByPubkey().get(pubkey) +export const forceLoadProfile = makeForceLoadItem(makeOutboxLoaderWithIndexers(PROFILE), getProfile) + +export const loadProfile = makeLoadItem(makeOutboxLoaderWithIndexers(PROFILE), getProfile) + +export const deriveProfile = makeDeriveItem(profilesByPubkey, loadProfile) + export const displayProfileByPubkey = (pubkey: string | undefined) => pubkey ? displayProfile(getProfile(pubkey), displayPubkey(pubkey)) : "" diff --git a/packages/app/src/relaySelections.ts b/packages/app/src/relaySelections.ts index 89b0aa4..a65b600 100644 --- a/packages/app/src/relaySelections.ts +++ b/packages/app/src/relaySelections.ts @@ -7,7 +7,7 @@ import { TrustedEvent, PublishedList, } from "@welshman/util" -import {deriveEventsMapped, collection} from "@welshman/store" +import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store" import {load, LoadOptions} from "@welshman/net" import {Router} from "@welshman/router" import {repository} from "./core.js" @@ -41,19 +41,21 @@ export const makeOutboxLoaderWithIndexers = ]) } -export const relaySelections = deriveEventsMapped(repository, { - filters: [{kinds: [RELAYS]}], - itemToEvent: item => item.event, +export const relaySelectionsByPubkey = deriveItemsByKey({ + repository, eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)), + filters: [{kinds: [RELAYS]}], + getKey: relaySelections => relaySelections.event.pubkey, }) -export const { - indexStore: relaySelectionsByPubkey, - deriveItem: deriveRelaySelections, - loadItem: loadRelaySelections, -} = collection({ - name: "relaySelections", - store: relaySelections, - getKey: relaySelections => relaySelections.event.pubkey, - load: makeOutboxLoaderWithIndexers(RELAYS), -}) +export const relaySelections = deriveItems(relaySelectionsByPubkey) + +export const getRelaySelectionsByPubkey = getter(relaySelectionsByPubkey) + +export const getRelaySelections = (pubkey: string) => getRelaySelectionsByPubkey().get(pubkey) + +export const forceLoadRelaySelections = makeForceLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelaySelections) + +export const loadRelaySelections = makeLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelaySelections) + +export const deriveRelaySelections = makeDeriveItem(relaySelectionsByPubkey, loadRelaySelections) diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index fc1c073..0c58d2d 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -1,4 +1,3 @@ -export * from "./synced.js" export * from "./misc.js" -export * from "./loader.js" +export * from "./synced.js" export * from "./repository.js" diff --git a/packages/store/src/loader.ts b/packages/store/src/loader.ts deleted file mode 100644 index f635733..0000000 --- a/packages/store/src/loader.ts +++ /dev/null @@ -1,68 +0,0 @@ -import {Maybe, now} from "@welshman/lib" - -export type LoaderOptions = { - getItem: (key: string) => T - getLastFetched: (key: string) => number - setLastFetched: (key: string, ts: number) => void - load: (key: string, ...args: any[]) => Promise - timeout?: number -} - -export const makeLoader = (options: LoaderOptions) => { - const timeout = options.timeout || 3600 - const pending = new Map>>() - const attempts = new Map() - - const baseLoad = async (key: string, force: boolean, ...args: any[]): Promise> => { - const stale = options.getItem(key) - const lastFetched = options.getLastFetched(key) - - // If we have an item, reload if it's stale - if (stale && lastFetched > now() - timeout && !force) { - return stale - } - - const pendingItem = pending.get(key) - - // If we already are loading, await and return - if (pendingItem) { - return pendingItem - } - - const attempt = attempts.get(key) || 0 - - // Use exponential backoff to throttle attempts - if (lastFetched > now() - Math.pow(2, attempt) && !force) { - return stale - } - - attempts.set(key, attempt + 1) - - options.setLastFetched(key, now()) - - const promise = options.load(key, ...args).then(() => options.getItem(key)) - - pending.set(key, promise) - - let item - try { - item = await promise - } catch (e) { - console.warn(`Failed to load ${name} item ${key}`, e) - } finally { - pending.delete(key) - } - - if (item) { - attempts.delete(key) - } - - return item - } - - const load = (key: string, ...args: any[]) => baseLoad(key, false, ...args) - - const forceLoad = (key: string, ...args: any[]) => baseLoad(key, true, ...args) - - return {load, forceLoad} -} diff --git a/packages/store/src/repository.ts b/packages/store/src/repository.ts index afd027b..2e14dc0 100644 --- a/packages/store/src/repository.ts +++ b/packages/store/src/repository.ts @@ -1,5 +1,5 @@ import {derived, readable, Readable} from "svelte/store" -import {on, indexBy, mapPop, Maybe, call, sortBy, first} from "@welshman/lib" +import {on, now, indexBy, mapPop, Maybe, call, sortBy, first} from "@welshman/lib" import {matchFilters, getIdFilters, Filter, TrustedEvent} from "@welshman/util" import {Repository, RepositoryUpdate, Tracker} from "@welshman/net" import {deriveDeduplicated} from "./misc.js" @@ -177,6 +177,8 @@ export type ItemsByKey = Map export type EventToItem = (event: TrustedEvent) => T +export type GetItem = (key: string, ...args: any[]) => Maybe + export type DeriveItemsByKeyOptions = { getKey: (item: T) => string filters: Filter[] @@ -273,6 +275,76 @@ export const makeDeriveItem = ( } } +// Item loaders + +export type LoadItem = (key: string, ...args: any[]) => Promise + +export const makeForceLoadItem = (loadItem: LoadItem, getItem: GetItem) => { + return (key: string, ...args: any[]) => loadItem(key, ...args).then(() => getItem(key)) +} + +export type MakeLoadItemOptions = { + getFetched?: (key: string) => number + setFetched?: (key: string, ts: number) => void + timeout?: number +} + +export const makeLoadItem = (loadItem: LoadItem, getItem: GetItem, options: MakeLoadItemOptions = {}) => { + const timeout = options.timeout || 3600 + const fetched = new Map() + const getFetched = options.getFetched || ((key: string) => fetched.get(key) || 0) + const setFetched = options.setFetched || ((key: string, ts: number) => fetched.set(key, ts)) + const pending = new Map>>() + const attempts = new Map() + + return async (key: string, ...args: any[]): Promise> => { + const stale = getItem(key) + const fetched = getFetched(key) + + // If we have an item, reload if it's stale + if (stale && fetched > now() - timeout) { + return stale + } + + const pendingItem = pending.get(key) + + // If we already are loading, await and return + if (pendingItem) { + return pendingItem + } + + const attempt = attempts.get(key) || 0 + + // Use exponential backoff to throttle attempts + if (fetched > now() - Math.pow(2, attempt)) { + return stale + } + + attempts.set(key, attempt + 1) + + setFetched(key, now()) + + const promise = loadItem(key, ...args).then(() => getItem(key)) + + pending.set(key, promise) + + let item + try { + item = await promise + } catch (e) { + console.warn(`Failed to load ${name} item ${key}`, e) + } finally { + pending.delete(key) + } + + if (item) { + attempts.delete(key) + } + + return item + } +} + // Miscellaneous other stuff export const deriveEvent = (repository: Repository, idOrAddress: string) =>