Modify loader, apply changes to app stores

packages/app/src/blossom.ts

@@ -1,22 +1,24 @@
 import {BLOSSOM_SERVERS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
-export const blossomServers = deriveEventsMapped<PublishedList>(repository, {
+export const blossomServersByPubkey = deriveItemsByKey({
-  filters: [{kinds: [BLOSSOM_SERVERS]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)),
+  filters: [{kinds: [BLOSSOM_SERVERS]}],
+  getKey: blossomServers => blossomServers.event.pubkey,
 })
 
-export const {
+export const blossomServers = deriveItems(blossomServersByPubkey)
-  indexStore: blossomServersByPubkey,
+
-  deriveItem: deriveBlossomServers,
+export const getBlossomServersByPubkey = getter(blossomServersByPubkey)
-  loadItem: loadBlossomServers,
+
-} = collection({
+export const getBlossomServers = (pubkey: string) => getBlossomServersByPubkey().get(pubkey)
-  name: "blossomServers",
+
-  store: blossomServers,
+export const forceLoadBlossomServers = makeForceLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServers)
-  getKey: blossomServers => blossomServers.event.pubkey,
+
-  load: makeOutboxLoader(BLOSSOM_SERVERS),
+export const loadBlossomServers = makeLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServers)
-})
+
+export const deriveBlossomServers = makeDeriveItem(blossomServersByPubkey, loadBlossomServers)

packages/app/src/follows.ts

@@ -1,22 +1,24 @@
 import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
-export const follows = deriveEventsMapped<PublishedList>(repository, {
+export const followsByPubkey = deriveItemsByKey({
-  filters: [{kinds: [FOLLOWS]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)),
+  filters: [{kinds: [FOLLOWS]}],
+  getKey: follows => follows.event.pubkey,
 })
 
-export const {
+export const follows = deriveItems(followsByPubkey)
-  indexStore: followsByPubkey,
+
-  deriveItem: deriveFollows,
+export const getFollowsByPubkey = getter(followsByPubkey)
-  loadItem: loadFollows,
+
-} = collection({
+export const getFollows = (pubkey: string) => getFollowsByPubkey().get(pubkey)
-  name: "follows",
+
-  store: follows,
+export const forceLoadFollows = makeForceLoadItem(makeOutboxLoader(FOLLOWS), getFollows)
-  getKey: follows => follows.event.pubkey,
+
-  load: makeOutboxLoader(FOLLOWS),
+export const loadFollows = makeLoadItem(makeOutboxLoader(FOLLOWS), getFollows)
-})
+
+export const deriveFollows = makeDeriveItem(followsByPubkey, loadFollows)

packages/app/src/inboxRelaySelections.ts

@@ -1,22 +1,24 @@
 import {INBOX_RELAYS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
-export const inboxRelaySelections = deriveEventsMapped<PublishedList>(repository, {
+export const inboxRelaySelectionsByPubkey = deriveItemsByKey({
-  filters: [{kinds: [INBOX_RELAYS]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)),
+  filters: [{kinds: [INBOX_RELAYS]}],
+  getKey: inboxRelaySelections => inboxRelaySelections.event.pubkey,
 })
 
-export const {
+export const inboxRelaySelections = deriveItems(inboxRelaySelectionsByPubkey)
-  indexStore: inboxRelaySelectionsByPubkey,
+
-  deriveItem: deriveInboxRelaySelections,
+export const getInboxRelaySelectionsByPubkey = getter(inboxRelaySelectionsByPubkey)
-  loadItem: loadInboxRelaySelections,
+
-} = collection({
+export const getInboxRelaySelections = (pubkey: string) => getInboxRelaySelectionsByPubkey().get(pubkey)
-  name: "inboxRelaySelections",
+
-  store: inboxRelaySelections,
+export const forceLoadInboxRelaySelections = makeForceLoadItem(makeOutboxLoader(INBOX_RELAYS), getInboxRelaySelections)
-  getKey: inboxRelaySelections => inboxRelaySelections.event.pubkey,
+
-  load: makeOutboxLoader(INBOX_RELAYS),
+export const loadInboxRelaySelections = makeLoadItem(makeOutboxLoader(INBOX_RELAYS), getInboxRelaySelections)
-})
+
+export const deriveInboxRelaySelections = makeDeriveItem(inboxRelaySelectionsByPubkey, loadInboxRelaySelections)

packages/app/src/mutes.ts

@@ -1,28 +1,30 @@
 import {MUTES, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {ensurePlaintext} from "./plaintext.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
-export const mutes = deriveEventsMapped<PublishedList>(repository, {
+export const mutesByPubkey = deriveItemsByKey({
-  filters: [{kinds: [MUTES]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: async (event: TrustedEvent) =>
     readList(
       asDecryptedEvent(event, {
         content: await ensurePlaintext(event),
       }),
     ),
+  filters: [{kinds: [MUTES]}],
+  getKey: mute => mute.event.pubkey,
 })
 
-export const {
+export const mutes = deriveItems(mutesByPubkey)
-  indexStore: mutesByPubkey,
+
-  deriveItem: deriveMutes,
+export const getMutesByPubkey = getter(mutesByPubkey)
-  loadItem: loadMutes,
+
-} = collection({
+export const getMutes = (pubkey: string) => getMutesByPubkey().get(pubkey)
-  name: "mutes",
+
-  store: mutes,
+export const forceLoadMutes = makeForceLoadItem(makeOutboxLoader(MUTES), getMutes)
-  getKey: mute => mute.event.pubkey,
+
-  load: makeOutboxLoader(MUTES),
+export const loadMutes = makeLoadItem(makeOutboxLoader(MUTES), getMutes)
-})
+
+export const deriveMutes = makeDeriveItem(mutesByPubkey, loadMutes)

packages/app/src/pins.ts

@@ -1,22 +1,24 @@
 import {PINS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
-export const pins = deriveEventsMapped<PublishedList>(repository, {
+export const pinsByPubkey = deriveItemsByKey({
-  filters: [{kinds: [PINS]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)),
+  filters: [{kinds: [PINS]}],
+  getKey: pins => pins.event.pubkey,
 })
 
-export const {
+export const pins = deriveItems(pinsByPubkey)
-  indexStore: pinsByPubkey,
+
-  deriveItem: derivePins,
+export const getPinsByPubkey = getter(pinsByPubkey)
-  loadItem: loadPins,
+
-} = collection({
+export const getPins = (pubkey: string) => getPinsByPubkey().get(pubkey)
-  name: "pins",
+
-  store: pins,
+export const forceLoadPins = makeForceLoadItem(makeOutboxLoader(PINS), getPins)
-  getKey: pins => pins.event.pubkey,
+
-  load: makeOutboxLoader(PINS),
+export const loadPins = makeLoadItem(makeOutboxLoader(PINS), getPins)
-})
+
+export const derivePins = makeDeriveItem(pinsByPubkey, loadPins)

packages/app/src/profiles.ts

@@ -1,6 +1,6 @@
 import {derived, readable} from "svelte/store"
 import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeDeriveItem, getter} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoaderWithIndexers} from "./relaySelections.js"
 
@@ -13,14 +13,16 @@ export const profilesByPubkey = deriveItemsByKey({
 
 export const profiles = deriveItems(profilesByPubkey)
 
-export const loadProfile = makeOutboxLoaderWithIndexers(PROFILE)
-
-export const deriveProfile = makeDeriveItem(profilesByPubkey, loadProfile)
-
 export const getProfilesByPubkey = getter(profilesByPubkey)
 
 export const getProfile = (pubkey: string) => getProfilesByPubkey().get(pubkey)
 
+export const forceLoadProfile = makeForceLoadItem(makeOutboxLoaderWithIndexers(PROFILE), getProfile)
+
+export const loadProfile = makeLoadItem(makeOutboxLoaderWithIndexers(PROFILE), getProfile)
+
+export const deriveProfile = makeDeriveItem(profilesByPubkey, loadProfile)
+
 export const displayProfileByPubkey = (pubkey: string | undefined) =>
   pubkey ? displayProfile(getProfile(pubkey), displayPubkey(pubkey)) : ""
 

packages/app/src/relaySelections.ts

@@ -7,7 +7,7 @@ import {
   TrustedEvent,
   PublishedList,
 } from "@welshman/util"
-import {deriveEventsMapped, collection} from "@welshman/store"
+import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {load, LoadOptions} from "@welshman/net"
 import {Router} from "@welshman/router"
 import {repository} from "./core.js"
@@ -41,19 +41,21 @@ export const makeOutboxLoaderWithIndexers =
     ])
   }
 
-export const relaySelections = deriveEventsMapped<PublishedList>(repository, {
+export const relaySelectionsByPubkey = deriveItemsByKey({
-  filters: [{kinds: [RELAYS]}],
+  repository,
-  itemToEvent: item => item.event,
   eventToItem: (event: TrustedEvent) => readList(asDecryptedEvent(event)),
+  filters: [{kinds: [RELAYS]}],
+  getKey: relaySelections => relaySelections.event.pubkey,
 })
 
-export const {
+export const relaySelections = deriveItems(relaySelectionsByPubkey)
-  indexStore: relaySelectionsByPubkey,
+
-  deriveItem: deriveRelaySelections,
+export const getRelaySelectionsByPubkey = getter(relaySelectionsByPubkey)
-  loadItem: loadRelaySelections,
+
-} = collection({
+export const getRelaySelections = (pubkey: string) => getRelaySelectionsByPubkey().get(pubkey)
-  name: "relaySelections",
+
-  store: relaySelections,
+export const forceLoadRelaySelections = makeForceLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelaySelections)
-  getKey: relaySelections => relaySelections.event.pubkey,
+
-  load: makeOutboxLoaderWithIndexers(RELAYS),
+export const loadRelaySelections = makeLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelaySelections)
-})
+
+export const deriveRelaySelections = makeDeriveItem(relaySelectionsByPubkey, loadRelaySelections)

packages/store/src/index.ts

@@ -1,4 +1,3 @@
-export * from "./synced.js"
 export * from "./misc.js"
-export * from "./loader.js"
+export * from "./synced.js"
 export * from "./repository.js"

packages/store/src/loader.ts

@@ -1,68 +0,0 @@
-import {Maybe, now} from "@welshman/lib"
-
-export type LoaderOptions<T> = {
-  getItem: (key: string) => T
-  getLastFetched: (key: string) => number
-  setLastFetched: (key: string, ts: number) => void
-  load: (key: string, ...args: any[]) => Promise<unknown>
-  timeout?: number
-}
-
-export const makeLoader = <T>(options: LoaderOptions<T>) => {
-  const timeout = options.timeout || 3600
-  const pending = new Map<string, Promise<Maybe<T>>>()
-  const attempts = new Map<string, number>()
-
-  const baseLoad = async (key: string, force: boolean, ...args: any[]): Promise<Maybe<T>> => {
-    const stale = options.getItem(key)
-    const lastFetched = options.getLastFetched(key)
-
-    // If we have an item, reload if it's stale
-    if (stale && lastFetched > now() - timeout && !force) {
-      return stale
-    }
-
-    const pendingItem = pending.get(key)
-
-    // If we already are loading, await and return
-    if (pendingItem) {
-      return pendingItem
-    }
-
-    const attempt = attempts.get(key) || 0
-
-    // Use exponential backoff to throttle attempts
-    if (lastFetched > now() - Math.pow(2, attempt) && !force) {
-      return stale
-    }
-
-    attempts.set(key, attempt + 1)
-
-    options.setLastFetched(key, now())
-
-    const promise = options.load(key, ...args).then(() => options.getItem(key))
-
-    pending.set(key, promise)
-
-    let item
-    try {
-      item = await promise
-    } catch (e) {
-      console.warn(`Failed to load ${name} item ${key}`, e)
-    } finally {
-      pending.delete(key)
-    }
-
-    if (item) {
-      attempts.delete(key)
-    }
-
-    return item
-  }
-
-  const load = (key: string, ...args: any[]) => baseLoad(key, false, ...args)
-
-  const forceLoad = (key: string, ...args: any[]) => baseLoad(key, true, ...args)
-
-  return {load, forceLoad}
-}

packages/store/src/repository.ts

@@ -1,5 +1,5 @@
 import {derived, readable, Readable} from "svelte/store"
-import {on, indexBy, mapPop, Maybe, call, sortBy, first} from "@welshman/lib"
+import {on, now, indexBy, mapPop, Maybe, call, sortBy, first} from "@welshman/lib"
 import {matchFilters, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
 import {Repository, RepositoryUpdate, Tracker} from "@welshman/net"
 import {deriveDeduplicated} from "./misc.js"
@@ -177,6 +177,8 @@ export type ItemsByKey<T> = Map<string, T>
 
 export type EventToItem<T> = (event: TrustedEvent) => T
 
+export type GetItem<T> = (key: string, ...args: any[]) => Maybe<T>
+
 export type DeriveItemsByKeyOptions<T> = {
   getKey: (item: T) => string
   filters: Filter[]
@@ -273,6 +275,76 @@ export const makeDeriveItem = <T>(
   }
 }
 
+// Item loaders
+
+export type LoadItem = (key: string, ...args: any[]) => Promise<unknown>
+
+export const makeForceLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>) => {
+  return (key: string, ...args: any[]) => loadItem(key, ...args).then(() => getItem(key))
+}
+
+export type MakeLoadItemOptions = {
+  getFetched?: (key: string) => number
+  setFetched?: (key: string, ts: number) => void
+  timeout?: number
+}
+
+export const makeLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>, options: MakeLoadItemOptions = {}) => {
+  const timeout = options.timeout || 3600
+  const fetched = new Map<string, number>()
+  const getFetched = options.getFetched || ((key: string) => fetched.get(key) || 0)
+  const setFetched = options.setFetched || ((key: string, ts: number) => fetched.set(key, ts))
+  const pending = new Map<string, Promise<Maybe<T>>>()
+  const attempts = new Map<string, number>()
+
+  return async (key: string, ...args: any[]): Promise<Maybe<T>> => {
+    const stale = getItem(key)
+    const fetched = getFetched(key)
+
+    // If we have an item, reload if it's stale
+    if (stale && fetched > now() - timeout) {
+      return stale
+    }
+
+    const pendingItem = pending.get(key)
+
+    // If we already are loading, await and return
+    if (pendingItem) {
+      return pendingItem
+    }
+
+    const attempt = attempts.get(key) || 0
+
+    // Use exponential backoff to throttle attempts
+    if (fetched > now() - Math.pow(2, attempt)) {
+      return stale
+    }
+
+    attempts.set(key, attempt + 1)
+
+    setFetched(key, now())
+
+    const promise = loadItem(key, ...args).then(() => getItem(key))
+
+    pending.set(key, promise)
+
+    let item
+    try {
+      item = await promise
+    } catch (e) {
+      console.warn(`Failed to load ${name} item ${key}`, e)
+    } finally {
+      pending.delete(key)
+    }
+
+    if (item) {
+      attempts.delete(key)
+    }
+
+    return item
+  }
+}
+
 // Miscellaneous other stuff
 
 export const deriveEvent = (repository: Repository, idOrAddress: string) =>