Move collection to store module

packages/app/src/collection.ts

@@ -1,114 +0,0 @@
-import {readable, derived, type Readable, type Subscriber} from "svelte/store"
-import {indexBy, remove, now} from "@welshman/lib"
-import {ReadableWithGetter, withGetter} from "@welshman/store"
-import {getFreshness, setFreshnessThrottled} from "./freshness.js"
-
-export type CachedLoaderOptions<T> = {
-  name: string
-  indexStore: ReadableWithGetter<Map<string, T>>
-  load: (key: string, relays: string[]) => Promise<any>
-  subscribers?: Subscriber<T>[]
-}
-
-export const makeCachedLoader = <T>({
-  name,
-  load,
-  indexStore,
-  subscribers = [],
-}: CachedLoaderOptions<T>) => {
-  const pending = new Map<string, Promise<T | void>>()
-  const loadAttempts = new Map<string, number>()
-
-  return async (key: string, relays: string[] = []) => {
-    const stale = indexStore.get().get(key)
-
-    // If we have no loader function, nothing we can do
-    if (!load) {
-      return stale
-    }
-
-    const freshness = getFreshness(name, key)
-
-    // If we have an item, reload if it's stale
-    if (stale && freshness > now() - 3600) {
-      return stale
-    }
-
-    // If we already are loading, await and return
-    if (pending.has(key)) {
-      return pending.get(key)!.then(() => indexStore.get().get(key))
-    }
-
-    const attempt = loadAttempts.get(key) || 0
-
-    // Use exponential backoff to throttle attempts
-    if (freshness > now() - Math.pow(2, attempt)) {
-      return stale
-    }
-
-    loadAttempts.set(key, attempt + 1)
-
-    setFreshnessThrottled({ns: name, key, ts: now()})
-
-    const promise = load(key, relays)
-
-    pending.set(key, promise)
-
-    try {
-      await promise
-    } catch (e) {
-      console.warn(`Failed to load ${name} item ${key}`, e)
-    } finally {
-      pending.delete(key)
-    }
-
-    const fresh = indexStore.get().get(key)
-
-    if (fresh) {
-      loadAttempts.delete(key)
-
-      for (const subscriber of subscribers) {
-        subscriber(fresh)
-      }
-    }
-
-    return fresh
-  }
-}
-
-export type CollectionOptions<T> = {
-  name: string
-  store: Readable<T[]>
-  getKey: (item: T) => string
-  load: (key: string, relays: string[]) => Promise<any>
-}
-
-export const collection = <T>({name, store, getKey, load}: CollectionOptions<T>) => {
-  const indexStore = withGetter(derived(store, $items => indexBy(getKey, $items)))
-
-  let subscribers: Subscriber<T>[] = []
-
-  const loadItem = makeCachedLoader({name, load, indexStore, subscribers})
-
-  const deriveItem = (key: string | undefined, relays: string[] = []) => {
-    if (!key) {
-      return readable(undefined)
-    }
-
-    // If we don't yet have the item, or it's stale, trigger a request for it. The derived
-    // store will update when it arrives
-    loadItem(key, relays)
-
-    return derived(indexStore, $index => $index.get(key))
-  }
-
-  const onItem = (cb: Subscriber<T>) => {
-    subscribers.push(cb)
-
-    return () => {
-      subscribers = remove(cb, subscribers)
-    }
-  }
-
-  return {indexStore, deriveItem, loadItem, onItem}
-}

packages/app/src/follows.ts

@@ -1,8 +1,7 @@
 import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const follows = deriveEventsMapped<PublishedList>(repository, {

packages/app/src/freshness.ts

@@ -1,29 +0,0 @@
-import {writable} from "svelte/store"
-import {assoc, batch} from "@welshman/lib"
-import {withGetter} from "@welshman/store"
-
-export type FreshnessUpdate = {
-  ns: string
-  key: string
-  ts: number
-}
-
-export const freshness = withGetter(writable<Record<string, number>>({}))
-
-export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}`
-
-export const getFreshness = (ns: string, key: string) =>
-  freshness.get()[getFreshnessKey(ns, key)] || 0
-
-export const setFreshnessImmediate = ({ns, key, ts}: FreshnessUpdate) =>
-  freshness.update(assoc(getFreshnessKey(ns, key), ts))
-
-export const setFreshnessThrottled = batch(100, (updates: FreshnessUpdate[]) =>
-  freshness.update($freshness => {
-    for (const {ns, key, ts} of updates) {
-      $freshness[getFreshnessKey(ns, key)] = ts
-    }
-
-    return $freshness
-  }),
-)

packages/app/src/handles.ts

@@ -1,6 +1,6 @@
 import {writable, derived} from "svelte/store"
 import {tryCatch, fetchJson, uniq, batcher, postJson, last} from "@welshman/lib"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {deriveProfile} from "./profiles.js"
 import {appContext} from "./context.js"
 

packages/app/src/index.ts

@@ -1,9 +1,7 @@
 export * from "./context.js"
 export * from "./core.js"
-export * from "./collection.js"
 export * from "./commands.js"
 export * from "./feeds.js"
-export * from "./freshness.js"
 export * from "./follows.js"
 export * from "./handles.js"
 export * from "./mutes.js"

packages/app/src/mutes.ts

@@ -1,8 +1,7 @@
 import {MUTES, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {ensurePlaintext} from "./plaintext.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 

packages/app/src/pins.ts

@@ -1,8 +1,7 @@
 import {PINS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const pins = deriveEventsMapped<PublishedList>(repository, {

packages/app/src/profiles.ts

@@ -1,9 +1,8 @@
 import {derived, readable} from "svelte/store"
 import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util"
 import {PublishedProfile} from "@welshman/util"
-import {deriveEventsMapped, withGetter} from "@welshman/store"
+import {deriveEventsMapped, collection, withGetter} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const profiles = withGetter(

packages/app/src/relaySelections.ts

@@ -1,19 +1,11 @@
-import {derived} from 'svelte/store'
-import {uniq, batcher, always} from "@welshman/lib"
-import {
-  INBOX_RELAYS,
-  RELAYS,
-  normalizeRelayUrl,
-  asDecryptedEvent,
-  readList,
-  getRelaysFromList,
-} from "@welshman/util"
-import {TrustedEvent, PublishedList, RelayMode, List} from "@welshman/util"
+import {derived} from "svelte/store"
+import {batcher, always} from "@welshman/lib"
+import {INBOX_RELAYS, RELAYS, asDecryptedEvent, readList, getRelaysFromList} from "@welshman/util"
+import {TrustedEvent, PublishedList, RelayMode} from "@welshman/util"
 import {request} from "@welshman/net"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {Router} from "@welshman/router"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 
 export type OutboxLoaderRequest = {
   pubkey: string

packages/app/src/relays.ts

@@ -24,7 +24,7 @@ import {
   isRelayUrl,
 } from "@welshman/util"
 import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {appContext} from "./context.js"
 
 export type RelayStats = {

packages/app/src/storageAdapters.ts

@@ -12,13 +12,13 @@ import {
 } from "@welshman/util"
 import {throttled, withGetter} from "@welshman/store"
 import {Tracker} from "@welshman/net"
+import {freshness} from "@welshman/store"
 import {Repository, RepositoryUpdate} from "@welshman/relay"
 import {getAll, bulkPut, bulkDelete} from "./storage.js"
 import {relays} from "./relays.js"
 import {handles, onHandle} from "./handles.js"
 import {zappers, onZapper} from "./zappers.js"
 import {plaintext} from "./plaintext.js"
-import {freshness} from "./freshness.js"
 import {repository, tracker} from "./core.js"
 import {sessions} from "./session.js"
 import {userFollows} from "./user.js"

packages/app/src/zappers.ts

@@ -10,7 +10,7 @@ import {
   batcher,
   postJson,
 } from "@welshman/lib"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {deriveProfile} from "./profiles.js"
 import {appContext} from "./context.js"