Move collection to store module

packages/app/__tests__/collection.test.ts

@@ -1,241 +0,0 @@
-import {describe, it, expect, beforeEach, vi, afterEach} from "vitest"
-import {get, writable} from "svelte/store"
-import {now, always} from "@welshman/lib"
-import {collection} from "../src/collection"
-import {freshness, setFreshnessImmediate} from "../src/freshness"
-
-describe("collection", () => {
-  beforeEach(() => {
-    vi.useFakeTimers()
-    vi.clearAllMocks()
-  })
-
-  afterEach(() => {
-    vi.resetModules()
-    vi.useRealTimers()
-    freshness.set({})
-  })
-
-  describe("basic functionality", () => {
-    it("should create a collection with indexStore", () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      expect(col.indexStore.get().get("1")).toEqual(items[0])
-    })
-
-    it("should update indexStore when store changes", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const newItem = {id: "1", value: "test"}
-      store.set([newItem])
-
-      expect(get(col.indexStore).get("1")).toEqual(newItem)
-    })
-  })
-
-  describe("loadItem", () => {
-    it("should return stale item if no loader provided", async () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toEqual(items[0])
-    })
-
-    it("should return undefined for non-existent items when no loader provided", async () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-
-    it("should use loader to fetch new items", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-
-    it("should handle concurrent loading of the same item", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // Start multiple concurrent loads
-      const loads = Promise.all([col.loadItem("1"), col.loadItem("1"), col.loadItem("1")])
-
-      await loads
-      // Should only call load once
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-
-    it("should respect freshness checks", async () => {
-      await vi.advanceTimersByTimeAsync(1000)
-      const store = writable<Array<{id: string; value: string}>>([{id: "1", value: "stale"}])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      // force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-      await col.loadItem("1")
-      // Should not call load because item is fresh
-      expect(mockLoad).toHaveBeenCalledTimes(0)
-    })
-
-    it("should reload stale items", async () => {
-      const mockLoad = vi.fn()
-      const store = writable([{id: "1", value: "test"}])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: (item: any) => item.id,
-        load: mockLoad,
-      })
-
-      // load the item to set freshness
-      await col.loadItem("1")
-
-      await vi.advanceTimersByTimeAsync(4000 * 1000)
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(2)
-    })
-
-    it("should implement exponential backoff for failed attempts", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue(undefined)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // First attempt
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-
-      //force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-
-      // Immediate retry should be throttled
-      await col.loadItem("1").catch(() => {})
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-  })
-
-  describe("deriveItem", () => {
-    it("should return readable undefined for null keys", () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem(undefined)
-      expect(get(derived)).toBeUndefined()
-    })
-
-    it("should create a derived store that updates with the source", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem("1")
-      expect(get(derived)).toBeUndefined()
-
-      // Update source store
-      store.set([{id: "1", value: "test"}])
-      expect(get(derived)).toEqual({id: "1", value: "test"})
-    })
-
-    it("should trigger load when deriving non-existent item", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      col.deriveItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-  })
-
-  describe("error handling", () => {
-    it("should handle loader failures gracefully", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn(() => {
-        return Promise.reject("load failed")
-      })
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-  })
-})

packages/app/src/collection.ts

@@ -1,114 +0,0 @@
-import {readable, derived, type Readable, type Subscriber} from "svelte/store"
-import {indexBy, remove, now} from "@welshman/lib"
-import {ReadableWithGetter, withGetter} from "@welshman/store"
-import {getFreshness, setFreshnessThrottled} from "./freshness.js"
-
-export type CachedLoaderOptions<T> = {
-  name: string
-  indexStore: ReadableWithGetter<Map<string, T>>
-  load: (key: string, relays: string[]) => Promise<any>
-  subscribers?: Subscriber<T>[]
-}
-
-export const makeCachedLoader = <T>({
-  name,
-  load,
-  indexStore,
-  subscribers = [],
-}: CachedLoaderOptions<T>) => {
-  const pending = new Map<string, Promise<T | void>>()
-  const loadAttempts = new Map<string, number>()
-
-  return async (key: string, relays: string[] = []) => {
-    const stale = indexStore.get().get(key)
-
-    // If we have no loader function, nothing we can do
-    if (!load) {
-      return stale
-    }
-
-    const freshness = getFreshness(name, key)
-
-    // If we have an item, reload if it's stale
-    if (stale && freshness > now() - 3600) {
-      return stale
-    }
-
-    // If we already are loading, await and return
-    if (pending.has(key)) {
-      return pending.get(key)!.then(() => indexStore.get().get(key))
-    }
-
-    const attempt = loadAttempts.get(key) || 0
-
-    // Use exponential backoff to throttle attempts
-    if (freshness > now() - Math.pow(2, attempt)) {
-      return stale
-    }
-
-    loadAttempts.set(key, attempt + 1)
-
-    setFreshnessThrottled({ns: name, key, ts: now()})
-
-    const promise = load(key, relays)
-
-    pending.set(key, promise)
-
-    try {
-      await promise
-    } catch (e) {
-      console.warn(`Failed to load ${name} item ${key}`, e)
-    } finally {
-      pending.delete(key)
-    }
-
-    const fresh = indexStore.get().get(key)
-
-    if (fresh) {
-      loadAttempts.delete(key)
-
-      for (const subscriber of subscribers) {
-        subscriber(fresh)
-      }
-    }
-
-    return fresh
-  }
-}
-
-export type CollectionOptions<T> = {
-  name: string
-  store: Readable<T[]>
-  getKey: (item: T) => string
-  load: (key: string, relays: string[]) => Promise<any>
-}
-
-export const collection = <T>({name, store, getKey, load}: CollectionOptions<T>) => {
-  const indexStore = withGetter(derived(store, $items => indexBy(getKey, $items)))
-
-  let subscribers: Subscriber<T>[] = []
-
-  const loadItem = makeCachedLoader({name, load, indexStore, subscribers})
-
-  const deriveItem = (key: string | undefined, relays: string[] = []) => {
-    if (!key) {
-      return readable(undefined)
-    }
-
-    // If we don't yet have the item, or it's stale, trigger a request for it. The derived
-    // store will update when it arrives
-    loadItem(key, relays)
-
-    return derived(indexStore, $index => $index.get(key))
-  }
-
-  const onItem = (cb: Subscriber<T>) => {
-    subscribers.push(cb)
-
-    return () => {
-      subscribers = remove(cb, subscribers)
-    }
-  }
-
-  return {indexStore, deriveItem, loadItem, onItem}
-}

packages/app/src/follows.ts

@@ -1,8 +1,7 @@
 import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const follows = deriveEventsMapped<PublishedList>(repository, {

packages/app/src/freshness.ts

@@ -1,29 +0,0 @@
-import {writable} from "svelte/store"
-import {assoc, batch} from "@welshman/lib"
-import {withGetter} from "@welshman/store"
-
-export type FreshnessUpdate = {
-  ns: string
-  key: string
-  ts: number
-}
-
-export const freshness = withGetter(writable<Record<string, number>>({}))
-
-export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}`
-
-export const getFreshness = (ns: string, key: string) =>
-  freshness.get()[getFreshnessKey(ns, key)] || 0
-
-export const setFreshnessImmediate = ({ns, key, ts}: FreshnessUpdate) =>
-  freshness.update(assoc(getFreshnessKey(ns, key), ts))
-
-export const setFreshnessThrottled = batch(100, (updates: FreshnessUpdate[]) =>
-  freshness.update($freshness => {
-    for (const {ns, key, ts} of updates) {
-      $freshness[getFreshnessKey(ns, key)] = ts
-    }
-
-    return $freshness
-  }),
-)

packages/app/src/handles.ts

@@ -1,6 +1,6 @@
 import {writable, derived} from "svelte/store"
 import {tryCatch, fetchJson, uniq, batcher, postJson, last} from "@welshman/lib"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {deriveProfile} from "./profiles.js"
 import {appContext} from "./context.js"
 

packages/app/src/index.ts

@@ -1,9 +1,7 @@
 export * from "./context.js"
 export * from "./core.js"
-export * from "./collection.js"
 export * from "./commands.js"
 export * from "./feeds.js"
-export * from "./freshness.js"
 export * from "./follows.js"
 export * from "./handles.js"
 export * from "./mutes.js"

packages/app/src/mutes.ts

@@ -1,8 +1,7 @@
 import {MUTES, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {ensurePlaintext} from "./plaintext.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 

packages/app/src/pins.ts

@@ -1,8 +1,7 @@
 import {PINS, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const pins = deriveEventsMapped<PublishedList>(repository, {

packages/app/src/profiles.ts

@@ -1,9 +1,8 @@
 import {derived, readable} from "svelte/store"
 import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util"
 import {PublishedProfile} from "@welshman/util"
-import {deriveEventsMapped, withGetter} from "@welshman/store"
+import {deriveEventsMapped, collection, withGetter} from "@welshman/store"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 import {makeOutboxLoader} from "./relaySelections.js"
 
 export const profiles = withGetter(

packages/app/src/relaySelections.ts

@@ -1,19 +1,11 @@
-import {derived} from 'svelte/store'
-import {uniq, batcher, always} from "@welshman/lib"
-import {
-  INBOX_RELAYS,
-  RELAYS,
-  normalizeRelayUrl,
-  asDecryptedEvent,
-  readList,
-  getRelaysFromList,
-} from "@welshman/util"
-import {TrustedEvent, PublishedList, RelayMode, List} from "@welshman/util"
+import {derived} from "svelte/store"
+import {batcher, always} from "@welshman/lib"
+import {INBOX_RELAYS, RELAYS, asDecryptedEvent, readList, getRelaysFromList} from "@welshman/util"
+import {TrustedEvent, PublishedList, RelayMode} from "@welshman/util"
 import {request} from "@welshman/net"
-import {deriveEventsMapped} from "@welshman/store"
+import {deriveEventsMapped, collection} from "@welshman/store"
 import {Router} from "@welshman/router"
 import {repository} from "./core.js"
-import {collection} from "./collection.js"
 
 export type OutboxLoaderRequest = {
   pubkey: string

packages/app/src/relays.ts

@@ -24,7 +24,7 @@ import {
   isRelayUrl,
 } from "@welshman/util"
 import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {appContext} from "./context.js"
 
 export type RelayStats = {

packages/app/src/storageAdapters.ts

@@ -12,13 +12,13 @@ import {
 } from "@welshman/util"
 import {throttled, withGetter} from "@welshman/store"
 import {Tracker} from "@welshman/net"
+import {freshness} from "@welshman/store"
 import {Repository, RepositoryUpdate} from "@welshman/relay"
 import {getAll, bulkPut, bulkDelete} from "./storage.js"
 import {relays} from "./relays.js"
 import {handles, onHandle} from "./handles.js"
 import {zappers, onZapper} from "./zappers.js"
 import {plaintext} from "./plaintext.js"
-import {freshness} from "./freshness.js"
 import {repository, tracker} from "./core.js"
 import {sessions} from "./session.js"
 import {userFollows} from "./user.js"

packages/app/src/zappers.ts

@@ -10,7 +10,7 @@ import {
   batcher,
   postJson,
 } from "@welshman/lib"
-import {collection} from "./collection.js"
+import {collection} from "@welshman/store"
 import {deriveProfile} from "./profiles.js"
 import {appContext} from "./context.js"