Fix some bugs with deriving events by url

2025-11-24 09:52:47 -08:00
parent 2fec078a5b
commit 250f5c772f
20 changed files with 455 additions and 360 deletions
@@ -1,240 +0,0 @@
-import {describe, it, expect, beforeEach, vi, afterEach} from "vitest"
-import {get, writable} from "svelte/store"
-import {now, always} from "@welshman/lib"
-import {collection, freshness, setFreshnessImmediate} from "../src/collection"
-
-describe("collection", () => {
-  beforeEach(() => {
-    vi.useFakeTimers()
-    vi.clearAllMocks()
-  })
-
-  afterEach(() => {
-    vi.resetModules()
-    vi.useRealTimers()
-    freshness.set({})
-  })
-
-  describe("basic functionality", () => {
-    it("should create a collection with indexStore", () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      expect(col.indexStore.get().get("1")).toEqual(items[0])
-    })
-
-    it("should update indexStore when store changes", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const newItem = {id: "1", value: "test"}
-      store.set([newItem])
-
-      expect(get(col.indexStore).get("1")).toEqual(newItem)
-    })
-  })
-
-  describe("loadItem", () => {
-    it("should return stale item if no loader provided", async () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toEqual(items[0])
-    })
-
-    it("should return undefined for non-existent items when no loader provided", async () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-
-    it("should use loader to fetch new items", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-
-    it("should handle concurrent loading of the same item", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // Start multiple concurrent loads
-      const loads = Promise.all([col.loadItem("1"), col.loadItem("1"), col.loadItem("1")])
-
-      await loads
-      // Should only call load once
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-
-    it("should respect freshness checks", async () => {
-      await vi.advanceTimersByTimeAsync(1000)
-      const store = writable<Array<{id: string; value: string}>>([{id: "1", value: "stale"}])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      // force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-      await col.loadItem("1")
-      // Should not call load because item is fresh
-      expect(mockLoad).toHaveBeenCalledTimes(0)
-    })
-
-    it("should reload stale items", async () => {
-      const mockLoad = vi.fn()
-      const store = writable([{id: "1", value: "test"}])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: (item: any) => item.id,
-        load: mockLoad,
-      })
-
-      // load the item to set freshness
-      await col.loadItem("1")
-
-      await vi.advanceTimersByTimeAsync(4000 * 1000)
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(2)
-    })
-
-    it("should implement exponential backoff for failed attempts", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue(undefined)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // First attempt
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-
-      //force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-
-      // Immediate retry should be throttled
-      await col.loadItem("1").catch(() => {})
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-  })
-
-  describe("deriveItem", () => {
-    it("should return readable undefined for null keys", () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem(undefined)
-      expect(get(derived)).toBeUndefined()
-    })
-
-    it("should create a derived store that updates with the source", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem("1")
-      expect(get(derived)).toBeUndefined()
-
-      // Update source store
-      store.set([{id: "1", value: "test"}])
-      expect(get(derived)).toEqual({id: "1", value: "test"})
-    })
-
-    it("should trigger load when deriving non-existent item", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      col.deriveItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-  })
-
-  describe("error handling", () => {
-    it("should handle loader failures gracefully", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn(() => {
-        return Promise.reject("load failed")
-      })
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-  })
-})
@@ -3,9 +3,7 @@ import {Repository} from "@welshman/net"
 import {get} from "svelte/store"
 import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"
 import {
-  custom,
  deriveEvents,
-  deriveEventsMapped,
  deriveIsDeleted,
  getter,
  synced,
@@ -1,5 +1,5 @@
-import {derived, readable, Readable} from "svelte/store"
-import {on, assoc, now, indexBy, mapPop, Maybe, MaybeAsync, call, sortBy, first} from "@welshman/lib"
+import {readable, Readable} from "svelte/store"
+import {on, assoc, now, mapPop, Maybe, MaybeAsync, call, sortBy, first} from "@welshman/lib"
 import {matchFilters, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
 import {Repository, RepositoryUpdate, Tracker} from "@welshman/net"
 import {deriveDeduplicated} from "./misc.js"
@@ -19,9 +19,15 @@ export const deriveEventsById = ({
  repository,
  includeDeleted,
 }: DeriveEventsByIdOptions) => {
-  const eventsById: EventsById = indexBy(e => e.id, repository.query(filters, {includeDeleted}))
+  const eventsById = new Map<string, TrustedEvent>()

  return readable(eventsById, set => {
+    for (const event of repository.query(filters, {includeDeleted})) {
+      eventsById.set(event.id, event)
+    }
+
+    set(eventsById)
+
    return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
      let dirty = false

@@ -45,8 +51,8 @@ export const deriveEventsById = ({
  })
 }

-export const deriveEvents = (eventsByIdStore: Readable<EventsById>) =>
-  deriveDeduplicated(eventsByIdStore, eventsById => Array.from(eventsById.values()))
+export const deriveArray = <T>(itemsByIdStore: Readable<Map<string, T>>) =>
+  deriveDeduplicated(itemsByIdStore, itemsById => Array.from(itemsById.values()))

 export const deriveEventsAsc = (eventsByIdStore: Readable<EventsById>) =>
  deriveDeduplicated(eventsByIdStore, eventsById => sortBy(e => e.created_at, eventsById.values()))
@@ -54,8 +60,158 @@ export const deriveEventsAsc = (eventsByIdStore: Readable<EventsById>) =>
 export const deriveEventsDesc = (eventsByIdStore: Readable<EventsById>) =>
  deriveDeduplicated(eventsByIdStore, eventsById => sortBy(e => -e.created_at, eventsById.values()))

+export type DeriveEventOptions = {
+  repository: Repository
+  includeDeleted?: boolean
+  onDerive?: (filters: Filter[], ...args: any[]) => void
+}
+
+export const makeDeriveEvent = ({
+  repository,
+  includeDeleted = false,
+  onDerive,
+}: DeriveEventOptions) => {
+  return (idOrAddress: string, ...args: any[]) => {
+    const filters = getIdFilters([idOrAddress])
+
+    onDerive?.(filters, ...args)
+
+    return readable<Maybe<TrustedEvent>>(undefined, set => {
+      const event = first(repository.query(filters, {includeDeleted}))
+
+      set(event)
+
+      return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+        for (const event of added) {
+          if (matchFilters(filters, event)) {
+            set(event)
+          }
+        }
+
+        for (const id of removed) {
+          if (event?.id === id) {
+            set(undefined)
+          }
+        }
+      })
+    })
+  }
+}
+
 // Events by id by url

+export type EventsByIdByUrl = Map<string, EventsById>
+
+export type DeriveEventsByIdByUrlOptions = DeriveEventsByIdOptions & {
+  tracker: Tracker
+}
+
+export const deriveEventsByIdByUrl = ({
+  filters,
+  tracker,
+  repository,
+  includeDeleted,
+}: DeriveEventsByIdByUrlOptions) => {
+  const eventsByIdByUrl: EventsByIdByUrl = new Map()
+
+  const addEvent = (url: string, event: TrustedEvent) => {
+    if (!matchFilters(filters, event)) return false
+
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (eventsById?.has(event.id)) return false
+
+    // Create a new map so we can detect which key changed
+    const newEventsById = new Map(eventsById)
+
+    newEventsById.set(event.id, event)
+    eventsByIdByUrl.set(url, newEventsById)
+
+    return true
+  }
+
+  const removeEvent = (url: string, id: string) => {
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (eventsById?.has(id)) {
+      eventsById.delete(id)
+
+      if (eventsById.size === 0) {
+        eventsByIdByUrl.delete(url)
+      } else {
+        // Create a new map so we can detect which key changed
+        eventsByIdByUrl.set(url, new Map(eventsById))
+      }
+
+      return true
+    }
+
+    return false
+  }
+
+  return readable(eventsByIdByUrl, set => {
+    for (const event of repository.query(filters, {includeDeleted})) {
+      for (const url of tracker.getRelays(event.id)) {
+        addEvent(url, event)
+      }
+    }
+
+    set(eventsByIdByUrl)
+
+    const unsubscribers = [
+      on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+        let dirty = false
+
+        for (const event of added) {
+          for (const url of tracker.getRelays(event.id)) {
+            dirty = dirty || addEvent(url, event)
+          }
+        }
+
+        for (const id of removed) {
+          for (const url of tracker.getRelays(id)) {
+            dirty = dirty || removeEvent(url, id)
+          }
+        }
+
+        if (dirty) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "add", (id: string, url: string) => {
+        const event = repository.getEvent(id)
+
+        if (event && addEvent(url, event)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "remove", (id: string, url: string) => {
+        if (removeEvent(url, id)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "load", () => {
+        eventsByIdByUrl.clear()
+
+        for (const event of repository.query(filters, {includeDeleted})) {
+          for (const url of tracker.getRelays(event.id)) {
+            addEvent(url, event)
+          }
+        }
+
+        set(eventsByIdByUrl)
+      }),
+      on(tracker, "clear", () => {
+        eventsByIdByUrl.clear()
+
+        set(eventsByIdByUrl)
+      }),
+    ]
+
+    return () => unsubscribers.forEach(call)
+  })
+}
+
 export type DeriveEventsByIdForUrlOptions = DeriveEventsByIdOptions & {
  url: string
  tracker: Tracker
@@ -72,7 +228,7 @@ export const deriveEventsByIdForUrl = ({

  const initialize = () => {
    const initialIds = Array.from(tracker.getIds(url))
-    const initialFilters = filters.map(assoc('ids', initialIds))
+    const initialFilters = filters.map(assoc("ids", initialIds))

    for (const event of repository.query(initialFilters, {includeDeleted})) {
      eventsById.set(event.id, event)
@@ -82,20 +238,24 @@ export const deriveEventsByIdForUrl = ({
  }

  return readable(initialize(), set => {
+    set(initialize())
+
    const unsubscribers = [
      on(repository, "update", ({added, removed}: RepositoryUpdate) => {
        let dirty = false

        for (const event of added) {
-          if (tracker.hasRelay(event.id, url) && !eventsById.has(event.id)) {
+          if (tracker.hasRelay(event.id, url) && matchFilters(filters, event)) {
            eventsById.set(event.id, event)
            dirty = true
          }
        }

        for (const id of removed) {
-          eventsById.delete(id)
-          dirty = true
+          if (eventsById.has(id)) {
+            eventsById.delete(id)
+            dirty = true
+          }
        }

        if (dirty) {
@@ -105,7 +265,7 @@ export const deriveEventsByIdForUrl = ({
      on(tracker, "add", (id: string, url: string) => {
        const event = repository.getEvent(id)

-        if (event && tracker.hasRelay(id, url) && !eventsById.has(id)) {
+        if (event && tracker.hasRelay(id, url) && matchFilters(filters, event)) {
          eventsById.set(id, event)
          set(eventsById)
        }
@@ -118,9 +278,7 @@ export const deriveEventsByIdForUrl = ({
      }),
      on(tracker, "load", () => {
        eventsById.clear()
-        initialize()
-
-        set(eventsById)
+        set(initialize())
      }),
      on(tracker, "clear", () => {
        eventsById.clear()
@@ -253,7 +411,11 @@ export type MakeLoadItemOptions = {
  timeout?: number
 }

-export const makeLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>, options: MakeLoadItemOptions = {}) => {
+export const makeLoadItem = <T>(
+  loadItem: LoadItem,
+  getItem: GetItem<T>,
+  options: MakeLoadItemOptions = {},
+) => {
  const timeout = options.timeout || 3600
  const fetched = new Map<string, number>()
  const getFetched = options.getFetched || ((key: string) => fetched.get(key) || 0)
@@ -311,18 +473,10 @@ export const makeLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>, options

 // Miscellaneous other stuff

-export const deriveEvent = (repository: Repository, idOrAddress: string) =>
-  derived(
-    deriveEventsById({
-      repository,
-      filters: getIdFilters([idOrAddress]),
-      includeDeleted: true,
-    }),
-    $m => first($m.values()),
-  )
-
 export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) =>
-  readable(repository.isDeleted(event), set => {
+  readable(false, set => {
+    set(repository.isDeleted(event))
+
    const unsubscribe = on(repository, "update", ({removed}: RepositoryUpdate) => {
      if (removed.has(event.id)) {
        set(true)