Re-work repository derivation

packages/store/src/custom.ts

@@ -1,65 +0,0 @@
-import {Subscriber, Unsubscriber} from "svelte/store"
-import {throttle} from "@welshman/lib"
-import {WritableWithGetter} from "./getter.js"
-
-type Start<T> = (set: Subscriber<T>) => Unsubscriber
-
-export type CustomStoreOpts<T> = {
-  throttle?: number
-  onUpdate?: (x: T) => void
-}
-
-export const custom = <T>(
-  start: Start<T>,
-  opts: CustomStoreOpts<T> = {},
-): WritableWithGetter<T> => {
-  const subs: Subscriber<T>[] = []
-
-  let value: T
-  let stop: () => void
-
-  const set = (newValue: T) => {
-    for (const sub of subs) {
-      sub(newValue)
-    }
-
-    value = newValue
-  }
-
-  return {
-    get: () => value,
-    set: (newValue: T) => {
-      set(newValue)
-      opts.onUpdate?.(newValue)
-    },
-    update: (f: (value: T) => T) => {
-      const newValue = f(value)
-
-      set(newValue)
-      opts.onUpdate?.(newValue)
-    },
-    subscribe: (sub: Subscriber<T>) => {
-      if (opts.throttle) {
-        sub = throttle(opts.throttle, sub)
-      }
-
-      if (subs.length === 0) {
-        stop = start(set)
-      }
-
-      subs.push(sub)
-      sub(value)
-
-      return () => {
-        subs.splice(
-          subs.findIndex(s => s === sub),
-          1,
-        )
-
-        if (subs.length === 0) {
-          stop()
-        }
-      }
-    },
-  }
-}

packages/store/src/index.ts

@@ -2,6 +2,5 @@ export * from "./synced.js"
 export * from "./getter.js"
 export * from "./throttle.js"
 export * from "./memoize.js"
-export * from "./custom.js"
 export * from "./repository.js"
 export * from "./collection.js"

packages/store/src/memoize.ts

@@ -1,4 +1,4 @@
-import {Readable, Subscriber} from "svelte/store"
+import {derived, Readable, Subscriber, Stores, StoresValues} from "svelte/store"
 import {memoize} from "@welshman/lib"
 
 export const memoized = <T>(store: Readable<T>) => {
@@ -6,3 +6,19 @@ export const memoized = <T>(store: Readable<T>) => {
 
   return {...store, subscribe: (f: Subscriber<T>) => subscribe(memoize(f))}
 }
+
+export const deriveDeduplicated = <S extends Stores, T>(
+  stores: S,
+  get: (storeValues: StoresValues<S>) => T,
+): Readable<T> => {
+  let prev: T
+
+  return derived(stores, (storeValues, set) => {
+    const result = get(storeValues)
+
+    if (prev !== result) {
+      prev = result
+      set(result)
+    }
+  })
+}

packages/store/src/repository.ts

@@ -1,5 +1,10 @@
-import {derived} from "svelte/store"
+import {derived, readable, Readable} from "svelte/store"
 import {
+  on,
+  indexBy,
+  mapPop,
+  Maybe,
+  call,
   sortBy,
   identity,
   ensurePlural,
@@ -9,164 +14,286 @@ import {
   first,
 } from "@welshman/lib"
 import {matchFilters, getIdAndAddress, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
-import {Repository} from "@welshman/net"
+import {Repository, RepositoryUpdate, Tracker} from "@welshman/net"
-import {custom} from "./custom.js"
+import {deriveDeduplicated} from './memoize.js'
 
-export type DeriveEventsMappedOptions<T> = {
+// Events by id
+
+export type EventsById = Map<string, TrustedEvent>
+
+export type DeriveEventsByIdOptions = {
   filters: Filter[]
-  eventToItem: (event: TrustedEvent) => undefined | T | T[] | Promise<undefined | T | T[]>
+  repository: Repository
-  itemToEvent: (item: T) => TrustedEvent
-  throttle?: number
   includeDeleted?: boolean
 }
 
-export const deriveEventsMapped = <T>(
+export const deriveEventsById = ({
-  repository: Repository,
+  filters,
-  {
+  repository,
-    filters,
+  includeDeleted,
-    eventToItem,
+}: DeriveEventsByIdOptions) => {
-    itemToEvent,
+  const eventsById: EventsById = indexBy(e => e.id, repository.query(filters, {includeDeleted}))
-    throttle = 0,
-    includeDeleted = false,
-  }: DeriveEventsMappedOptions<T>,
-) =>
-  custom<T[]>(
-    setter => {
-      let data: T[] = []
-      const deferred = new Set()
 
-      const defer = (event: TrustedEvent, promise: Promise<undefined | T | T[]>) => {
+  return readable(eventsById, set => {
-        deferred.add(event.id)
+    return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+      let dirty = false
 
-        void promise.then(items => {
+      for (const event of added) {
-          if (deferred.has(event.id)) {
+        if (matchFilters(filters, event)) {
-            deferred.delete(event.id)
+          dirty = true
-
+          eventsById.set(event.id, event)
-            for (const item of removeUndefined(ensurePlural(items))) {
-              data.push(item)
-            }
-
-            setter(sortBy(item => -itemToEvent(item).created_at, data))
-          }
-        })
-      }
-
-      for (const event of repository.query(filters, {includeDeleted})) {
-        const items = eventToItem(event)
-
-        if (!items) {
-          continue
-        }
-
-        if (items instanceof Promise) {
-          defer(event, items)
-        } else {
-          for (const item of removeUndefined(ensurePlural(items))) {
-            data.push(item)
-          }
         }
       }
 
-      setter(sortBy(item => -itemToEvent(item).created_at, data))
+      for (const id of removed) {
-
+        if (mapPop(id, eventsById)) {
-      const onUpdate = batch(300, (updates: {added: TrustedEvent[]; removed: Set<string>}[]) => {
+          dirty = true
-        const removed = new Set()
-        const added = new Map()
-
-        // Apply updates in order
-        for (const update of updates) {
-          for (const event of update.added.values()) {
-            added.set(event.id, event)
-            removed.delete(event.id)
-          }
-
-          for (const id of update.removed) {
-            removed.add(id)
-            added.delete(id)
-            deferred.delete(id)
-          }
         }
+      }
 
+      if (dirty) {
+        set(eventsById)
+      }
+    })
+  })
+}
+
+export const deriveEvents = (eventsByIdStore: Readable<EventsById>) =>
+  deriveDeduplicated(eventsByIdStore, eventsById => Array.from(eventsById.values()))
+
+export const deriveEventsAsc = (eventsStore: Readable<TrustedEvent[]>) =>
+  deriveDeduplicated(eventsStore, events => sortBy(e => e.created_at, events))
+
+export const deriveEventsDesc = (eventsStore: Readable<TrustedEvent[]>) =>
+  deriveDeduplicated(eventsStore, events => sortBy(e => -e.created_at, events))
+
+// Events by id by url
+
+export type EventsByIdByUrl = Map<string, EventsById>
+
+export type DeriveEventsByIdByUrlOptions = DeriveEventsByIdOptions & {
+  tracker: Tracker
+}
+
+export const deriveEventsByIdByUrl = ({
+  filters,
+  tracker,
+  repository,
+  includeDeleted,
+}: DeriveEventsByIdByUrlOptions) => {
+  const eventsByIdByUrl: EventsByIdByUrl = new Map()
+
+  const addEvent = (url: string, event: TrustedEvent) => {
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (!eventsById?.has(event.id)) {
+      // Create a new map so we can detect which key changed
+      const newEventsById = new Map(eventsById)
+
+      newEventsById.set(event.id, event)
+      eventsByIdByUrl.set(url, newEventsById)
+
+      return true
+    }
+
+    return false
+  }
+
+  const removeEvent = (url: string, id: string) => {
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (eventsById?.has(id)) {
+      eventsById.delete(id)
+
+      if (eventsById.size === 0) {
+        eventsByIdByUrl.delete(url)
+      } else {
+        // Create a new map so we can detect which key changed
+        eventsByIdByUrl.set(url, new Map(eventsById))
+      }
+
+      return true
+    }
+
+    return false
+  }
+
+  for (const event of repository.query(filters, {includeDeleted})) {
+    for (const url of tracker.getRelays(event.id)) {
+      addEvent(url, event)
+    }
+  }
+
+  return readable(eventsByIdByUrl, set => {
+    const unsubscribers = [
+      on(repository, "update", ({added, removed}: RepositoryUpdate) => {
         let dirty = false
-        for (const event of added.values()) {
-          if (matchFilters(filters, event)) {
-            const items = eventToItem(event)
 
-            if (items instanceof Promise) {
+        for (const event of added) {
-              defer(event, items)
+          for (const url of tracker.getRelays(event.id)) {
-            } else if (items) {
+            dirty = dirty || addEvent(url, event)
-              dirty = true
-
-              for (const item of removeUndefined(ensurePlural(items))) {
-                data.push(item as T)
-              }
-            }
           }
         }
 
-        if (!includeDeleted && removed.size > 0) {
+        for (const id of removed) {
-          const [deleted, ok] = partition(
+          for (const url of tracker.getRelays(id)) {
-            (item: T) => getIdAndAddress(itemToEvent(item)).some((id: string) => removed.has(id)),
+            dirty = dirty || removeEvent(url, id)
-            data,
-          )
-
-          if (deleted.length > 0) {
-            dirty = true
-            data = ok
           }
         }
 
         if (dirty) {
-          setter(sortBy(item => -itemToEvent(item).created_at, data))
+          set(eventsByIdByUrl)
         }
-      })
+      }),
+      on(tracker, "add", (id: string, url: string) => {
+        const event = repository.getEvent(id)
 
-      repository.on("update", onUpdate)
+        if (event && addEvent(url, event)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "remove", (id: string, url: string) => {
+        if (removeEvent(url, id)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "load", () => {
+        eventsByIdByUrl.clear()
 
-      return () => repository.off("update", onUpdate)
+        for (const event of repository.query(filters, {includeDeleted})) {
-    },
+          for (const url of tracker.getRelays(event.id)) {
-    {throttle},
+            addEvent(url, event)
-  )
+          }
+        }
 
-export type DeriveEventsOptions<T> = Omit<
+        set(eventsByIdByUrl)
-  DeriveEventsMappedOptions<T>,
+      }),
-  "itemToEvent" | "eventToItem"
+      on(tracker, "clear", () => {
->
+        eventsByIdByUrl.clear()
 
-export const deriveEvents = <T>(repository: Repository, opts: DeriveEventsOptions<T>) =>
+        set(eventsByIdByUrl)
-  deriveEventsMapped<TrustedEvent>(repository, {
+      }),
-    ...opts,
+    ]
-    eventToItem: identity,
+
-    itemToEvent: identity,
+    return () => unsubscribers.forEach(call)
   })
+}
+
+export const deriveEventsByIdForUrl = (url: string, eventsByIdByUrlStore: Readable<EventsByIdByUrl>) =>
+  deriveDeduplicated(eventsByIdByUrlStore, eventsByIdByUrl => eventsByIdByUrl.get(url))
+
+
+// Items by key
+
+export type ItemsByKey<T> = Map<string, T>
+
+export type EventToItem<T> = (event: TrustedEvent) => T
+
+export type DeriveItemsByKeyOptions<T> = {
+  getKey: (item: T) => string
+  filters: Filter[]
+  repository: Repository
+  eventToItem: EventToItem<T>
+  includeDeleted?: boolean
+}
+
+export const deriveItemsByKey = <T>({
+  getKey,
+  filters,
+  repository,
+  eventToItem,
+  includeDeleted,
+}: DeriveItemsByKeyOptions<T>) => {
+  const deferred = new Map<string, Promise<Maybe<T>>>()
+  const itemsByKey = new Map<string, T>()
+  const idsByKey = new Map<string, string>()
+  const keysById = new Map<string, string>()
+
+  return readable(itemsByKey, set => {
+    const addEvent = async (event: TrustedEvent) => {
+      if (deferred.has(event.id)) return
+      if (keysById.has(event.id)) return
+
+      const itemOrPromise = eventToItem(event)
+
+      if (itemOrPromise instanceof Promise) {
+        deferred.set(event.id, itemOrPromise)
+      }
+
+      const item = await itemOrPromise
+
+      deferred.delete(event.id)
+
+      if (item) {
+        const key = getKey(item)
+
+        itemsByKey.set(key, item)
+        idsByKey.set(key, event.id)
+        keysById.set(event.id, key)
+
+        set(itemsByKey)
+      }
+    }
+
+    for (const event of repository.query(filters, {includeDeleted})) {
+      addEvent(event)
+    }
+
+    return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+      let dirty = false
+
+      for (const event of added) {
+        if (matchFilters(filters, event)) {
+          addEvent(event)
+          dirty = true
+        }
+      }
+
+      if (!includeDeleted) {
+        for (const id of removed) {
+          const key = mapPop(id, keysById)
+
+          if (key) {
+            idsByKey.delete(key)
+            itemsByKey.delete(key)
+            dirty = true
+          }
+        }
+      }
+
+      if (dirty) {
+        set(itemsByKey)
+      }
+    })
+  })
+}
+
+export const deriveItems = <T>(itemsByKeyStore: Readable<ItemsByKey<T>>) =>
+  deriveDeduplicated(itemsByKeyStore, itemsByKey => Array.from(itemsByKey.values()))
+
+export const deriveItemsSorted = <T>(sortFn: (item: T) => number, itemsStore: Readable<T[]>) =>
+  deriveDeduplicated(itemsStore, items => sortBy(sortFn, items))
+
+// Miscellaneous other stuff
 
 export const deriveEvent = (repository: Repository, idOrAddress: string) =>
   derived(
-    deriveEvents(repository, {
+    deriveEventsById({
+      repository,
       filters: getIdFilters([idOrAddress]),
       includeDeleted: true,
     }),
-    first,
+    $m => first($m.values()),
   )
 
 export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) =>
-  custom<boolean>(setter => {
+  readable(repository.isDeleted(event), set => {
-    setter(repository.isDeleted(event))
+    const unsubscribe = on(repository, 'update', ({removed}: RepositoryUpdate) => {
+      if (removed.has(event.id)) {
+        set(true)
+        unsubscribe()
+      }
+    })
 
-    const onUpdate = batch(300, () => setter(repository.isDeleted(event)))
+    return unsubscribe
-
-    repository.on("update", onUpdate)
-
-    return () => repository.off("update", onUpdate)
-  })
-
-export const deriveIsDeletedByAddress = (repository: Repository, event: TrustedEvent) =>
-  custom<boolean>(setter => {
-    setter(repository.isDeletedByAddress(event))
-
-    const onUpdate = batch(300, () => setter(repository.isDeletedByAddress(event)))
-
-    repository.on("update", onUpdate)
-
-    return () => repository.off("update", onUpdate)
   })