Fix some bugs with deriving events by url

packages/app/src/blossom.ts

@@ -1,6 +1,13 @@
 import {BLOSSOM_SERVERS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
+import {TrustedEvent} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 
@@ -17,8 +24,17 @@ export const getBlossomServerListsByPubkey = getter(blossomServerListsByPubkey)
 
 export const getBlossomServerList = (pubkey: string) => getBlossomServerListsByPubkey().get(pubkey)
 
-export const forceLoadBlossomServerList = makeForceLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServerList)
+export const forceLoadBlossomServerList = makeForceLoadItem(
+  makeOutboxLoader(BLOSSOM_SERVERS),
+  getBlossomServerList,
+)
 
-export const loadBlossomServerList = makeLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServerList)
+export const loadBlossomServerList = makeLoadItem(
+  makeOutboxLoader(BLOSSOM_SERVERS),
+  getBlossomServerList,
+)
 
-export const deriveBlossomServerList = makeDeriveItem(blossomServerListsByPubkey, loadBlossomServerList)
+export const deriveBlossomServerList = makeDeriveItem(
+  blossomServerListsByPubkey,
+  loadBlossomServerList,
+)

packages/app/src/follows.ts

@@ -1,6 +1,13 @@
 import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
+import {TrustedEvent} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 

packages/app/src/handles.ts

@@ -1,5 +1,5 @@
-import {writable, derived} from "svelte/store"
+import {writable, derived, Subscriber} from "svelte/store"
-import {tryCatch, fetchJson, uniq, batcher, postJson, last} from "@welshman/lib"
+import {tryCatch, fetchJson, batcher, postJson, last} from "@welshman/lib"
 import {getter, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem} from "@welshman/store"
 import {deriveProfile, loadProfile} from "./profiles.js"
 import {appContext} from "./context.js"
@@ -50,8 +50,22 @@ export const getHandles = getter(handles)
 
 export const getHandle = (nip05: string) => getHandlesByNip05().get(nip05)
 
+export const handleSubscribers: Subscriber<Handle>[] = []
+
+export const notifyHandle = (handle: Handle) => handleSubscribers.forEach(sub => sub(handle))
+
+export const onHandle = (sub: (handle: Handle) => void) => {
+  handleSubscribers.push(sub)
+
+  return () =>
+    handleSubscribers.splice(
+      handleSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchHandle = batcher(800, async (nip05s: string[]) => {
-  const handlesByNip05 = new Map<string, Handle>()
+  const result = new Map<string, Handle>()
 
   // Use dufflepud if we it's set up to protect user privacy, otherwise fetch directly
   if (appContext.dufflepudUrl) {
@@ -61,7 +75,7 @@ export const fetchHandle = batcher(800, async (nip05s: string[]) => {
 
     for (const {handle: nip05, info} of res?.data || []) {
       if (info) {
-        handlesByNip05.set(nip05, info)
+        result.set(nip05, {...info, nip05})
       }
     }
   } else {
@@ -74,18 +88,24 @@ export const fetchHandle = batcher(800, async (nip05s: string[]) => {
 
     for (const {nip05, info} of results) {
       if (info) {
-        handlesByNip05.set(nip05, info)
+        result.set(nip05, {...info, nip05})
       }
     }
   }
 
-  return nip05s.map(nip05 => {
+  handlesByNip05.update($handlesByNip05 => {
-    const info = handlesByNip05.get(nip05)
+    for (const [nip05, info] of result) {
-
+      $handlesByNip05.set(nip05, info)
-    if (info) {
-      return {...info, nip05}
     }
+
+    return $handlesByNip05
   })
+
+  for (const info of result.values()) {
+    notifyHandle(info)
+  }
+
+  return nip05s.map(nip05 => result.get(nip05))
 })
 
 export const forceLoadHandle = makeForceLoadItem(fetchHandle, getHandle)

packages/app/src/messagingRelayLists.ts

@@ -1,6 +1,13 @@
 import {MESSAGING_RELAYS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
+import {TrustedEvent} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 
@@ -17,10 +24,20 @@ export const getMessagingRelayListsByPubkey = getter(messagingRelayListsByPubkey
 
 export const getMessagingRelayLists = getter(messagingRelayLists)
 
-export const getMessagingRelayList = (pubkey: string) => getMessagingRelayListsByPubkey().get(pubkey)
+export const getMessagingRelayList = (pubkey: string) =>
+  getMessagingRelayListsByPubkey().get(pubkey)
 
-export const forceLoadMessagingRelayList = makeForceLoadItem(makeOutboxLoader(MESSAGING_RELAYS), getMessagingRelayList)
+export const forceLoadMessagingRelayList = makeForceLoadItem(
+  makeOutboxLoader(MESSAGING_RELAYS),
+  getMessagingRelayList,
+)
 
-export const loadMessagingRelayList = makeLoadItem(makeOutboxLoader(MESSAGING_RELAYS), getMessagingRelayList)
+export const loadMessagingRelayList = makeLoadItem(
+  makeOutboxLoader(MESSAGING_RELAYS),
+  getMessagingRelayList,
+)
 
-export const deriveMessagingRelayList = makeDeriveItem(messagingRelayListsByPubkey, loadMessagingRelayList)
+export const deriveMessagingRelayList = makeDeriveItem(
+  messagingRelayListsByPubkey,
+  loadMessagingRelayList,
+)

packages/app/src/mutes.ts

@@ -1,6 +1,13 @@
 import {MUTES, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {ensurePlaintext} from "./plaintext.js"
 import {makeOutboxLoader} from "./relayLists.js"

packages/app/src/pins.ts

@@ -1,6 +1,13 @@
 import {PINS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
+import {TrustedEvent} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 

packages/app/src/profiles.ts

@@ -1,6 +1,13 @@
 import {derived, readable} from "svelte/store"
 import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoaderWithIndexers} from "./relayLists.js"
 

packages/app/src/relayLists.ts

@@ -1,13 +1,13 @@
 import {batcher} from "@welshman/lib"
+import {RELAYS, Filter, asDecryptedEvent, readList, TrustedEvent} from "@welshman/util"
 import {
-  RELAYS,
+  deriveItemsByKey,
-  Filter,
+  deriveItems,
-  asDecryptedEvent,
+  makeForceLoadItem,
-  readList,
+  makeLoadItem,
-  TrustedEvent,
+  makeDeriveItem,
-  PublishedList,
+  getter,
-} from "@welshman/util"
+} from "@welshman/store"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
 import {load, LoadOptions} from "@welshman/net"
 import {Router} from "@welshman/router"
 import {repository} from "./core.js"
@@ -56,7 +56,10 @@ export const getRelayLists = getter(relayLists)
 
 export const getRelayList = (pubkey: string) => getRelayListsByPubkey().get(pubkey)
 
-export const forceLoadRelayList = makeForceLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelayList)
+export const forceLoadRelayList = makeForceLoadItem(
+  makeOutboxLoaderWithIndexers(RELAYS),
+  getRelayList,
+)
 
 export const loadRelayList = makeLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelayList)
 

packages/app/src/relayStats.ts

@@ -1,6 +1,6 @@
-import {writable, derived} from "svelte/store"
+import {writable, Subscriber} from "svelte/store"
-import {withGetter} from "@welshman/store"
+import {getter, makeDeriveItem} from "@welshman/store"
-import {prop, groupBy, indexBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib"
+import {groupBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib"
 import {isOnionUrl, isLocalUrl, isIPAddress, isRelayUrl} from "@welshman/util"
 import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
 
@@ -48,20 +48,34 @@ export const makeRelayStats = (url: string): RelayStats => ({
   notice_count: 0,
 })
 
-export const relayStats = withGetter(writable<RelayStats[]>([]))
+export const relayStatsByUrl = writable(new Map<string, RelayStats>())
 
-export const relayStatsByUrl = withGetter(
+export const getRelayStatsByUrl = getter(relayStatsByUrl)
-  derived(relayStats, $relayStats => indexBy(prop("url"), $relayStats)),
-)
 
-export const deriveRelayStats = (url: string) =>
+export const getRelayStats = (url: string) => getRelayStatsByUrl().get(url)
-  derived(relayStatsByUrl, $relayStatsByUrl => $relayStatsByUrl.get(url))
+
+export const relayStatsSubscribers: Subscriber<RelayStats>[] = []
+
+export const notifyRelayStats = (relayStats: RelayStats) =>
+  relayStatsSubscribers.forEach(sub => sub(relayStats))
+
+export const onRelayStats = (sub: (relayStats: RelayStats) => void) => {
+  relayStatsSubscribers.push(sub)
+
+  return () =>
+    relayStatsSubscribers.splice(
+      relayStatsSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
+export const deriveRelayStats = makeDeriveItem(relayStatsByUrl)
 
 export const getRelayQuality = (url: string) => {
   // Skip non-relays entirely
   if (!isRelayUrl(url)) return 0
 
-  const relayStats = relayStatsByUrl.get().get(url)
+  const relayStats = getRelayStats(url)
 
   // If we have recent errors, skip it
   if (relayStats) {
@@ -90,9 +104,7 @@ export const getRelayQuality = (url: string) => {
 type RelayStatsUpdate = [string, (stats: RelayStats) => void]
 
 const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
-  relayStats.update($relayStats => {
+  relayStatsByUrl.update($relayStatsByUrl => {
-    const $relayStatsByUrl = indexBy(r => r.url, $relayStats)
-
     for (const [url, items] of groupBy(([url]) => url, updates)) {
       if (!url || !isRelayUrl(url)) {
         console.warn(`Attempted to update stats for an invalid relay url: ${url}`)
@@ -109,7 +121,7 @@ const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
       $relayStatsByUrl.set(url, {...$relayStatsItem})
     }
 
-    return Array.from($relayStatsByUrl.values())
+    return $relayStatsByUrl
   })
 })
 

packages/app/src/relays.ts

@@ -1,18 +1,7 @@
-import {writable, derived} from "svelte/store"
+import {writable, derived, Subscriber} from "svelte/store"
-import {
+import {batcher, fetchJson, postJson, Maybe, noop} from "@welshman/lib"
-  uniq,
-  removeUndefined,
-  prop,
-  indexBy,
-  batcher,
-  fetchJson,
-  postJson,
-  Maybe,
-  noop,
-} from "@welshman/lib"
-import {withGetter} from "@welshman/store"
 import {RelayProfile} from "@welshman/util"
-import {normalizeRelayUrl, displayRelayUrl, displayRelayProfile, isRelayUrl} from "@welshman/util"
+import {displayRelayUrl, displayRelayProfile} from "@welshman/util"
 import {getter, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem} from "@welshman/store"
 import {appContext} from "./context.js"
 
@@ -26,6 +15,20 @@ export const getRelays = getter(relays)
 
 export const getRelay = (url: string) => getRelaysByUrl().get(url)
 
+export const relaySubscribers: Subscriber<RelayProfile>[] = []
+
+export const notifyRelay = (relay: RelayProfile) => relaySubscribers.forEach(sub => sub(relay))
+
+export const onRelay = (sub: (relay: RelayProfile) => void) => {
+  relaySubscribers.push(sub)
+
+  return () =>
+    relaySubscribers.splice(
+      relaySubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchRelayDirectly = async (url: string): Promise<Maybe<RelayProfile>> => {
   try {
     const json = fetchJson(url.replace(/^ws/, "http"), {
@@ -35,7 +38,17 @@ export const fetchRelayDirectly = async (url: string): Promise<Maybe<RelayProfil
     })
 
     if (json) {
-      return {...json, url}
+      const info = {...json, url}
+
+      relaysByUrl.update($relaysByUrl => {
+        $relaysByUrl.set(url, info)
+
+        return $relaysByUrl
+      })
+
+      notifyRelay(info)
+
+      return info
     }
   } catch (e) {
     // pass
@@ -49,19 +62,27 @@ export const fetchRelayUsingProxy = batcher(800, async (urls: string[]) => {
   }
 
   const res: any = await postJson(`${appContext.dufflepudUrl}/relay/info`, {urls})
-  const relaysByUrl = new Map<string, RelayProfile>()
+  const result = new Map<string, RelayProfile>()
 
   for (const {url, info} of res?.data || []) {
-    relaysByUrl.set(url, info)
+    if (info) {
+      result.set(url, {...info, url})
+    }
   }
 
-  return urls.map(url => {
+  relaysByUrl.update($relaysByUrl => {
-    const info = relaysByUrl.get(url)
+    for (const [url, info] of result) {
-
+      $relaysByUrl.set(url, info)
-    if (info) {
-      return {...info, url}
     }
+
+    return $relaysByUrl
   })
+
+  for (const info of result.values()) {
+    notifyRelay(info)
+  }
+
+  return urls.map(url => result.get(url))
 })
 
 export const fetchRelay = (url: string) =>

packages/app/src/topics.ts

@@ -1,4 +1,4 @@
-import {readable} from 'svelte/store'
+import {readable} from "svelte/store"
 import {on, call} from "@welshman/lib"
 import {deriveItems} from "@welshman/store"
 import {getTopicTagValues} from "@welshman/util"
@@ -29,7 +29,7 @@ export const topicsByName = call(() => {
   }
 
   return readable<Map<string, Topic>>(topicsByName, set => {
-    return on(repository, 'update', ({added}) => {
+    return on(repository, "update", ({added}) => {
       let dirty = false
 
       for (const event of added) {

packages/app/src/user.ts

@@ -5,9 +5,17 @@ import {profilesByPubkey, forceLoadProfile, loadProfile} from "./profiles.js"
 import {followListsByPubkey, forceLoadFollowList, loadFollowList} from "./follows.js"
 import {pinListsByPubkey, forceLoadPinList, loadPinList} from "./pins.js"
 import {muteListsByPubkey, forceLoadMuteList, loadMuteList} from "./mutes.js"
-import {blossomServerListsByPubkey, forceLoadBlossomServerList, loadBlossomServerList} from "./blossom.js"
+import {
+  blossomServerListsByPubkey,
+  forceLoadBlossomServerList,
+  loadBlossomServerList,
+} from "./blossom.js"
 import {relayListsByPubkey, forceLoadRelayList, loadRelayList} from "./relayLists.js"
-import {messagingRelayListsByPubkey, forceLoadMessagingRelayList, loadMessagingRelayList} from "./messagingRelayLists.js"
+import {
+  messagingRelayListsByPubkey,
+  forceLoadMessagingRelayList,
+  loadMessagingRelayList,
+} from "./messagingRelayLists.js"
 import {wotGraph} from "./wot.js"
 
 export type UserDataLoader = (pubkey: string, relays?: string[], force?: boolean) => unknown

packages/app/src/wot.ts

@@ -6,11 +6,9 @@ import {pubkey} from "./session.js"
 import {followLists, getFollowListsByPubkey, getFollowList} from "./follows.js"
 import {muteLists, getMuteList} from "./mutes.js"
 
-export const getFollows = (pubkey: string) =>
+export const getFollows = (pubkey: string) => getPubkeyTagValues(getListTags(getFollowList(pubkey)))
-  getPubkeyTagValues(getListTags(getFollowList(pubkey)))
 
-export const getMutes = (pubkey: string) =>
+export const getMutes = (pubkey: string) => getPubkeyTagValues(getListTags(getMuteList(pubkey)))
-  getPubkeyTagValues(getListTags(getMuteList(pubkey)))
 
 export const getNetwork = (pubkey: string) => {
   const pubkeys = new Set(getFollows(pubkey))

packages/app/src/zappers.ts

@@ -1,9 +1,8 @@
-import {writable, derived} from "svelte/store"
+import {writable, derived, Subscriber} from "svelte/store"
 import {Zapper, TrustedEvent, Zap, getTagValues, getLnUrl, zapFromEvent} from "@welshman/util"
 import {
   removeUndefined,
   fetchJson,
-  uniq,
   bech32ToHex,
   hexToBech32,
   tryCatch,
@@ -22,6 +21,20 @@ export const getZappersByLnurl = getter(zappersByLnurl)
 
 export const getZapper = (lnurl: string) => getZappersByLnurl().get(lnurl)
 
+export const zapperSubscribers: Subscriber<Zapper>[] = []
+
+export const notifyZapper = (zapper: Zapper) => zapperSubscribers.forEach(sub => sub(zapper))
+
+export const onZapper = (sub: (zapper: Zapper) => void) => {
+  zapperSubscribers.push(sub)
+
+  return () =>
+    zapperSubscribers.splice(
+      zapperSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchZapper = batcher(800, async (lnurls: string[]) => {
   const base = appContext.dufflepudUrl
   const result = new Map<string, Zapper>()
@@ -35,8 +48,12 @@ export const fetchZapper = batcher(800, async (lnurls: string[]) => {
         async () => await postJson(`${base}/zapper/info`, {lnurls: hexUrls}),
       )
 
-      for (const {lnurl, info} of res?.data || []) {
+      for (const {hexUrl, info} of res?.data || []) {
-        tryCatch(() => result.set(hexToBech32("lnurl", lnurl), info))
+        if (info) {
+          const lnurl = hexToBech32("lnurl", hexUrl)
+
+          tryCatch(() => result.set(lnurl, {...info, lnurl}))
+        }
       }
     }
   } else {
@@ -51,11 +68,23 @@ export const fetchZapper = batcher(800, async (lnurls: string[]) => {
 
     for (const {lnurl, info} of results) {
       if (info) {
-        result.set(lnurl, info)
+        result.set(lnurl, {...info, lnurl})
       }
     }
   }
 
+  zappersByLnurl.update($zappersByLnurl => {
+    for (const [nip05, info] of result) {
+      $zappersByLnurl.set(nip05, info)
+    }
+
+    return $zappersByLnurl
+  })
+
+  for (const info of result.values()) {
+    notifyZapper(info)
+  }
+
   return lnurls.map(lnurl => {
     const info = result.get(lnurl)
 

packages/lib/src/Tools.ts

@@ -559,7 +559,7 @@ export const flatten = <T>(xs: (T | T[])[], ...args: unknown[]) => xs.flatMap(id
  * @param xs - Array to partition
  * @returns Tuple of [matching, non-matching] arrays
  */
-export const partition = <T>(f: (x: T) => boolean, xs: T[]) => {
+export const partition = <T>(f: (x: T) => boolean, xs: Iterable<T>) => {
   const a: T[] = []
   const b: T[] = []
 
@@ -574,21 +574,50 @@ export const partition = <T>(f: (x: T) => boolean, xs: T[]) => {
   return [a, b]
 }
 
+/** Maps any iterable */
+export const map = <T, R>(f: (x: T) => R, xs: Iterable<T>) => Array.from(xs).map(f)
+
 /**
  * Keeps items based on predicate
- * @param f - Whether to remove an item from the array
+ * @param f - Whether to keep an item
- * @param xs - Array of items to filter
+ * @param xs - Items to filter
  * @returns Filtered array
  */
-export const filter = <T>(f: (x: T) => any, xs: T[]) => xs.filter(f)
+export const filter = <T>(f: (x: T) => any, xs: Iterable<T>) => Array.from(xs).filter(f)
 
 /**
  * Removes items based on predicate
  * @param f - Whether to remove an item from the array
- * @param xs - Array of items to filter
+ * @param xs - Items to filter
  * @returns Filtered array
  */
-export const reject = <T>(f: (x: T) => any, xs: T[]) => xs.filter(complement(f))
+export const reject = <T>(f: (x: T) => any, xs: Iterable<T>) => Array.from(xs).filter(complement(f))
+
+/**
+ * Finds a single item based on predicate
+ * @param f - Whether an item matches
+ * @param xs - Items to filter
+ * @returns first matching item
+ */
+export const find = <T>(f: (x: T) => any, xs: Iterable<T>) => {
+  for (const x of xs) {
+    if (f(x)) return x
+  }
+}
+
+/**
+ * Finds a single item based on predicate
+ * @param f - Whether an item matches
+ * @param xs - Items to filter
+ * @returns whether an item matches
+ */
+export const some = <T>(f: (x: T) => any, xs: Iterable<T>) => {
+  for (const x of xs) {
+    if (f(x)) return true
+  }
+
+  return false
+}
 
 /**
  * Returns array with duplicate elements removed
@@ -603,7 +632,7 @@ export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
  * @param xs - Input array
  * @returns Array with elements unique by key
  */
-export const uniqBy = <T>(f: (x: T) => any, xs: T[]) => {
+export const uniqBy = <T>(f: (x: T) => any, xs: Iterable<T>) => {
   const s = new Set<any>()
   const r = []
 

packages/net/src/repository.ts

@@ -110,6 +110,7 @@ export class Repository extends Emitter {
       removed.add(id)
     }
 
+    console.log("UPDATE")
     this.emit("update", {added, removed})
   }
 

packages/router/src/index.ts

@@ -187,7 +187,8 @@ export class Router {
 
   FromPubkeys = (pubkeys: string[]) => this.merge(pubkeys.map(pubkey => this.FromPubkey(pubkey)))
 
-  MessagesForPubkeys = (pubkeys: string[]) => this.merge(pubkeys.map(pubkey => this.MessagesForPubkey(pubkey)))
+  MessagesForPubkeys = (pubkeys: string[]) =>
+    this.merge(pubkeys.map(pubkey => this.MessagesForPubkey(pubkey)))
 
   Event = (event: TrustedEvent) =>
     this.FromRelays(this.getRelaysForPubkey(event.pubkey, RelayMode.Write))

packages/store/__tests__/collection.test.ts

@@ -1,240 +0,0 @@
-import {describe, it, expect, beforeEach, vi, afterEach} from "vitest"
-import {get, writable} from "svelte/store"
-import {now, always} from "@welshman/lib"
-import {collection, freshness, setFreshnessImmediate} from "../src/collection"
-
-describe("collection", () => {
-  beforeEach(() => {
-    vi.useFakeTimers()
-    vi.clearAllMocks()
-  })
-
-  afterEach(() => {
-    vi.resetModules()
-    vi.useRealTimers()
-    freshness.set({})
-  })
-
-  describe("basic functionality", () => {
-    it("should create a collection with indexStore", () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      expect(col.indexStore.get().get("1")).toEqual(items[0])
-    })
-
-    it("should update indexStore when store changes", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const newItem = {id: "1", value: "test"}
-      store.set([newItem])
-
-      expect(get(col.indexStore).get("1")).toEqual(newItem)
-    })
-  })
-
-  describe("loadItem", () => {
-    it("should return stale item if no loader provided", async () => {
-      const items = [{id: "1", value: "test"}]
-      const store = writable(items)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toEqual(items[0])
-    })
-
-    it("should return undefined for non-existent items when no loader provided", async () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-
-    it("should use loader to fetch new items", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-
-    it("should handle concurrent loading of the same item", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // Start multiple concurrent loads
-      const loads = Promise.all([col.loadItem("1"), col.loadItem("1"), col.loadItem("1")])
-
-      await loads
-      // Should only call load once
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-
-    it("should respect freshness checks", async () => {
-      await vi.advanceTimersByTimeAsync(1000)
-      const store = writable<Array<{id: string; value: string}>>([{id: "1", value: "stale"}])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      // force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-      await col.loadItem("1")
-      // Should not call load because item is fresh
-      expect(mockLoad).toHaveBeenCalledTimes(0)
-    })
-
-    it("should reload stale items", async () => {
-      const mockLoad = vi.fn()
-      const store = writable([{id: "1", value: "test"}])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: (item: any) => item.id,
-        load: mockLoad,
-      })
-
-      // load the item to set freshness
-      await col.loadItem("1")
-
-      await vi.advanceTimersByTimeAsync(4000 * 1000)
-
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(2)
-    })
-
-    it("should implement exponential backoff for failed attempts", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn().mockResolvedValue(undefined)
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      // First attempt
-      await col.loadItem("1")
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-
-      //force freshness
-      setFreshnessImmediate({ns: "test", key: "1", ts: now()})
-
-      // Immediate retry should be throttled
-      await col.loadItem("1").catch(() => {})
-      expect(mockLoad).toHaveBeenCalledTimes(1)
-    })
-  })
-
-  describe("deriveItem", () => {
-    it("should return readable undefined for null keys", () => {
-      const store = writable<Array<{id: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem(undefined)
-      expect(get(derived)).toBeUndefined()
-    })
-
-    it("should create a derived store that updates with the source", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: always(Promise.resolve()),
-      })
-
-      const derived = col.deriveItem("1")
-      expect(get(derived)).toBeUndefined()
-
-      // Update source store
-      store.set([{id: "1", value: "test"}])
-      expect(get(derived)).toEqual({id: "1", value: "test"})
-    })
-
-    it("should trigger load when deriving non-existent item", () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn()
-
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-
-      col.deriveItem("1")
-      expect(mockLoad).toHaveBeenCalledWith("1", [])
-    })
-  })
-
-  describe("error handling", () => {
-    it("should handle loader failures gracefully", async () => {
-      const store = writable<Array<{id: string; value: string}>>([])
-      const mockLoad = vi.fn(() => {
-        return Promise.reject("load failed")
-      })
-      const col = collection({
-        name: "test",
-        store,
-        getKey: item => item.id,
-        load: mockLoad,
-      })
-      const result = await col.loadItem("1")
-      expect(result).toBeUndefined()
-    })
-  })
-})

packages/store/__tests__/index.test.ts

@@ -3,9 +3,7 @@ import {Repository} from "@welshman/net"
 import {get} from "svelte/store"
 import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"
 import {
-  custom,
   deriveEvents,
-  deriveEventsMapped,
   deriveIsDeleted,
   getter,
   synced,

packages/store/src/repository.ts

@@ -1,5 +1,5 @@
-import {derived, readable, Readable} from "svelte/store"
+import {readable, Readable} from "svelte/store"
-import {on, assoc, now, indexBy, mapPop, Maybe, MaybeAsync, call, sortBy, first} from "@welshman/lib"
+import {on, assoc, now, mapPop, Maybe, MaybeAsync, call, sortBy, first} from "@welshman/lib"
 import {matchFilters, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
 import {Repository, RepositoryUpdate, Tracker} from "@welshman/net"
 import {deriveDeduplicated} from "./misc.js"
@@ -19,9 +19,15 @@ export const deriveEventsById = ({
   repository,
   includeDeleted,
 }: DeriveEventsByIdOptions) => {
-  const eventsById: EventsById = indexBy(e => e.id, repository.query(filters, {includeDeleted}))
+  const eventsById = new Map<string, TrustedEvent>()
 
   return readable(eventsById, set => {
+    for (const event of repository.query(filters, {includeDeleted})) {
+      eventsById.set(event.id, event)
+    }
+
+    set(eventsById)
+
     return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
       let dirty = false
 
@@ -45,8 +51,8 @@ export const deriveEventsById = ({
   })
 }
 
-export const deriveEvents = (eventsByIdStore: Readable<EventsById>) =>
+export const deriveArray = <T>(itemsByIdStore: Readable<Map<string, T>>) =>
-  deriveDeduplicated(eventsByIdStore, eventsById => Array.from(eventsById.values()))
+  deriveDeduplicated(itemsByIdStore, itemsById => Array.from(itemsById.values()))
 
 export const deriveEventsAsc = (eventsByIdStore: Readable<EventsById>) =>
   deriveDeduplicated(eventsByIdStore, eventsById => sortBy(e => e.created_at, eventsById.values()))
@@ -54,8 +60,158 @@ export const deriveEventsAsc = (eventsByIdStore: Readable<EventsById>) =>
 export const deriveEventsDesc = (eventsByIdStore: Readable<EventsById>) =>
   deriveDeduplicated(eventsByIdStore, eventsById => sortBy(e => -e.created_at, eventsById.values()))
 
+export type DeriveEventOptions = {
+  repository: Repository
+  includeDeleted?: boolean
+  onDerive?: (filters: Filter[], ...args: any[]) => void
+}
+
+export const makeDeriveEvent = ({
+  repository,
+  includeDeleted = false,
+  onDerive,
+}: DeriveEventOptions) => {
+  return (idOrAddress: string, ...args: any[]) => {
+    const filters = getIdFilters([idOrAddress])
+
+    onDerive?.(filters, ...args)
+
+    return readable<Maybe<TrustedEvent>>(undefined, set => {
+      const event = first(repository.query(filters, {includeDeleted}))
+
+      set(event)
+
+      return on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+        for (const event of added) {
+          if (matchFilters(filters, event)) {
+            set(event)
+          }
+        }
+
+        for (const id of removed) {
+          if (event?.id === id) {
+            set(undefined)
+          }
+        }
+      })
+    })
+  }
+}
+
 // Events by id by url
 
+export type EventsByIdByUrl = Map<string, EventsById>
+
+export type DeriveEventsByIdByUrlOptions = DeriveEventsByIdOptions & {
+  tracker: Tracker
+}
+
+export const deriveEventsByIdByUrl = ({
+  filters,
+  tracker,
+  repository,
+  includeDeleted,
+}: DeriveEventsByIdByUrlOptions) => {
+  const eventsByIdByUrl: EventsByIdByUrl = new Map()
+
+  const addEvent = (url: string, event: TrustedEvent) => {
+    if (!matchFilters(filters, event)) return false
+
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (eventsById?.has(event.id)) return false
+
+    // Create a new map so we can detect which key changed
+    const newEventsById = new Map(eventsById)
+
+    newEventsById.set(event.id, event)
+    eventsByIdByUrl.set(url, newEventsById)
+
+    return true
+  }
+
+  const removeEvent = (url: string, id: string) => {
+    const eventsById = eventsByIdByUrl.get(url)
+
+    if (eventsById?.has(id)) {
+      eventsById.delete(id)
+
+      if (eventsById.size === 0) {
+        eventsByIdByUrl.delete(url)
+      } else {
+        // Create a new map so we can detect which key changed
+        eventsByIdByUrl.set(url, new Map(eventsById))
+      }
+
+      return true
+    }
+
+    return false
+  }
+
+  return readable(eventsByIdByUrl, set => {
+    for (const event of repository.query(filters, {includeDeleted})) {
+      for (const url of tracker.getRelays(event.id)) {
+        addEvent(url, event)
+      }
+    }
+
+    set(eventsByIdByUrl)
+
+    const unsubscribers = [
+      on(repository, "update", ({added, removed}: RepositoryUpdate) => {
+        let dirty = false
+
+        for (const event of added) {
+          for (const url of tracker.getRelays(event.id)) {
+            dirty = dirty || addEvent(url, event)
+          }
+        }
+
+        for (const id of removed) {
+          for (const url of tracker.getRelays(id)) {
+            dirty = dirty || removeEvent(url, id)
+          }
+        }
+
+        if (dirty) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "add", (id: string, url: string) => {
+        const event = repository.getEvent(id)
+
+        if (event && addEvent(url, event)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "remove", (id: string, url: string) => {
+        if (removeEvent(url, id)) {
+          set(eventsByIdByUrl)
+        }
+      }),
+      on(tracker, "load", () => {
+        eventsByIdByUrl.clear()
+
+        for (const event of repository.query(filters, {includeDeleted})) {
+          for (const url of tracker.getRelays(event.id)) {
+            addEvent(url, event)
+          }
+        }
+
+        set(eventsByIdByUrl)
+      }),
+      on(tracker, "clear", () => {
+        eventsByIdByUrl.clear()
+
+        set(eventsByIdByUrl)
+      }),
+    ]
+
+    return () => unsubscribers.forEach(call)
+  })
+}
+
 export type DeriveEventsByIdForUrlOptions = DeriveEventsByIdOptions & {
   url: string
   tracker: Tracker
@@ -72,7 +228,7 @@ export const deriveEventsByIdForUrl = ({
 
   const initialize = () => {
     const initialIds = Array.from(tracker.getIds(url))
-    const initialFilters = filters.map(assoc('ids', initialIds))
+    const initialFilters = filters.map(assoc("ids", initialIds))
 
     for (const event of repository.query(initialFilters, {includeDeleted})) {
       eventsById.set(event.id, event)
@@ -82,20 +238,24 @@ export const deriveEventsByIdForUrl = ({
   }
 
   return readable(initialize(), set => {
+    set(initialize())
+
     const unsubscribers = [
       on(repository, "update", ({added, removed}: RepositoryUpdate) => {
         let dirty = false
 
         for (const event of added) {
-          if (tracker.hasRelay(event.id, url) && !eventsById.has(event.id)) {
+          if (tracker.hasRelay(event.id, url) && matchFilters(filters, event)) {
             eventsById.set(event.id, event)
             dirty = true
           }
         }
 
         for (const id of removed) {
-          eventsById.delete(id)
+          if (eventsById.has(id)) {
-          dirty = true
+            eventsById.delete(id)
+            dirty = true
+          }
         }
 
         if (dirty) {
@@ -105,7 +265,7 @@ export const deriveEventsByIdForUrl = ({
       on(tracker, "add", (id: string, url: string) => {
         const event = repository.getEvent(id)
 
-        if (event && tracker.hasRelay(id, url) && !eventsById.has(id)) {
+        if (event && tracker.hasRelay(id, url) && matchFilters(filters, event)) {
           eventsById.set(id, event)
           set(eventsById)
         }
@@ -118,9 +278,7 @@ export const deriveEventsByIdForUrl = ({
       }),
       on(tracker, "load", () => {
         eventsById.clear()
-        initialize()
+        set(initialize())
-
-        set(eventsById)
       }),
       on(tracker, "clear", () => {
         eventsById.clear()
@@ -253,7 +411,11 @@ export type MakeLoadItemOptions = {
   timeout?: number
 }
 
-export const makeLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>, options: MakeLoadItemOptions = {}) => {
+export const makeLoadItem = <T>(
+  loadItem: LoadItem,
+  getItem: GetItem<T>,
+  options: MakeLoadItemOptions = {},
+) => {
   const timeout = options.timeout || 3600
   const fetched = new Map<string, number>()
   const getFetched = options.getFetched || ((key: string) => fetched.get(key) || 0)
@@ -311,18 +473,10 @@ export const makeLoadItem = <T>(loadItem: LoadItem, getItem: GetItem<T>, options
 
 // Miscellaneous other stuff
 
-export const deriveEvent = (repository: Repository, idOrAddress: string) =>
-  derived(
-    deriveEventsById({
-      repository,
-      filters: getIdFilters([idOrAddress]),
-      includeDeleted: true,
-    }),
-    $m => first($m.values()),
-  )
-
 export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) =>
-  readable(repository.isDeleted(event), set => {
+  readable(false, set => {
+    set(repository.isDeleted(event))
+
     const unsubscribe = on(repository, "update", ({removed}: RepositoryUpdate) => {
       if (removed.has(event.id)) {
         set(true)