Fix some bugs with deriving events by url

packages/app/src/blossom.ts

@@ -1,6 +1,13 @@
 import {BLOSSOM_SERVERS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {TrustedEvent} from "@welshman/util"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 
@@ -17,8 +24,17 @@ export const getBlossomServerListsByPubkey = getter(blossomServerListsByPubkey)
 
 export const getBlossomServerList = (pubkey: string) => getBlossomServerListsByPubkey().get(pubkey)
 
-export const forceLoadBlossomServerList = makeForceLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServerList)
+export const forceLoadBlossomServerList = makeForceLoadItem(
+  makeOutboxLoader(BLOSSOM_SERVERS),
+  getBlossomServerList,
+)
 
-export const loadBlossomServerList = makeLoadItem(makeOutboxLoader(BLOSSOM_SERVERS), getBlossomServerList)
+export const loadBlossomServerList = makeLoadItem(
+  makeOutboxLoader(BLOSSOM_SERVERS),
+  getBlossomServerList,
+)
 
-export const deriveBlossomServerList = makeDeriveItem(blossomServerListsByPubkey, loadBlossomServerList)
+export const deriveBlossomServerList = makeDeriveItem(
+  blossomServerListsByPubkey,
+  loadBlossomServerList,
+)

packages/app/src/follows.ts

@@ -1,6 +1,13 @@
 import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {TrustedEvent} from "@welshman/util"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 

packages/app/src/handles.ts

@@ -1,5 +1,5 @@
-import {writable, derived} from "svelte/store"
-import {tryCatch, fetchJson, uniq, batcher, postJson, last} from "@welshman/lib"
+import {writable, derived, Subscriber} from "svelte/store"
+import {tryCatch, fetchJson, batcher, postJson, last} from "@welshman/lib"
 import {getter, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem} from "@welshman/store"
 import {deriveProfile, loadProfile} from "./profiles.js"
 import {appContext} from "./context.js"
@@ -50,8 +50,22 @@ export const getHandles = getter(handles)
 
 export const getHandle = (nip05: string) => getHandlesByNip05().get(nip05)
 
+export const handleSubscribers: Subscriber<Handle>[] = []
+
+export const notifyHandle = (handle: Handle) => handleSubscribers.forEach(sub => sub(handle))
+
+export const onHandle = (sub: (handle: Handle) => void) => {
+  handleSubscribers.push(sub)
+
+  return () =>
+    handleSubscribers.splice(
+      handleSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchHandle = batcher(800, async (nip05s: string[]) => {
-  const handlesByNip05 = new Map<string, Handle>()
+  const result = new Map<string, Handle>()
 
   // Use dufflepud if we it's set up to protect user privacy, otherwise fetch directly
   if (appContext.dufflepudUrl) {
@@ -61,7 +75,7 @@ export const fetchHandle = batcher(800, async (nip05s: string[]) => {
 
     for (const {handle: nip05, info} of res?.data || []) {
       if (info) {
-        handlesByNip05.set(nip05, info)
+        result.set(nip05, {...info, nip05})
       }
     }
   } else {
@@ -74,18 +88,24 @@ export const fetchHandle = batcher(800, async (nip05s: string[]) => {
 
     for (const {nip05, info} of results) {
       if (info) {
-        handlesByNip05.set(nip05, info)
+        result.set(nip05, {...info, nip05})
       }
     }
   }
 
-  return nip05s.map(nip05 => {
-    const info = handlesByNip05.get(nip05)
-
-    if (info) {
-      return {...info, nip05}
+  handlesByNip05.update($handlesByNip05 => {
+    for (const [nip05, info] of result) {
+      $handlesByNip05.set(nip05, info)
     }
+
+    return $handlesByNip05
   })
+
+  for (const info of result.values()) {
+    notifyHandle(info)
+  }
+
+  return nip05s.map(nip05 => result.get(nip05))
 })
 
 export const forceLoadHandle = makeForceLoadItem(fetchHandle, getHandle)

packages/app/src/messagingRelayLists.ts

@@ -1,6 +1,13 @@
 import {MESSAGING_RELAYS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {TrustedEvent} from "@welshman/util"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 
@@ -17,10 +24,20 @@ export const getMessagingRelayListsByPubkey = getter(messagingRelayListsByPubkey
 
 export const getMessagingRelayLists = getter(messagingRelayLists)
 
-export const getMessagingRelayList = (pubkey: string) => getMessagingRelayListsByPubkey().get(pubkey)
+export const getMessagingRelayList = (pubkey: string) =>
+  getMessagingRelayListsByPubkey().get(pubkey)
 
-export const forceLoadMessagingRelayList = makeForceLoadItem(makeOutboxLoader(MESSAGING_RELAYS), getMessagingRelayList)
+export const forceLoadMessagingRelayList = makeForceLoadItem(
+  makeOutboxLoader(MESSAGING_RELAYS),
+  getMessagingRelayList,
+)
 
-export const loadMessagingRelayList = makeLoadItem(makeOutboxLoader(MESSAGING_RELAYS), getMessagingRelayList)
+export const loadMessagingRelayList = makeLoadItem(
+  makeOutboxLoader(MESSAGING_RELAYS),
+  getMessagingRelayList,
+)
 
-export const deriveMessagingRelayList = makeDeriveItem(messagingRelayListsByPubkey, loadMessagingRelayList)
+export const deriveMessagingRelayList = makeDeriveItem(
+  messagingRelayListsByPubkey,
+  loadMessagingRelayList,
+)

packages/app/src/mutes.ts

@@ -1,6 +1,13 @@
 import {MUTES, asDecryptedEvent, readList} from "@welshman/util"
 import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {ensurePlaintext} from "./plaintext.js"
 import {makeOutboxLoader} from "./relayLists.js"

packages/app/src/pins.ts

@@ -1,6 +1,13 @@
 import {PINS, asDecryptedEvent, readList} from "@welshman/util"
-import {TrustedEvent, PublishedList} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {TrustedEvent} from "@welshman/util"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoader} from "./relayLists.js"
 

packages/app/src/profiles.ts

@@ -1,6 +1,13 @@
 import {derived, readable} from "svelte/store"
 import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+import {
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {repository} from "./core.js"
 import {makeOutboxLoaderWithIndexers} from "./relayLists.js"
 

packages/app/src/relayLists.ts

@@ -1,13 +1,13 @@
 import {batcher} from "@welshman/lib"
+import {RELAYS, Filter, asDecryptedEvent, readList, TrustedEvent} from "@welshman/util"
 import {
-  RELAYS,
-  Filter,
-  asDecryptedEvent,
-  readList,
-  TrustedEvent,
-  PublishedList,
-} from "@welshman/util"
-import {deriveItemsByKey, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem, getter} from "@welshman/store"
+  deriveItemsByKey,
+  deriveItems,
+  makeForceLoadItem,
+  makeLoadItem,
+  makeDeriveItem,
+  getter,
+} from "@welshman/store"
 import {load, LoadOptions} from "@welshman/net"
 import {Router} from "@welshman/router"
 import {repository} from "./core.js"
@@ -56,7 +56,10 @@ export const getRelayLists = getter(relayLists)
 
 export const getRelayList = (pubkey: string) => getRelayListsByPubkey().get(pubkey)
 
-export const forceLoadRelayList = makeForceLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelayList)
+export const forceLoadRelayList = makeForceLoadItem(
+  makeOutboxLoaderWithIndexers(RELAYS),
+  getRelayList,
+)
 
 export const loadRelayList = makeLoadItem(makeOutboxLoaderWithIndexers(RELAYS), getRelayList)
 

packages/app/src/relayStats.ts

@@ -1,6 +1,6 @@
-import {writable, derived} from "svelte/store"
-import {withGetter} from "@welshman/store"
-import {prop, groupBy, indexBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib"
+import {writable, Subscriber} from "svelte/store"
+import {getter, makeDeriveItem} from "@welshman/store"
+import {groupBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib"
 import {isOnionUrl, isLocalUrl, isIPAddress, isRelayUrl} from "@welshman/util"
 import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
 
@@ -48,20 +48,34 @@ export const makeRelayStats = (url: string): RelayStats => ({
   notice_count: 0,
 })
 
-export const relayStats = withGetter(writable<RelayStats[]>([]))
+export const relayStatsByUrl = writable(new Map<string, RelayStats>())
 
-export const relayStatsByUrl = withGetter(
-  derived(relayStats, $relayStats => indexBy(prop("url"), $relayStats)),
-)
+export const getRelayStatsByUrl = getter(relayStatsByUrl)
 
-export const deriveRelayStats = (url: string) =>
-  derived(relayStatsByUrl, $relayStatsByUrl => $relayStatsByUrl.get(url))
+export const getRelayStats = (url: string) => getRelayStatsByUrl().get(url)
+
+export const relayStatsSubscribers: Subscriber<RelayStats>[] = []
+
+export const notifyRelayStats = (relayStats: RelayStats) =>
+  relayStatsSubscribers.forEach(sub => sub(relayStats))
+
+export const onRelayStats = (sub: (relayStats: RelayStats) => void) => {
+  relayStatsSubscribers.push(sub)
+
+  return () =>
+    relayStatsSubscribers.splice(
+      relayStatsSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
+export const deriveRelayStats = makeDeriveItem(relayStatsByUrl)
 
 export const getRelayQuality = (url: string) => {
   // Skip non-relays entirely
   if (!isRelayUrl(url)) return 0
 
-  const relayStats = relayStatsByUrl.get().get(url)
+  const relayStats = getRelayStats(url)
 
   // If we have recent errors, skip it
   if (relayStats) {
@@ -90,9 +104,7 @@ export const getRelayQuality = (url: string) => {
 type RelayStatsUpdate = [string, (stats: RelayStats) => void]
 
 const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
-  relayStats.update($relayStats => {
-    const $relayStatsByUrl = indexBy(r => r.url, $relayStats)
-
+  relayStatsByUrl.update($relayStatsByUrl => {
     for (const [url, items] of groupBy(([url]) => url, updates)) {
       if (!url || !isRelayUrl(url)) {
         console.warn(`Attempted to update stats for an invalid relay url: ${url}`)
@@ -109,7 +121,7 @@ const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
       $relayStatsByUrl.set(url, {...$relayStatsItem})
     }
 
-    return Array.from($relayStatsByUrl.values())
+    return $relayStatsByUrl
   })
 })
 

packages/app/src/relays.ts

@@ -1,18 +1,7 @@
-import {writable, derived} from "svelte/store"
-import {
-  uniq,
-  removeUndefined,
-  prop,
-  indexBy,
-  batcher,
-  fetchJson,
-  postJson,
-  Maybe,
-  noop,
-} from "@welshman/lib"
-import {withGetter} from "@welshman/store"
+import {writable, derived, Subscriber} from "svelte/store"
+import {batcher, fetchJson, postJson, Maybe, noop} from "@welshman/lib"
 import {RelayProfile} from "@welshman/util"
-import {normalizeRelayUrl, displayRelayUrl, displayRelayProfile, isRelayUrl} from "@welshman/util"
+import {displayRelayUrl, displayRelayProfile} from "@welshman/util"
 import {getter, deriveItems, makeForceLoadItem, makeLoadItem, makeDeriveItem} from "@welshman/store"
 import {appContext} from "./context.js"
 
@@ -26,6 +15,20 @@ export const getRelays = getter(relays)
 
 export const getRelay = (url: string) => getRelaysByUrl().get(url)
 
+export const relaySubscribers: Subscriber<RelayProfile>[] = []
+
+export const notifyRelay = (relay: RelayProfile) => relaySubscribers.forEach(sub => sub(relay))
+
+export const onRelay = (sub: (relay: RelayProfile) => void) => {
+  relaySubscribers.push(sub)
+
+  return () =>
+    relaySubscribers.splice(
+      relaySubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchRelayDirectly = async (url: string): Promise<Maybe<RelayProfile>> => {
   try {
     const json = fetchJson(url.replace(/^ws/, "http"), {
@@ -35,7 +38,17 @@ export const fetchRelayDirectly = async (url: string): Promise<Maybe<RelayProfil
     })
 
     if (json) {
-      return {...json, url}
+      const info = {...json, url}
+
+      relaysByUrl.update($relaysByUrl => {
+        $relaysByUrl.set(url, info)
+
+        return $relaysByUrl
+      })
+
+      notifyRelay(info)
+
+      return info
     }
   } catch (e) {
     // pass
@@ -49,19 +62,27 @@ export const fetchRelayUsingProxy = batcher(800, async (urls: string[]) => {
   }
 
   const res: any = await postJson(`${appContext.dufflepudUrl}/relay/info`, {urls})
-  const relaysByUrl = new Map<string, RelayProfile>()
+  const result = new Map<string, RelayProfile>()
 
   for (const {url, info} of res?.data || []) {
-    relaysByUrl.set(url, info)
+    if (info) {
+      result.set(url, {...info, url})
+    }
   }
 
-  return urls.map(url => {
-    const info = relaysByUrl.get(url)
-
-    if (info) {
-      return {...info, url}
+  relaysByUrl.update($relaysByUrl => {
+    for (const [url, info] of result) {
+      $relaysByUrl.set(url, info)
     }
+
+    return $relaysByUrl
   })
+
+  for (const info of result.values()) {
+    notifyRelay(info)
+  }
+
+  return urls.map(url => result.get(url))
 })
 
 export const fetchRelay = (url: string) =>

packages/app/src/topics.ts

@@ -1,4 +1,4 @@
-import {readable} from 'svelte/store'
+import {readable} from "svelte/store"
 import {on, call} from "@welshman/lib"
 import {deriveItems} from "@welshman/store"
 import {getTopicTagValues} from "@welshman/util"
@@ -29,7 +29,7 @@ export const topicsByName = call(() => {
   }
 
   return readable<Map<string, Topic>>(topicsByName, set => {
-    return on(repository, 'update', ({added}) => {
+    return on(repository, "update", ({added}) => {
       let dirty = false
 
       for (const event of added) {

packages/app/src/user.ts

@@ -5,9 +5,17 @@ import {profilesByPubkey, forceLoadProfile, loadProfile} from "./profiles.js"
 import {followListsByPubkey, forceLoadFollowList, loadFollowList} from "./follows.js"
 import {pinListsByPubkey, forceLoadPinList, loadPinList} from "./pins.js"
 import {muteListsByPubkey, forceLoadMuteList, loadMuteList} from "./mutes.js"
-import {blossomServerListsByPubkey, forceLoadBlossomServerList, loadBlossomServerList} from "./blossom.js"
+import {
+  blossomServerListsByPubkey,
+  forceLoadBlossomServerList,
+  loadBlossomServerList,
+} from "./blossom.js"
 import {relayListsByPubkey, forceLoadRelayList, loadRelayList} from "./relayLists.js"
-import {messagingRelayListsByPubkey, forceLoadMessagingRelayList, loadMessagingRelayList} from "./messagingRelayLists.js"
+import {
+  messagingRelayListsByPubkey,
+  forceLoadMessagingRelayList,
+  loadMessagingRelayList,
+} from "./messagingRelayLists.js"
 import {wotGraph} from "./wot.js"
 
 export type UserDataLoader = (pubkey: string, relays?: string[], force?: boolean) => unknown

packages/app/src/wot.ts

@@ -6,11 +6,9 @@ import {pubkey} from "./session.js"
 import {followLists, getFollowListsByPubkey, getFollowList} from "./follows.js"
 import {muteLists, getMuteList} from "./mutes.js"
 
-export const getFollows = (pubkey: string) =>
-  getPubkeyTagValues(getListTags(getFollowList(pubkey)))
+export const getFollows = (pubkey: string) => getPubkeyTagValues(getListTags(getFollowList(pubkey)))
 
-export const getMutes = (pubkey: string) =>
-  getPubkeyTagValues(getListTags(getMuteList(pubkey)))
+export const getMutes = (pubkey: string) => getPubkeyTagValues(getListTags(getMuteList(pubkey)))
 
 export const getNetwork = (pubkey: string) => {
   const pubkeys = new Set(getFollows(pubkey))

packages/app/src/zappers.ts

@@ -1,9 +1,8 @@
-import {writable, derived} from "svelte/store"
+import {writable, derived, Subscriber} from "svelte/store"
 import {Zapper, TrustedEvent, Zap, getTagValues, getLnUrl, zapFromEvent} from "@welshman/util"
 import {
   removeUndefined,
   fetchJson,
-  uniq,
   bech32ToHex,
   hexToBech32,
   tryCatch,
@@ -22,6 +21,20 @@ export const getZappersByLnurl = getter(zappersByLnurl)
 
 export const getZapper = (lnurl: string) => getZappersByLnurl().get(lnurl)
 
+export const zapperSubscribers: Subscriber<Zapper>[] = []
+
+export const notifyZapper = (zapper: Zapper) => zapperSubscribers.forEach(sub => sub(zapper))
+
+export const onZapper = (sub: (zapper: Zapper) => void) => {
+  zapperSubscribers.push(sub)
+
+  return () =>
+    zapperSubscribers.splice(
+      zapperSubscribers.findIndex(s => s === sub),
+      1,
+    )
+}
+
 export const fetchZapper = batcher(800, async (lnurls: string[]) => {
   const base = appContext.dufflepudUrl
   const result = new Map<string, Zapper>()
@@ -35,8 +48,12 @@ export const fetchZapper = batcher(800, async (lnurls: string[]) => {
         async () => await postJson(`${base}/zapper/info`, {lnurls: hexUrls}),
       )
 
-      for (const {lnurl, info} of res?.data || []) {
-        tryCatch(() => result.set(hexToBech32("lnurl", lnurl), info))
+      for (const {hexUrl, info} of res?.data || []) {
+        if (info) {
+          const lnurl = hexToBech32("lnurl", hexUrl)
+
+          tryCatch(() => result.set(lnurl, {...info, lnurl}))
+        }
       }
     }
   } else {
@@ -51,11 +68,23 @@ export const fetchZapper = batcher(800, async (lnurls: string[]) => {
 
     for (const {lnurl, info} of results) {
       if (info) {
-        result.set(lnurl, info)
+        result.set(lnurl, {...info, lnurl})
       }
     }
   }
 
+  zappersByLnurl.update($zappersByLnurl => {
+    for (const [nip05, info] of result) {
+      $zappersByLnurl.set(nip05, info)
+    }
+
+    return $zappersByLnurl
+  })
+
+  for (const info of result.values()) {
+    notifyZapper(info)
+  }
+
   return lnurls.map(lnurl => {
     const info = result.get(lnurl)