Make dufflepud optional, fetch zappers/handles directly

packages/app/src/collection.ts

@@ -17,13 +17,31 @@ export const collection = <T, LoadArgs extends any[]>({
   const indexStore = withGetter(derived(store, $items => indexBy(getKey, $items)))
   const getItem = (key: string) => indexStore.get().get(key)
   const pending = new Map<string, Promise<Maybe<T>>>()
+  const loadAttempts = new Map<string, number>()
 
   const loadItem = async (key: string, ...args: LoadArgs) => {
-    const item = indexStore.get().get(key)
+    const item = undefined//indexStore.get().get(key)
-    const delta = item ? 3600 : 30
+    const freshness = getFreshness(name, key)
 
-    if (getFreshness(name, key) > now() - delta) {
+    if (name === 'zappers' && key === '6e468422dfb74a5738702a8823b9b28168abab8655faacb6853cd0ee15deee93') {
-      return item
+      console.log(item)
+    }
+
+    // If we have an item, reload anyway if it's stale. If not, retry with exponential backoff
+    if (item) {
+      loadAttempts.delete(key)
+
+      if (freshness > now() - 3600) {
+        return item
+      }
+    } else {
+      const attempt = loadAttempts.get(key) || 0
+
+      if (freshness > now() - Math.pow(2, attempt)) {
+        return undefined
+      }
+
+      loadAttempts.set(key, attempt + 1)
     }
 
     if (pending.has(key)) {

packages/app/src/context.ts

@@ -1,6 +1,6 @@
 import {partition} from "@welshman/lib"
 import {defaultOptimizeSubscriptions, getDefaultNetContext as originalGetDefaultNetContext} from "@welshman/net"
-import type {Subscription, RelaysAndFilters} from "@welshman/net"
+import type {Subscription, RelaysAndFilters, NetContext} from "@welshman/net"
 import {unionFilters, isSignedEvent, hasValidSignature} from "@welshman/util"
 import type {TrustedEvent} from "@welshman/util"
 import {tracker, repository} from './core'
@@ -15,7 +15,7 @@ export type AppContext = {
   dufflepudUrl?: string
 }
 
-export const getDefaultNetContext = () => ({
+export const getDefaultNetContext = (overrides: Partial<NetContext> = {}) => ({
   ...originalGetDefaultNetContext(),
   onAuth: onAuth,
   onEvent: (url: string, event: TrustedEvent) => tracker.track(event.id, url),
@@ -35,11 +35,13 @@ export const getDefaultNetContext = () => ({
 
     return selections
   },
+  ...overrides,
 })
 
-export const getDefaultAppContext = () => ({
+export const getDefaultAppContext = (overrides: Partial<AppContext> = {}) => ({
   router: makeRouter(),
   requestDelay: 50,
   requestTimeout: 3000,
+  ...overrides,
 })
 

packages/app/src/handles.ts

@@ -1,7 +1,7 @@
 import {writable, derived} from 'svelte/store'
 import {withGetter} from '@welshman/store'
 import {type SubscribeRequest} from "@welshman/net"
-import {ctx, uniq, uniqBy, batcher, postJson, last} from '@welshman/lib'
+import {ctx, fetchJson, uniq, batcher, postJson, last} from '@welshman/lib'
 import {collection} from './collection'
 import {deriveProfile} from './profiles'
 
@@ -12,20 +12,64 @@ export type Handle = {
   relays?: string[]
 }
 
+export const NIP05_REGEX = /^(?:([\w.+-]+)@)?([\w_-]+(\.[\w_-]+)+)$/
+
+export async function queryProfile(nip05: string) {
+  const match = nip05.match(NIP05_REGEX)
+
+  if (!match) return undefined
+
+  const [_, name = '_', domain] = match
+
+  try {
+    const {names, relays = {}, nip46 = {}} = await fetchJson(`https://${domain}/.well-known/nostr.json?name=${name}`)
+
+    const pubkey = names[name]
+
+    if (!pubkey) {
+      return undefined
+    }
+
+    return {
+      nip05,
+      pubkey,
+      nip46: nip46[pubkey],
+      relays: relays[pubkey],
+    }
+  } catch (_e) {
+    return undefined
+  }
+}
+
 export const handles = withGetter(writable<Handle[]>([]))
 
-export const fetchHandles = (handles: string[]) => {
+export const fetchHandles = async (nip05s: string[]) => {
   const base = ctx.app.dufflepudUrl!
-
-  if (!base) {
-    throw new Error("ctx.app.dufflepudUrl is required to fetch nip05 info")
-  }
-
-  const res: any = postJson(`${base}/handle/info`, {handles})
   const handlesByNip05 = new Map<string, Handle>()
 
-  for (const {handle, info} of res?.data || []) {
+  // Attempt fetching directly first
-    handlesByNip05.set(handle, info)
+  const results = await Promise.all(
+    nip05s.map(async nip05 => ({nip05, info: await queryProfile(nip05)}))
+  )
+
+  const dufflepudNip05s: string[] = []
+
+  // If we got a response, great, if not (due to CORS), proxy via dufflepud
+  for (const {nip05, info} of results) {
+    if (info) {
+      handlesByNip05.set(nip05, info)
+    } else {
+      dufflepudNip05s.push(nip05)
+    }
+  }
+
+  // Fetch via dufflepud if we have an endpoint
+  if (base && dufflepudNip05s.length > 0) {
+    const res: any = await postJson(`${base}/handle/info`, {handles: dufflepudNip05s})
+
+    for (const {handle: nip05, info} of res?.data || []) {
+      handlesByNip05.set(nip05, info)
+    }
   }
 
   return handlesByNip05
@@ -43,12 +87,17 @@ export const {
     const fresh = await fetchHandles(uniq(nip05s))
     const stale = handlesByNip05.get()
     const items: Handle[] = nip05s.map(nip05 => {
-      const handle = fresh.get(nip05) || stale.get(nip05) || {}
+      const newHandle = fresh.get(nip05)
+      const oldHandle = stale.get(nip05)
 
-      return {...handle, nip05}
+      if (newHandle) {
+        stale.set(nip05, {...newHandle, nip05})
+      }
+
+      return {...oldHandle, ...newHandle, nip05}
     })
 
-    handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items]))
+    handles.set(Array.from(stale.values()))
 
     return items
   }),
@@ -64,7 +113,7 @@ export const deriveHandleForPubkey = (pubkey: string, request: Partial<Subscribe
 
       loadHandle($profile.nip05)
 
-      return $handlesByNip05.get($profile?.nip05)
+      return $handlesByNip05.get($profile.nip05)
     }
   )
 

packages/app/src/zappers.ts

@@ -2,26 +2,44 @@ import {writable, derived} from 'svelte/store'
 import {withGetter} from '@welshman/store'
 import {type Zapper} from '@welshman/util'
 import {type SubscribeRequest} from "@welshman/net"
-import {ctx, uniq, identity, bech32ToHex, tryCatch, uniqBy, batcher, postJson} from '@welshman/lib'
+import {ctx, fetchJson, uniq, bech32ToHex, hexToBech32, tryCatch, batcher, postJson} from '@welshman/lib'
 import {collection} from './collection'
 import {deriveProfile} from './profiles'
 
 export const zappers = withGetter(writable<Zapper[]>([]))
 
-export const fetchZappers = (lnurls: string[]) => {
+export const fetchZappers = async (lnurls: string[]) => {
   const base = ctx.app.dufflepudUrl!
+  const zappersByLnurl = new Map<string, Zapper>()
 
-  if (!base) {
+  // Attempt fetching directly first
-    throw new Error("ctx.app.dufflepudUrl is required to fetch zapper info")
+  const results = await Promise.all(
+    lnurls.map(async lnurl => {
+      const hexUrl = tryCatch(() => bech32ToHex(lnurl))
+      const info = hexUrl ? await fetchJson(hexUrl) : undefined
+
+      return {lnurl, hexUrl, info}
+    })
+  )
+
+  const dufflepudLnurls: string[] = []
+
+  // If we got a response, great, if not (due to CORS), proxy via dufflepud
+  for (const {lnurl, hexUrl, info} of results) {
+    if (info) {
+      zappersByLnurl.set(lnurl, info)
+    } else if (hexUrl) {
+      dufflepudLnurls.push(hexUrl)
+    }
   }
 
-  const zappersByLnurl = new Map<string, Zapper>()
+  // Fetch via dufflepud if we have an endpoint
-  const res: any = postJson(`${base}/zapper/info`, {
+  if (base && dufflepudLnurls.length > 0) {
-    lnurls: lnurls.map(lnurl => tryCatch(() => bech32ToHex(lnurl))).filter(identity),
+    const res: any = await postJson(`${base}/zapper/info`, {lnurls: dufflepudLnurls})
-  })
 
-  for (const {lnurl, info} of res?.data || []) {
+    for (const {lnurl, info} of res?.data || []) {
-    tryCatch(() => zappersByLnurl.set(bech32ToHex(lnurl), info))
+      tryCatch(() => zappersByLnurl.set(hexToBech32("lnurl", lnurl), info))
+    }
   }
 
   return zappersByLnurl
@@ -39,12 +57,17 @@ export const {
     const fresh = await fetchZappers(uniq(lnurls))
     const stale = zappersByLnurl.get()
     const items: Zapper[] = lnurls.map(lnurl => {
-      const zapper = fresh.get(lnurl) || stale.get(lnurl) || {}
+      const newZapper = fresh.get(lnurl)
+      const oldZapper = stale.get(lnurl)
 
-      return {...zapper, lnurl}
+      if (newZapper) {
+        stale.set(lnurl, {...newZapper, lnurl})
+      }
+
+      return {...oldZapper, ...newZapper, lnurl}
     })
 
-    zappers.update($zappers => uniqBy($zapper => $zapper.lnurl, [...$zappers, ...items]))
+    zappers.set(Array.from(stale.values()))
 
     return items
   }),
@@ -60,7 +83,7 @@ export const deriveZapperForPubkey = (pubkey: string, request: Partial<Subscribe
 
       loadZapper($profile.lnurl)
 
-      return $zappersByLnurl.get($profile?.lnurl)
+      return $zappersByLnurl.get($profile.lnurl)
     }
   )
 

packages/net/src/Context.ts

@@ -27,7 +27,7 @@ export const defaultOptimizeSubscriptions = (subs: Subscription[]) =>
       return {relays: [relay], filters}
     })
 
-export const getDefaultNetContext = () => ({
+export const getDefaultNetContext = (overrides: Partial<NetContext> = {}) => ({
   onOk: noop,
   onAuth: noop,
   onEvent: noop,
@@ -37,4 +37,5 @@ export const getDefaultNetContext = () => ({
   getExecutor: (relays: string[]) => new Executor(new Relays(relays.map((relay: string) => ctx.net.pool.get(relay)))),
   matchFilters: (url: string, filters: Filter[], event: TrustedEvent) => matchFilters(filters, event),
   optimizeSubscriptions: defaultOptimizeSubscriptions,
+  ...overrides,
 })

packages/net/src/Socket.ts

@@ -42,7 +42,7 @@ export class Socket {
       if (Array.isArray(message)) {
         this.opts.onMessage(message as Message)
       } else {
-        console.warn("Invalid messages received:", message)
+        console.warn(`Invalid message received on ${this.url}:`, message)
       }
     } catch (e) {
       // pass