From 18db1421a9b042bb18764e3ed05ea16eb45e198a Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Thu, 5 Sep 2024 15:26:45 -0700 Subject: [PATCH] Make dufflepud optional, fetch zappers/handles directly --- packages/app/src/collection.ts | 26 ++++++++++-- packages/app/src/context.ts | 8 ++-- packages/app/src/handles.ts | 77 +++++++++++++++++++++++++++------- packages/app/src/zappers.ts | 51 +++++++++++++++------- packages/net/src/Context.ts | 3 +- packages/net/src/Socket.ts | 2 +- 6 files changed, 130 insertions(+), 37 deletions(-) diff --git a/packages/app/src/collection.ts b/packages/app/src/collection.ts index e5039cc..59d5ff5 100644 --- a/packages/app/src/collection.ts +++ b/packages/app/src/collection.ts @@ -17,13 +17,31 @@ export const collection = ({ const indexStore = withGetter(derived(store, $items => indexBy(getKey, $items))) const getItem = (key: string) => indexStore.get().get(key) const pending = new Map>>() + const loadAttempts = new Map() const loadItem = async (key: string, ...args: LoadArgs) => { - const item = indexStore.get().get(key) - const delta = item ? 3600 : 30 + const item = undefined//indexStore.get().get(key) + const freshness = getFreshness(name, key) - if (getFreshness(name, key) > now() - delta) { - return item + if (name === 'zappers' && key === '6e468422dfb74a5738702a8823b9b28168abab8655faacb6853cd0ee15deee93') { + console.log(item) + } + + // If we have an item, reload anyway if it's stale. If not, retry with exponential backoff + if (item) { + loadAttempts.delete(key) + + if (freshness > now() - 3600) { + return item + } + } else { + const attempt = loadAttempts.get(key) || 0 + + if (freshness > now() - Math.pow(2, attempt)) { + return undefined + } + + loadAttempts.set(key, attempt + 1) } if (pending.has(key)) { diff --git a/packages/app/src/context.ts b/packages/app/src/context.ts index c6c6449..500d196 100644 --- a/packages/app/src/context.ts +++ b/packages/app/src/context.ts @@ -1,6 +1,6 @@ import {partition} from "@welshman/lib" import {defaultOptimizeSubscriptions, getDefaultNetContext as originalGetDefaultNetContext} from "@welshman/net" -import type {Subscription, RelaysAndFilters} from "@welshman/net" +import type {Subscription, RelaysAndFilters, NetContext} from "@welshman/net" import {unionFilters, isSignedEvent, hasValidSignature} from "@welshman/util" import type {TrustedEvent} from "@welshman/util" import {tracker, repository} from './core' @@ -15,7 +15,7 @@ export type AppContext = { dufflepudUrl?: string } -export const getDefaultNetContext = () => ({ +export const getDefaultNetContext = (overrides: Partial = {}) => ({ ...originalGetDefaultNetContext(), onAuth: onAuth, onEvent: (url: string, event: TrustedEvent) => tracker.track(event.id, url), @@ -35,11 +35,13 @@ export const getDefaultNetContext = () => ({ return selections }, + ...overrides, }) -export const getDefaultAppContext = () => ({ +export const getDefaultAppContext = (overrides: Partial = {}) => ({ router: makeRouter(), requestDelay: 50, requestTimeout: 3000, + ...overrides, }) diff --git a/packages/app/src/handles.ts b/packages/app/src/handles.ts index 2353a41..07c3b02 100644 --- a/packages/app/src/handles.ts +++ b/packages/app/src/handles.ts @@ -1,7 +1,7 @@ import {writable, derived} from 'svelte/store' import {withGetter} from '@welshman/store' import {type SubscribeRequest} from "@welshman/net" -import {ctx, uniq, uniqBy, batcher, postJson, last} from '@welshman/lib' +import {ctx, fetchJson, uniq, batcher, postJson, last} from '@welshman/lib' import {collection} from './collection' import {deriveProfile} from './profiles' @@ -12,20 +12,64 @@ export type Handle = { relays?: string[] } +export const NIP05_REGEX = /^(?:([\w.+-]+)@)?([\w_-]+(\.[\w_-]+)+)$/ + +export async function queryProfile(nip05: string) { + const match = nip05.match(NIP05_REGEX) + + if (!match) return undefined + + const [_, name = '_', domain] = match + + try { + const {names, relays = {}, nip46 = {}} = await fetchJson(`https://${domain}/.well-known/nostr.json?name=${name}`) + + const pubkey = names[name] + + if (!pubkey) { + return undefined + } + + return { + nip05, + pubkey, + nip46: nip46[pubkey], + relays: relays[pubkey], + } + } catch (_e) { + return undefined + } +} + export const handles = withGetter(writable([])) -export const fetchHandles = (handles: string[]) => { +export const fetchHandles = async (nip05s: string[]) => { const base = ctx.app.dufflepudUrl! - - if (!base) { - throw new Error("ctx.app.dufflepudUrl is required to fetch nip05 info") - } - - const res: any = postJson(`${base}/handle/info`, {handles}) const handlesByNip05 = new Map() - for (const {handle, info} of res?.data || []) { - handlesByNip05.set(handle, info) + // Attempt fetching directly first + const results = await Promise.all( + nip05s.map(async nip05 => ({nip05, info: await queryProfile(nip05)})) + ) + + const dufflepudNip05s: string[] = [] + + // If we got a response, great, if not (due to CORS), proxy via dufflepud + for (const {nip05, info} of results) { + if (info) { + handlesByNip05.set(nip05, info) + } else { + dufflepudNip05s.push(nip05) + } + } + + // Fetch via dufflepud if we have an endpoint + if (base && dufflepudNip05s.length > 0) { + const res: any = await postJson(`${base}/handle/info`, {handles: dufflepudNip05s}) + + for (const {handle: nip05, info} of res?.data || []) { + handlesByNip05.set(nip05, info) + } } return handlesByNip05 @@ -43,12 +87,17 @@ export const { const fresh = await fetchHandles(uniq(nip05s)) const stale = handlesByNip05.get() const items: Handle[] = nip05s.map(nip05 => { - const handle = fresh.get(nip05) || stale.get(nip05) || {} + const newHandle = fresh.get(nip05) + const oldHandle = stale.get(nip05) - return {...handle, nip05} + if (newHandle) { + stale.set(nip05, {...newHandle, nip05}) + } + + return {...oldHandle, ...newHandle, nip05} }) - handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items])) + handles.set(Array.from(stale.values())) return items }), @@ -64,7 +113,7 @@ export const deriveHandleForPubkey = (pubkey: string, request: Partial([])) -export const fetchZappers = (lnurls: string[]) => { +export const fetchZappers = async (lnurls: string[]) => { const base = ctx.app.dufflepudUrl! + const zappersByLnurl = new Map() - if (!base) { - throw new Error("ctx.app.dufflepudUrl is required to fetch zapper info") + // Attempt fetching directly first + const results = await Promise.all( + lnurls.map(async lnurl => { + const hexUrl = tryCatch(() => bech32ToHex(lnurl)) + const info = hexUrl ? await fetchJson(hexUrl) : undefined + + return {lnurl, hexUrl, info} + }) + ) + + const dufflepudLnurls: string[] = [] + + // If we got a response, great, if not (due to CORS), proxy via dufflepud + for (const {lnurl, hexUrl, info} of results) { + if (info) { + zappersByLnurl.set(lnurl, info) + } else if (hexUrl) { + dufflepudLnurls.push(hexUrl) + } } - const zappersByLnurl = new Map() - const res: any = postJson(`${base}/zapper/info`, { - lnurls: lnurls.map(lnurl => tryCatch(() => bech32ToHex(lnurl))).filter(identity), - }) + // Fetch via dufflepud if we have an endpoint + if (base && dufflepudLnurls.length > 0) { + const res: any = await postJson(`${base}/zapper/info`, {lnurls: dufflepudLnurls}) - for (const {lnurl, info} of res?.data || []) { - tryCatch(() => zappersByLnurl.set(bech32ToHex(lnurl), info)) + for (const {lnurl, info} of res?.data || []) { + tryCatch(() => zappersByLnurl.set(hexToBech32("lnurl", lnurl), info)) + } } return zappersByLnurl @@ -39,12 +57,17 @@ export const { const fresh = await fetchZappers(uniq(lnurls)) const stale = zappersByLnurl.get() const items: Zapper[] = lnurls.map(lnurl => { - const zapper = fresh.get(lnurl) || stale.get(lnurl) || {} + const newZapper = fresh.get(lnurl) + const oldZapper = stale.get(lnurl) - return {...zapper, lnurl} + if (newZapper) { + stale.set(lnurl, {...newZapper, lnurl}) + } + + return {...oldZapper, ...newZapper, lnurl} }) - zappers.update($zappers => uniqBy($zapper => $zapper.lnurl, [...$zappers, ...items])) + zappers.set(Array.from(stale.values())) return items }), @@ -60,7 +83,7 @@ export const deriveZapperForPubkey = (pubkey: string, request: Partial return {relays: [relay], filters} }) -export const getDefaultNetContext = () => ({ +export const getDefaultNetContext = (overrides: Partial = {}) => ({ onOk: noop, onAuth: noop, onEvent: noop, @@ -37,4 +37,5 @@ export const getDefaultNetContext = () => ({ getExecutor: (relays: string[]) => new Executor(new Relays(relays.map((relay: string) => ctx.net.pool.get(relay)))), matchFilters: (url: string, filters: Filter[], event: TrustedEvent) => matchFilters(filters, event), optimizeSubscriptions: defaultOptimizeSubscriptions, + ...overrides, }) diff --git a/packages/net/src/Socket.ts b/packages/net/src/Socket.ts index 1f486f2..2e8f97f 100644 --- a/packages/net/src/Socket.ts +++ b/packages/net/src/Socket.ts @@ -42,7 +42,7 @@ export class Socket { if (Array.isArray(message)) { this.opts.onMessage(message as Message) } else { - console.warn("Invalid messages received:", message) + console.warn(`Invalid message received on ${this.url}:`, message) } } catch (e) { // pass