From bf6c240c7358e29018b2022479189b7971aeb23b Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Fri, 24 Oct 2025 09:34:50 -0700 Subject: [PATCH] Split relays and relay stats up --- packages/app/src/index.ts | 21 +- packages/app/src/relayStats.ts | 224 +++++++++++++++ packages/app/src/relays.ts | 265 +----------------- packages/app/src/search.ts | 6 +- packages/app/src/sync.ts | 8 +- .../editor/src/plugins/TippySuggestion.ts | 45 +-- 6 files changed, 279 insertions(+), 290 deletions(-) create mode 100644 packages/app/src/relayStats.ts diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts index 30ff9ec..2af885a 100644 --- a/packages/app/src/index.ts +++ b/packages/app/src/index.ts @@ -10,6 +10,7 @@ export * from "./plaintext.js" export * from "./profiles.js" export * from "./pins.js" export * from "./relays.js" +export * from "./relayStats.js" export * from "./relaySelections.js" export * from "./inboxRelaySelections.js" export * from "./search.js" @@ -23,13 +24,21 @@ export * from "./wot.js" export * from "./zappers.js" import {derived} from "svelte/store" -import {sortBy, throttleWithValue, tryCatch} from "@welshman/lib" -import {isEphemeralKind, isDVMKind, WRAP, RelayMode, getRelaysFromList} from "@welshman/util" +import {sortBy, throttleWithValue} from "@welshman/lib" +import { + isEphemeralKind, + isDVMKind, + WRAP, + RelayMode, + RelayProfile, + getRelaysFromList, +} from "@welshman/util" import {routerContext} from "@welshman/router" import {Pool, SocketEvent, isRelayEvent, netContext} from "@welshman/net" import {pubkey, unwrapAndStore} from "./session.js" import {repository, tracker} from "./core.js" -import {Relay, relays, loadRelay, trackRelayStats, getRelayQuality} from "./relays.js" +import {relays, loadRelay} from "./relays.js" +import {trackRelayStats, getRelayQuality} from "./relayStats.js" import {relaySelectionsByPubkey} from "./relaySelections.js" import {inboxRelaySelectionsByPubkey} from "./inboxRelaySelections.js" @@ -62,7 +71,7 @@ Pool.get().subscribe(socket => { // Configure the router and add a few other relay utils -const _relayGetter = (fn?: (relay: Relay) => any) => +const _relayGetter = (fn?: (relay: RelayProfile) => any) => throttleWithValue(200, () => { let _relays = relays.get() @@ -90,6 +99,4 @@ routerContext.getPubkeyRelays = getPubkeyRelays routerContext.getRelayQuality = getRelayQuality routerContext.getDefaultRelays = _relayGetter() routerContext.getIndexerRelays = _relayGetter() -routerContext.getSearchRelays = _relayGetter(r => - tryCatch(() => r.profile?.supported_nips?.includes(50)), -) +routerContext.getSearchRelays = _relayGetter(r => r?.supported_nips?.includes?.(50)) diff --git a/packages/app/src/relayStats.ts b/packages/app/src/relayStats.ts new file mode 100644 index 0000000..341f685 --- /dev/null +++ b/packages/app/src/relayStats.ts @@ -0,0 +1,224 @@ +import {writable, derived} from "svelte/store" +import {withGetter} from "@welshman/store" +import {prop, groupBy, indexBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib" +import {isOnionUrl, isLocalUrl, isIPAddress, isRelayUrl} from "@welshman/util" +import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net" + +export type RelayStats = { + url: string + first_seen: number + recent_errors: number[] + open_count: number + close_count: number + publish_count: number + request_count: number + event_count: number + last_open: number + last_close: number + last_error: number + last_publish: number + last_request: number + last_event: number + last_auth: number + publish_success_count: number + publish_failure_count: number + eose_count: number + notice_count: number +} + +export const makeRelayStats = (url: string): RelayStats => ({ + url, + first_seen: now(), + recent_errors: [], + open_count: 0, + close_count: 0, + publish_count: 0, + request_count: 0, + event_count: 0, + last_open: 0, + last_close: 0, + last_error: 0, + last_publish: 0, + last_request: 0, + last_event: 0, + last_auth: 0, + publish_success_count: 0, + publish_failure_count: 0, + eose_count: 0, + notice_count: 0, +}) + +export const relayStats = withGetter(writable([])) + +export const relayStatsByUrl = withGetter( + derived(relayStats, $relayStats => indexBy(prop("url"), $relayStats)), +) + +export const deriveRelayStats = (url: string) => + derived(relayStatsByUrl, $relayStatsByUrl => $relayStatsByUrl.get(url)) + +export const getRelayQuality = (url: string) => { + // Skip non-relays entirely + if (!isRelayUrl(url)) return 0 + + const relayStats = relayStatsByUrl.get().get(url) + + // If we have recent errors, skip it + if (relayStats) { + if (relayStats.recent_errors.filter(n => n > ago(MINUTE)).length > 0) return 0 + if (relayStats.recent_errors.filter(n => n > ago(HOUR)).length > 3) return 0 + if (relayStats.recent_errors.filter(n => n > ago(DAY)).length > 10) return 0 + } + + // Prefer stuff we're connected to + if (Pool.get().has(url)) return 1 + + // Prefer stuff we've connected to in the past + if (relayStats) return 0.9 + + // If it's not weird url give it an ok score + if (!isIPAddress(url) && !isLocalUrl(url) && !isOnionUrl(url) && !url.startsWith("ws://")) { + return 0.8 + } + + // Default to a "meh" score + return 0.7 +} + +// Utilities for syncing stats from connections to relays + +type RelayStatsUpdate = [string, (stats: RelayStats) => void] + +const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => { + relayStats.update($relayStats => { + const $relayStatsByUrl = indexBy(r => r.url, $relayStats) + + for (const [url, items] of groupBy(([url]) => url, updates)) { + if (!url || !isRelayUrl(url)) { + console.warn(`Attempted to update stats for an invalid relay url: ${url}`) + continue + } + + const $relayStatsItem: RelayStats = $relayStatsByUrl.get(url) || makeRelayStats(url) + + for (const [_, update] of items) { + update($relayStatsItem) + } + + // Copy so the database gets updated, since we're mutating in updates + $relayStatsByUrl.set(url, {...$relayStatsItem}) + } + + return Array.from($relayStatsByUrl.values()) + }) +}) + +const onSocketSend = ([verb]: ClientMessage, url: string) => { + if (verb === "REQ") { + updateRelayStats([ + url, + stats => { + stats.request_count++ + stats.last_request = now() + }, + ]) + } else if (verb === "EVENT") { + updateRelayStats([ + url, + stats => { + stats.publish_count++ + stats.last_publish = now() + }, + ]) + } +} + +const onSocketReceive = ([verb, ...extra]: RelayMessage, url: string) => { + if (verb === "OK") { + const [_, ok] = extra + + updateRelayStats([ + url, + stats => { + if (ok) { + stats.publish_success_count++ + } else { + stats.publish_failure_count++ + } + }, + ]) + } else if (verb === "AUTH") { + updateRelayStats([ + url, + stats => { + stats.last_auth = now() + }, + ]) + } else if (verb === "EVENT") { + updateRelayStats([ + url, + stats => { + stats.event_count++ + stats.last_event = now() + }, + ]) + } else if (verb === "EOSE") { + updateRelayStats([ + url, + stats => { + stats.eose_count++ + }, + ]) + } else if (verb === "NOTICE") { + updateRelayStats([ + url, + stats => { + stats.notice_count++ + }, + ]) + } +} + +const onSocketStatus = (status: string, url: string) => { + if (status === SocketStatus.Open) { + updateRelayStats([ + url, + stats => { + stats.last_open = now() + stats.open_count++ + }, + ]) + } + + if (status === SocketStatus.Closed) { + updateRelayStats([ + url, + stats => { + stats.last_close = now() + stats.close_count++ + }, + ]) + } + + if (status === SocketStatus.Error) { + updateRelayStats([ + url, + stats => { + stats.last_error = now() + stats.recent_errors = uniq(stats.recent_errors.concat(now())).slice(-10) + }, + ]) + } +} + +export const trackRelayStats = (socket: Socket) => { + socket.on(SocketEvent.Send, onSocketSend) + socket.on(SocketEvent.Receive, onSocketReceive) + socket.on(SocketEvent.Status, onSocketStatus) + + return () => { + socket.off(SocketEvent.Send, onSocketSend) + socket.off(SocketEvent.Receive, onSocketReceive) + socket.off(SocketEvent.Status, onSocketStatus) + } +} diff --git a/packages/app/src/relays.ts b/packages/app/src/relays.ts index fc89f8e..0849953 100644 --- a/packages/app/src/relays.ts +++ b/packages/app/src/relays.ts @@ -1,88 +1,12 @@ import {writable, derived} from "svelte/store" import {withGetter} from "@welshman/store" -import { - groupBy, - indexBy, - batch, - now, - uniq, - batcher, - postJson, - ago, - DAY, - HOUR, - MINUTE, -} from "@welshman/lib" +import {uniq, batcher, postJson} from "@welshman/lib" import {RelayProfile} from "@welshman/util" -import { - normalizeRelayUrl, - displayRelayUrl, - displayRelayProfile, - isOnionUrl, - isLocalUrl, - isIPAddress, - isRelayUrl, -} from "@welshman/util" -import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net" +import {normalizeRelayUrl, displayRelayUrl, displayRelayProfile, isRelayUrl} from "@welshman/util" import {collection} from "@welshman/store" import {appContext} from "./context.js" -export type RelayStats = { - first_seen: number - recent_errors: number[] - open_count: number - close_count: number - publish_count: number - request_count: number - event_count: number - last_open: number - last_close: number - last_error: number - last_publish: number - last_request: number - last_event: number - last_auth: number - publish_success_count: number - publish_failure_count: number - eose_count: number - notice_count: number -} - -export const makeRelayStats = (): RelayStats => ({ - first_seen: now(), - recent_errors: [], - open_count: 0, - close_count: 0, - publish_count: 0, - request_count: 0, - event_count: 0, - last_open: 0, - last_close: 0, - last_error: 0, - last_publish: 0, - last_request: 0, - last_event: 0, - last_auth: 0, - publish_success_count: 0, - publish_failure_count: 0, - eose_count: 0, - notice_count: 0, -}) - -export type Relay = { - url: string - stats?: RelayStats - profile?: RelayProfile -} - -export const relays = withGetter(writable([])) - -export const relaysByPubkey = derived(relays, $relays => - groupBy( - $relay => $relay.profile?.pubkey, - $relays.filter($relay => $relay.profile?.pubkey), - ), -) +export const relays = withGetter(writable([])) export const fetchRelayProfiles = async (urls: string[]) => { const profilesByUrl = new Map() @@ -122,14 +46,13 @@ export const { } = collection({ name: "relays", store: relays, - getKey: (relay: Relay) => relay.url, + getKey: (relay: RelayProfile) => relay.url, load: batcher(800, async (rawUrls: string[]) => { const urls = rawUrls.map(normalizeRelayUrl) const fresh = await fetchRelayProfiles(uniq(urls)) const stale = relaysByUrl.get() for (const url of urls) { - const relay = stale.get(url) const profile = fresh.get(url) if (!url || !isRelayUrl(url)) { @@ -138,7 +61,7 @@ export const { } if (profile) { - stale.set(url, {...relay, profile, url}) + stale.set(url, {...profile, url}) } } @@ -149,181 +72,7 @@ export const { }) export const displayRelayByPubkey = (url: string) => - displayRelayProfile(relaysByUrl.get().get(url)?.profile, displayRelayUrl(url)) + displayRelayProfile(relaysByUrl.get().get(url), displayRelayUrl(url)) export const deriveRelayDisplay = (url: string) => - derived(deriveRelay(url), $relay => displayRelayProfile($relay?.profile, displayRelayUrl(url))) - -export const getRelayQuality = (url: string) => { - const relay = relaysByUrl.get().get(url) - - // Skip non-relays entirely - if (!isRelayUrl(url)) return 0 - - // If we have recent errors, skip it - if (relay?.stats) { - if (relay.stats.recent_errors.filter(n => n > ago(MINUTE)).length > 0) return 0 - if (relay.stats.recent_errors.filter(n => n > ago(HOUR)).length > 3) return 0 - if (relay.stats.recent_errors.filter(n => n > ago(DAY)).length > 10) return 0 - } - - // Prefer stuff we're connected to - if (Pool.get().has(url)) return 1 - - // Prefer stuff we've connected to in the past - if (relay?.stats) return 0.9 - - // If it's not weird url give it an ok score - if (!isIPAddress(url) && !isLocalUrl(url) && !isOnionUrl(url) && !url.startsWith("ws://")) { - return 0.8 - } - - // Default to a "meh" score - return 0.7 -} - -// Utilities for syncing stats from connections to relays - -type RelayStatsUpdate = [string, (stats: RelayStats) => void] - -const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => { - relays.update($relays => { - const $relaysByUrl = indexBy(r => r.url, $relays) - const $itemsByUrl = groupBy(([url]) => url, updates) - - for (const [url, items] of $itemsByUrl.entries()) { - const $relay: Relay = $relaysByUrl.get(url) || {url} - - if (!url || !isRelayUrl(url)) { - console.warn(`Attempted to update stats for an invalid relay url: ${url}`) - continue - } - - if (!$relay.stats) { - $relay.stats = makeRelayStats() - } else if ($relay.stats.notice_count === undefined) { - // Migrate from old stats - $relay.stats = {...makeRelayStats(), ...$relay.stats} - } - - for (const [_, update] of items) { - update($relay.stats) - } - - // Copy so the database gets updated, since we're mutating in updates - $relaysByUrl.set(url, {...$relay}) - } - - return Array.from($relaysByUrl.values()) - }) -}) - -const onSocketSend = ([verb]: ClientMessage, url: string) => { - if (verb === "REQ") { - updateRelayStats([ - url, - stats => { - stats.request_count++ - stats.last_request = now() - }, - ]) - } else if (verb === "EVENT") { - updateRelayStats([ - url, - stats => { - stats.publish_count++ - stats.last_publish = now() - }, - ]) - } -} - -const onSocketReceive = ([verb, ...extra]: RelayMessage, url: string) => { - if (verb === "OK") { - const [_, ok] = extra - - updateRelayStats([ - url, - stats => { - if (ok) { - stats.publish_success_count++ - } else { - stats.publish_failure_count++ - } - }, - ]) - } else if (verb === "AUTH") { - updateRelayStats([ - url, - stats => { - stats.last_auth = now() - }, - ]) - } else if (verb === "EVENT") { - updateRelayStats([ - url, - stats => { - stats.event_count++ - stats.last_event = now() - }, - ]) - } else if (verb === "EOSE") { - updateRelayStats([ - url, - stats => { - stats.eose_count++ - }, - ]) - } else if (verb === "NOTICE") { - updateRelayStats([ - url, - stats => { - stats.notice_count++ - }, - ]) - } -} - -const onSocketStatus = (status: string, url: string) => { - if (status === SocketStatus.Open) { - updateRelayStats([ - url, - stats => { - stats.last_open = now() - stats.open_count++ - }, - ]) - } - - if (status === SocketStatus.Closed) { - updateRelayStats([ - url, - stats => { - stats.last_close = now() - stats.close_count++ - }, - ]) - } - - if (status === SocketStatus.Error) { - updateRelayStats([ - url, - stats => { - stats.last_error = now() - stats.recent_errors = uniq(stats.recent_errors.concat(now())).slice(-10) - }, - ]) - } -} - -export const trackRelayStats = (socket: Socket) => { - socket.on(SocketEvent.Send, onSocketSend) - socket.on(SocketEvent.Receive, onSocketReceive) - socket.on(SocketEvent.Status, onSocketStatus) - - return () => { - socket.off(SocketEvent.Send, onSocketSend) - socket.off(SocketEvent.Receive, onSocketReceive) - socket.off(SocketEvent.Status, onSocketStatus) - } -} + derived(deriveRelay(url), $relay => displayRelayProfile($relay, displayRelayUrl(url))) diff --git a/packages/app/src/search.ts b/packages/app/src/search.ts index 8172d70..d629e6b 100644 --- a/packages/app/src/search.ts +++ b/packages/app/src/search.ts @@ -2,14 +2,14 @@ import Fuse, {IFuseOptions, FuseResult} from "fuse.js" import {debounce} from "throttle-debounce" import {derived} from "svelte/store" import {dec, inc, sortBy} from "@welshman/lib" -import {PROFILE, PublishedProfile} from "@welshman/util" +import {PROFILE, PublishedProfile, RelayProfile} from "@welshman/util" import {load} from "@welshman/net" import {throttled} from "@welshman/store" import {Router} from "@welshman/router" import {wotGraph, maxWot} from "./wot.js" import {profiles} from "./profiles.js" import {topics, Topic} from "./topics.js" -import {relays, Relay} from "./relays.js" +import {relays} from "./relays.js" import {handlesByNip05} from "./handles.js" export type SearchOptions = { @@ -102,7 +102,7 @@ export const topicSearch = derived(topics, $topics => export const relaySearch = derived(relays, $relays => createSearch($relays, { - getValue: (relay: Relay) => relay.url, + getValue: (relay: RelayProfile) => relay.url, fuseOptions: { keys: ["url", "name", {name: "description", weight: 0.3}], }, diff --git a/packages/app/src/sync.ts b/packages/app/src/sync.ts index 57e2792..381502a 100644 --- a/packages/app/src/sync.ts +++ b/packages/app/src/sync.ts @@ -8,11 +8,11 @@ const query = (filters: Filter[]) => repository.query(filters, {shouldSort: filters.every(f => f.limit === undefined)}) export const hasNegentropy = (url: string) => { - const p = relaysByUrl.get().get(url)?.profile + const relay = relaysByUrl.get().get(url) - if (p?.negentropy) return true - if (p?.supported_nips?.includes?.(77)) return true - if (p?.software?.includes?.("strfry") && !p?.version?.match(/^0\./)) return true + if (relay?.negentropy) return true + if (relay?.supported_nips?.includes?.(77)) return true + if (relay?.software?.includes?.("strfry") && !relay?.version?.match(/^0\./)) return true return false } diff --git a/packages/editor/src/plugins/TippySuggestion.ts b/packages/editor/src/plugins/TippySuggestion.ts index 4aa57d4..74aef7f 100644 --- a/packages/editor/src/plugins/TippySuggestion.ts +++ b/packages/editor/src/plugins/TippySuggestion.ts @@ -25,24 +25,12 @@ export type SuggestionsWrapperProps = { createSuggestion: CreateSuggestion } -export interface ISuggestionsWrapperConstructor { - new (target: HTMLElement, props: SuggestionsWrapperProps): ISuggestionsWrapper -} - export interface ISuggestionsWrapper { - setProps: (props: SuggestionsWrapperProps) => void + setProps: (props: Partial) => void onKeyDown: (event: Event) => boolean destroy: () => void } -function createSuggestionsWrapper( - ctor: ISuggestionsWrapperConstructor, - target: HTMLElement, - props: SuggestionsWrapperProps, -): ISuggestionsWrapper { - return new ctor(target, props) -} - export class DefaultSuggestionsWrapper implements ISuggestionsWrapper { index = 0 items: string[] = [] @@ -126,8 +114,9 @@ export class DefaultSuggestionsWrapper implements ISuggestionsWrapper { this.render() } - setProps(props: SuggestionsWrapperProps) { - this.props = props + setProps(props: Partial) { + Object.assign(this.props, props) + this.search() this.render() } @@ -176,15 +165,27 @@ export class DefaultSuggestionsWrapper implements ISuggestionsWrapper { } } +type Unsubscribe = () => void + +type Subscribe = (fn: () => void) => Unsubscribe + +type Signal = { + subscribe: Subscribe +} + export type TippySuggestionOptions = { char: string name: string editor: Editor search: (term: string) => string[] select: (value: string, props: any) => void + updateSignal?: Signal allowCreate?: boolean createSuggestion?: CreateSuggestion - suggestionsWrapper?: ISuggestionsWrapperConstructor + createSuggestionsWrapper?: ( + target: HTMLElement, + props: SuggestionsWrapperProps, + ) => ISuggestionsWrapper } export const TippySuggestion = ({ @@ -193,9 +194,11 @@ export const TippySuggestion = ({ editor, search, select, + updateSignal, allowCreate = false, createSuggestion = defaultCreateSuggestion, - suggestionsWrapper = DefaultSuggestionsWrapper, + createSuggestionsWrapper = (target: HTMLElement, props: SuggestionsWrapperProps) => + new DefaultSuggestionsWrapper(target, props), }: TippySuggestionOptions) => Suggestion({ char, @@ -231,6 +234,7 @@ export const TippySuggestion = ({ render: () => { let popover: Instance[] let wrapper: ISuggestionsWrapper + let unsubscribe: Unsubscribe | undefined const mapProps = (props: any) => ({ term: props.query, @@ -257,7 +261,11 @@ export const TippySuggestion = ({ if (!props.query) popover[0].hide() - wrapper = createSuggestionsWrapper(suggestionsWrapper, target, mapProps(props)) + wrapper = createSuggestionsWrapper(target, mapProps(props)) + + unsubscribe = updateSignal?.subscribe(() => { + wrapper.setProps({}) + }) }, onUpdate: props => { if (props.query) { @@ -286,6 +294,7 @@ export const TippySuggestion = ({ onExit: () => { popover[0].destroy() wrapper.destroy() + unsubscribe?.() }, } },