Split relays and relay stats up
This commit is contained in:
Jon Staab
@@ -10,6 +10,7 @@ export * from "./plaintext.js"
|
||||
export * from "./profiles.js"
|
||||
export * from "./pins.js"
|
||||
export * from "./relays.js"
|
||||
export * from "./relayStats.js"
|
||||
export * from "./relaySelections.js"
|
||||
export * from "./inboxRelaySelections.js"
|
||||
export * from "./search.js"
|
||||
@@ -23,13 +24,21 @@ export * from "./wot.js"
|
||||
export * from "./zappers.js"
|
||||
|
||||
import {derived} from "svelte/store"
|
||||
import {sortBy, throttleWithValue, tryCatch} from "@welshman/lib"
|
||||
import {isEphemeralKind, isDVMKind, WRAP, RelayMode, getRelaysFromList} from "@welshman/util"
|
||||
import {sortBy, throttleWithValue} from "@welshman/lib"
|
||||
import {
|
||||
isEphemeralKind,
|
||||
isDVMKind,
|
||||
WRAP,
|
||||
RelayMode,
|
||||
RelayProfile,
|
||||
getRelaysFromList,
|
||||
} from "@welshman/util"
|
||||
import {routerContext} from "@welshman/router"
|
||||
import {Pool, SocketEvent, isRelayEvent, netContext} from "@welshman/net"
|
||||
import {pubkey, unwrapAndStore} from "./session.js"
|
||||
import {repository, tracker} from "./core.js"
|
||||
import {Relay, relays, loadRelay, trackRelayStats, getRelayQuality} from "./relays.js"
|
||||
import {relays, loadRelay} from "./relays.js"
|
||||
import {trackRelayStats, getRelayQuality} from "./relayStats.js"
|
||||
import {relaySelectionsByPubkey} from "./relaySelections.js"
|
||||
import {inboxRelaySelectionsByPubkey} from "./inboxRelaySelections.js"
|
||||
|
||||
@@ -62,7 +71,7 @@ Pool.get().subscribe(socket => {
|
||||
|
||||
// Configure the router and add a few other relay utils
|
||||
|
||||
const _relayGetter = (fn?: (relay: Relay) => any) =>
|
||||
const _relayGetter = (fn?: (relay: RelayProfile) => any) =>
|
||||
throttleWithValue(200, () => {
|
||||
let _relays = relays.get()
|
||||
|
||||
@@ -90,6 +99,4 @@ routerContext.getPubkeyRelays = getPubkeyRelays
|
||||
routerContext.getRelayQuality = getRelayQuality
|
||||
routerContext.getDefaultRelays = _relayGetter()
|
||||
routerContext.getIndexerRelays = _relayGetter()
|
||||
routerContext.getSearchRelays = _relayGetter(r =>
|
||||
tryCatch(() => r.profile?.supported_nips?.includes(50)),
|
||||
)
|
||||
routerContext.getSearchRelays = _relayGetter(r => r?.supported_nips?.includes?.(50))
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
import {writable, derived} from "svelte/store"
|
||||
import {withGetter} from "@welshman/store"
|
||||
import {prop, groupBy, indexBy, batch, now, uniq, ago, DAY, HOUR, MINUTE} from "@welshman/lib"
|
||||
import {isOnionUrl, isLocalUrl, isIPAddress, isRelayUrl} from "@welshman/util"
|
||||
import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
|
||||
|
||||
export type RelayStats = {
|
||||
url: string
|
||||
first_seen: number
|
||||
recent_errors: number[]
|
||||
open_count: number
|
||||
close_count: number
|
||||
publish_count: number
|
||||
request_count: number
|
||||
event_count: number
|
||||
last_open: number
|
||||
last_close: number
|
||||
last_error: number
|
||||
last_publish: number
|
||||
last_request: number
|
||||
last_event: number
|
||||
last_auth: number
|
||||
publish_success_count: number
|
||||
publish_failure_count: number
|
||||
eose_count: number
|
||||
notice_count: number
|
||||
}
|
||||
|
||||
export const makeRelayStats = (url: string): RelayStats => ({
|
||||
url,
|
||||
first_seen: now(),
|
||||
recent_errors: [],
|
||||
open_count: 0,
|
||||
close_count: 0,
|
||||
publish_count: 0,
|
||||
request_count: 0,
|
||||
event_count: 0,
|
||||
last_open: 0,
|
||||
last_close: 0,
|
||||
last_error: 0,
|
||||
last_publish: 0,
|
||||
last_request: 0,
|
||||
last_event: 0,
|
||||
last_auth: 0,
|
||||
publish_success_count: 0,
|
||||
publish_failure_count: 0,
|
||||
eose_count: 0,
|
||||
notice_count: 0,
|
||||
})
|
||||
|
||||
export const relayStats = withGetter(writable<RelayStats[]>([]))
|
||||
|
||||
export const relayStatsByUrl = withGetter(
|
||||
derived(relayStats, $relayStats => indexBy(prop("url"), $relayStats)),
|
||||
)
|
||||
|
||||
export const deriveRelayStats = (url: string) =>
|
||||
derived(relayStatsByUrl, $relayStatsByUrl => $relayStatsByUrl.get(url))
|
||||
|
||||
export const getRelayQuality = (url: string) => {
|
||||
// Skip non-relays entirely
|
||||
if (!isRelayUrl(url)) return 0
|
||||
|
||||
const relayStats = relayStatsByUrl.get().get(url)
|
||||
|
||||
// If we have recent errors, skip it
|
||||
if (relayStats) {
|
||||
if (relayStats.recent_errors.filter(n => n > ago(MINUTE)).length > 0) return 0
|
||||
if (relayStats.recent_errors.filter(n => n > ago(HOUR)).length > 3) return 0
|
||||
if (relayStats.recent_errors.filter(n => n > ago(DAY)).length > 10) return 0
|
||||
}
|
||||
|
||||
// Prefer stuff we're connected to
|
||||
if (Pool.get().has(url)) return 1
|
||||
|
||||
// Prefer stuff we've connected to in the past
|
||||
if (relayStats) return 0.9
|
||||
|
||||
// If it's not weird url give it an ok score
|
||||
if (!isIPAddress(url) && !isLocalUrl(url) && !isOnionUrl(url) && !url.startsWith("ws://")) {
|
||||
return 0.8
|
||||
}
|
||||
|
||||
// Default to a "meh" score
|
||||
return 0.7
|
||||
}
|
||||
|
||||
// Utilities for syncing stats from connections to relays
|
||||
|
||||
type RelayStatsUpdate = [string, (stats: RelayStats) => void]
|
||||
|
||||
const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
|
||||
relayStats.update($relayStats => {
|
||||
const $relayStatsByUrl = indexBy(r => r.url, $relayStats)
|
||||
|
||||
for (const [url, items] of groupBy(([url]) => url, updates)) {
|
||||
if (!url || !isRelayUrl(url)) {
|
||||
console.warn(`Attempted to update stats for an invalid relay url: ${url}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const $relayStatsItem: RelayStats = $relayStatsByUrl.get(url) || makeRelayStats(url)
|
||||
|
||||
for (const [_, update] of items) {
|
||||
update($relayStatsItem)
|
||||
}
|
||||
|
||||
// Copy so the database gets updated, since we're mutating in updates
|
||||
$relayStatsByUrl.set(url, {...$relayStatsItem})
|
||||
}
|
||||
|
||||
return Array.from($relayStatsByUrl.values())
|
||||
})
|
||||
})
|
||||
|
||||
const onSocketSend = ([verb]: ClientMessage, url: string) => {
|
||||
if (verb === "REQ") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.request_count++
|
||||
stats.last_request = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EVENT") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.publish_count++
|
||||
stats.last_publish = now()
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
const onSocketReceive = ([verb, ...extra]: RelayMessage, url: string) => {
|
||||
if (verb === "OK") {
|
||||
const [_, ok] = extra
|
||||
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
if (ok) {
|
||||
stats.publish_success_count++
|
||||
} else {
|
||||
stats.publish_failure_count++
|
||||
}
|
||||
},
|
||||
])
|
||||
} else if (verb === "AUTH") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_auth = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EVENT") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.event_count++
|
||||
stats.last_event = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EOSE") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.eose_count++
|
||||
},
|
||||
])
|
||||
} else if (verb === "NOTICE") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.notice_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
const onSocketStatus = (status: string, url: string) => {
|
||||
if (status === SocketStatus.Open) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_open = now()
|
||||
stats.open_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
if (status === SocketStatus.Closed) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_close = now()
|
||||
stats.close_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
if (status === SocketStatus.Error) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_error = now()
|
||||
stats.recent_errors = uniq(stats.recent_errors.concat(now())).slice(-10)
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
export const trackRelayStats = (socket: Socket) => {
|
||||
socket.on(SocketEvent.Send, onSocketSend)
|
||||
socket.on(SocketEvent.Receive, onSocketReceive)
|
||||
socket.on(SocketEvent.Status, onSocketStatus)
|
||||
|
||||
return () => {
|
||||
socket.off(SocketEvent.Send, onSocketSend)
|
||||
socket.off(SocketEvent.Receive, onSocketReceive)
|
||||
socket.off(SocketEvent.Status, onSocketStatus)
|
||||
}
|
||||
}
|
||||
+7
-258
@@ -1,88 +1,12 @@
|
||||
import {writable, derived} from "svelte/store"
|
||||
import {withGetter} from "@welshman/store"
|
||||
import {
|
||||
groupBy,
|
||||
indexBy,
|
||||
batch,
|
||||
now,
|
||||
uniq,
|
||||
batcher,
|
||||
postJson,
|
||||
ago,
|
||||
DAY,
|
||||
HOUR,
|
||||
MINUTE,
|
||||
} from "@welshman/lib"
|
||||
import {uniq, batcher, postJson} from "@welshman/lib"
|
||||
import {RelayProfile} from "@welshman/util"
|
||||
import {
|
||||
normalizeRelayUrl,
|
||||
displayRelayUrl,
|
||||
displayRelayProfile,
|
||||
isOnionUrl,
|
||||
isLocalUrl,
|
||||
isIPAddress,
|
||||
isRelayUrl,
|
||||
} from "@welshman/util"
|
||||
import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net"
|
||||
import {normalizeRelayUrl, displayRelayUrl, displayRelayProfile, isRelayUrl} from "@welshman/util"
|
||||
import {collection} from "@welshman/store"
|
||||
import {appContext} from "./context.js"
|
||||
|
||||
export type RelayStats = {
|
||||
first_seen: number
|
||||
recent_errors: number[]
|
||||
open_count: number
|
||||
close_count: number
|
||||
publish_count: number
|
||||
request_count: number
|
||||
event_count: number
|
||||
last_open: number
|
||||
last_close: number
|
||||
last_error: number
|
||||
last_publish: number
|
||||
last_request: number
|
||||
last_event: number
|
||||
last_auth: number
|
||||
publish_success_count: number
|
||||
publish_failure_count: number
|
||||
eose_count: number
|
||||
notice_count: number
|
||||
}
|
||||
|
||||
export const makeRelayStats = (): RelayStats => ({
|
||||
first_seen: now(),
|
||||
recent_errors: [],
|
||||
open_count: 0,
|
||||
close_count: 0,
|
||||
publish_count: 0,
|
||||
request_count: 0,
|
||||
event_count: 0,
|
||||
last_open: 0,
|
||||
last_close: 0,
|
||||
last_error: 0,
|
||||
last_publish: 0,
|
||||
last_request: 0,
|
||||
last_event: 0,
|
||||
last_auth: 0,
|
||||
publish_success_count: 0,
|
||||
publish_failure_count: 0,
|
||||
eose_count: 0,
|
||||
notice_count: 0,
|
||||
})
|
||||
|
||||
export type Relay = {
|
||||
url: string
|
||||
stats?: RelayStats
|
||||
profile?: RelayProfile
|
||||
}
|
||||
|
||||
export const relays = withGetter(writable<Relay[]>([]))
|
||||
|
||||
export const relaysByPubkey = derived(relays, $relays =>
|
||||
groupBy(
|
||||
$relay => $relay.profile?.pubkey,
|
||||
$relays.filter($relay => $relay.profile?.pubkey),
|
||||
),
|
||||
)
|
||||
export const relays = withGetter(writable<RelayProfile[]>([]))
|
||||
|
||||
export const fetchRelayProfiles = async (urls: string[]) => {
|
||||
const profilesByUrl = new Map<string, RelayProfile>()
|
||||
@@ -122,14 +46,13 @@ export const {
|
||||
} = collection({
|
||||
name: "relays",
|
||||
store: relays,
|
||||
getKey: (relay: Relay) => relay.url,
|
||||
getKey: (relay: RelayProfile) => relay.url,
|
||||
load: batcher(800, async (rawUrls: string[]) => {
|
||||
const urls = rawUrls.map(normalizeRelayUrl)
|
||||
const fresh = await fetchRelayProfiles(uniq(urls))
|
||||
const stale = relaysByUrl.get()
|
||||
|
||||
for (const url of urls) {
|
||||
const relay = stale.get(url)
|
||||
const profile = fresh.get(url)
|
||||
|
||||
if (!url || !isRelayUrl(url)) {
|
||||
@@ -138,7 +61,7 @@ export const {
|
||||
}
|
||||
|
||||
if (profile) {
|
||||
stale.set(url, {...relay, profile, url})
|
||||
stale.set(url, {...profile, url})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,181 +72,7 @@ export const {
|
||||
})
|
||||
|
||||
export const displayRelayByPubkey = (url: string) =>
|
||||
displayRelayProfile(relaysByUrl.get().get(url)?.profile, displayRelayUrl(url))
|
||||
displayRelayProfile(relaysByUrl.get().get(url), displayRelayUrl(url))
|
||||
|
||||
export const deriveRelayDisplay = (url: string) =>
|
||||
derived(deriveRelay(url), $relay => displayRelayProfile($relay?.profile, displayRelayUrl(url)))
|
||||
|
||||
export const getRelayQuality = (url: string) => {
|
||||
const relay = relaysByUrl.get().get(url)
|
||||
|
||||
// Skip non-relays entirely
|
||||
if (!isRelayUrl(url)) return 0
|
||||
|
||||
// If we have recent errors, skip it
|
||||
if (relay?.stats) {
|
||||
if (relay.stats.recent_errors.filter(n => n > ago(MINUTE)).length > 0) return 0
|
||||
if (relay.stats.recent_errors.filter(n => n > ago(HOUR)).length > 3) return 0
|
||||
if (relay.stats.recent_errors.filter(n => n > ago(DAY)).length > 10) return 0
|
||||
}
|
||||
|
||||
// Prefer stuff we're connected to
|
||||
if (Pool.get().has(url)) return 1
|
||||
|
||||
// Prefer stuff we've connected to in the past
|
||||
if (relay?.stats) return 0.9
|
||||
|
||||
// If it's not weird url give it an ok score
|
||||
if (!isIPAddress(url) && !isLocalUrl(url) && !isOnionUrl(url) && !url.startsWith("ws://")) {
|
||||
return 0.8
|
||||
}
|
||||
|
||||
// Default to a "meh" score
|
||||
return 0.7
|
||||
}
|
||||
|
||||
// Utilities for syncing stats from connections to relays
|
||||
|
||||
type RelayStatsUpdate = [string, (stats: RelayStats) => void]
|
||||
|
||||
const updateRelayStats = batch(500, (updates: RelayStatsUpdate[]) => {
|
||||
relays.update($relays => {
|
||||
const $relaysByUrl = indexBy(r => r.url, $relays)
|
||||
const $itemsByUrl = groupBy(([url]) => url, updates)
|
||||
|
||||
for (const [url, items] of $itemsByUrl.entries()) {
|
||||
const $relay: Relay = $relaysByUrl.get(url) || {url}
|
||||
|
||||
if (!url || !isRelayUrl(url)) {
|
||||
console.warn(`Attempted to update stats for an invalid relay url: ${url}`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (!$relay.stats) {
|
||||
$relay.stats = makeRelayStats()
|
||||
} else if ($relay.stats.notice_count === undefined) {
|
||||
// Migrate from old stats
|
||||
$relay.stats = {...makeRelayStats(), ...$relay.stats}
|
||||
}
|
||||
|
||||
for (const [_, update] of items) {
|
||||
update($relay.stats)
|
||||
}
|
||||
|
||||
// Copy so the database gets updated, since we're mutating in updates
|
||||
$relaysByUrl.set(url, {...$relay})
|
||||
}
|
||||
|
||||
return Array.from($relaysByUrl.values())
|
||||
})
|
||||
})
|
||||
|
||||
const onSocketSend = ([verb]: ClientMessage, url: string) => {
|
||||
if (verb === "REQ") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.request_count++
|
||||
stats.last_request = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EVENT") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.publish_count++
|
||||
stats.last_publish = now()
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
const onSocketReceive = ([verb, ...extra]: RelayMessage, url: string) => {
|
||||
if (verb === "OK") {
|
||||
const [_, ok] = extra
|
||||
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
if (ok) {
|
||||
stats.publish_success_count++
|
||||
} else {
|
||||
stats.publish_failure_count++
|
||||
}
|
||||
},
|
||||
])
|
||||
} else if (verb === "AUTH") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_auth = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EVENT") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.event_count++
|
||||
stats.last_event = now()
|
||||
},
|
||||
])
|
||||
} else if (verb === "EOSE") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.eose_count++
|
||||
},
|
||||
])
|
||||
} else if (verb === "NOTICE") {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.notice_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
const onSocketStatus = (status: string, url: string) => {
|
||||
if (status === SocketStatus.Open) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_open = now()
|
||||
stats.open_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
if (status === SocketStatus.Closed) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_close = now()
|
||||
stats.close_count++
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
if (status === SocketStatus.Error) {
|
||||
updateRelayStats([
|
||||
url,
|
||||
stats => {
|
||||
stats.last_error = now()
|
||||
stats.recent_errors = uniq(stats.recent_errors.concat(now())).slice(-10)
|
||||
},
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
export const trackRelayStats = (socket: Socket) => {
|
||||
socket.on(SocketEvent.Send, onSocketSend)
|
||||
socket.on(SocketEvent.Receive, onSocketReceive)
|
||||
socket.on(SocketEvent.Status, onSocketStatus)
|
||||
|
||||
return () => {
|
||||
socket.off(SocketEvent.Send, onSocketSend)
|
||||
socket.off(SocketEvent.Receive, onSocketReceive)
|
||||
socket.off(SocketEvent.Status, onSocketStatus)
|
||||
}
|
||||
}
|
||||
derived(deriveRelay(url), $relay => displayRelayProfile($relay, displayRelayUrl(url)))
|
||||
|
||||
@@ -2,14 +2,14 @@ import Fuse, {IFuseOptions, FuseResult} from "fuse.js"
|
||||
import {debounce} from "throttle-debounce"
|
||||
import {derived} from "svelte/store"
|
||||
import {dec, inc, sortBy} from "@welshman/lib"
|
||||
import {PROFILE, PublishedProfile} from "@welshman/util"
|
||||
import {PROFILE, PublishedProfile, RelayProfile} from "@welshman/util"
|
||||
import {load} from "@welshman/net"
|
||||
import {throttled} from "@welshman/store"
|
||||
import {Router} from "@welshman/router"
|
||||
import {wotGraph, maxWot} from "./wot.js"
|
||||
import {profiles} from "./profiles.js"
|
||||
import {topics, Topic} from "./topics.js"
|
||||
import {relays, Relay} from "./relays.js"
|
||||
import {relays} from "./relays.js"
|
||||
import {handlesByNip05} from "./handles.js"
|
||||
|
||||
export type SearchOptions<V, T> = {
|
||||
@@ -102,7 +102,7 @@ export const topicSearch = derived(topics, $topics =>
|
||||
|
||||
export const relaySearch = derived(relays, $relays =>
|
||||
createSearch($relays, {
|
||||
getValue: (relay: Relay) => relay.url,
|
||||
getValue: (relay: RelayProfile) => relay.url,
|
||||
fuseOptions: {
|
||||
keys: ["url", "name", {name: "description", weight: 0.3}],
|
||||
},
|
||||
|
||||
@@ -8,11 +8,11 @@ const query = (filters: Filter[]) =>
|
||||
repository.query(filters, {shouldSort: filters.every(f => f.limit === undefined)})
|
||||
|
||||
export const hasNegentropy = (url: string) => {
|
||||
const p = relaysByUrl.get().get(url)?.profile
|
||||
const relay = relaysByUrl.get().get(url)
|
||||
|
||||
if (p?.negentropy) return true
|
||||
if (p?.supported_nips?.includes?.(77)) return true
|
||||
if (p?.software?.includes?.("strfry") && !p?.version?.match(/^0\./)) return true
|
||||
if (relay?.negentropy) return true
|
||||
if (relay?.supported_nips?.includes?.(77)) return true
|
||||
if (relay?.software?.includes?.("strfry") && !relay?.version?.match(/^0\./)) return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user