coracle/welshman
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continue re-working router to not use values, but include weight

Browse Source
This commit is contained in:
Jon Staab
2024-11-04 09:39:25 -08:00
parent 75aec594e2
commit ea86d1dbf0
4 changed files with 107 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/feeds.ts
+1 -1
View File
@@ -14,7 +14,7 @@ export const feedLoader = new FeedLoader({
await load({onEvent, filters, relays}) await load({onEvent, filters, relays})
} else { } else {
await Promise.all( await Promise.all(
getFilterSelections(filters) Array.from(getFilterSelections(filters))
.map(opts => load({onEvent, ...opts})) .map(opts => load({onEvent, ...opts}))
) )
} }
packages/app/src/router.ts
+95 -239
View File
@@ -90,12 +90,6 @@ export type RouterOptions = {
*/ */
getRelayQuality?: (url: string) => number getRelayQuality?: (url: string) => number
/**
* Retrieves the redundancy setting, which is how many relays to use per selection value.
* @returns The redundancy setting as a number.
*/
getRedundancy?: () => number
/** /**
* Retrieves the limit setting, which is the maximum number of relays that should be * Retrieves the limit setting, which is the maximum number of relays that should be
* returned from getUrls and getSelections. * returned from getUrls and getSelections.
@@ -104,28 +98,22 @@ export type RouterOptions = {
getLimit?: () => number getLimit?: () => number
} }
export type ValuesByRelay = Map<string, string[]> export type Selection = {
weight: number,
export type RelayValues = { relays: string[],
relay: string
values: string[]
} }
export type ValueRelays = { const makeSelection = (relays: string[], weight = 1): Selection => ({relays, weight})
value: string
relays: string[]
weight: number
}
// Fallback policies // Fallback policies
export type FallbackPolicy = (count: number, limit: number) => number export type FallbackPolicy = (count: number, limit: number) => number
export const addNoFallbacks = (count: number, redundancy: number) => 0 export const addNoFallbacks = (count: number, limit: number) => 0
export const addMinimalFallbacks = (count: number, redundancy: number) => (count > 0 ? 0 : 1) export const addMinimalFallbacks = (count: number, limit: number) => count > 0 ? 0 : 1
export const addMaximalFallbacks = (count: number, redundancy: number) => redundancy - count export const addMaximalFallbacks = (count: number, limit: number) => limit - count
export class Router { export class Router {
constructor(readonly options: RouterOptions) {} constructor(readonly options: RouterOptions) {}
@@ -144,48 +132,17 @@ export class Router {
return pubkey ? this.getRelaysForPubkey(pubkey) : [] return pubkey ? this.getRelaysForPubkey(pubkey) : []
} }
// Utilities for creating ValueRelays
selection = (value: string, relays: Iterable<string>, weight = 1): ValueRelays =>
({value, relays: Array.from(relays), weight})
// Utilities for processing hints
relaySelectionsFromMap = (valuesByRelay: ValuesByRelay) =>
sortBy(
({values}) => -values.length,
Array.from(valuesByRelay).map(([relay, values]: [string, string[]]) => ({
relay,
values: uniq(values),
}))
)
scoreRelaySelection = ({values, relay, weight}: RelayValues) =>
values.length * (this.options.getRelayQuality?.(relay) || 1) * weight
sortRelaySelections = (relaySelections: RelayValues[]) => {
const scores = new Map<string, number>()
const getScore = (relayValues: RelayValues) => -(scores.get(relayValues.relay) || 0)
for (const relayValues of relaySelections) {
scores.set(relayValues.relay, this.scoreRelaySelection(relayValues))
}
return sortBy(getScore, relaySelections.filter(getScore))
}
// Utilities for creating scenarios // Utilities for creating scenarios
scenario = (selections: ValueRelays[]) => new RouterScenario(this, selections) scenario = (selections: Selection[]) => new RouterScenario(this, selections)
merge = (scenarios: RouterScenario[]) => merge = (scenarios: RouterScenario[]) =>
this.scenario(scenarios.flatMap((scenario: RouterScenario) => scenario.selections)) this.scenario(scenarios.flatMap((scenario: RouterScenario) => scenario.selections))
// Routing scenarios // Routing scenarios
FromRelays = (relays: string[]) =>
FromRelays = (relays: string[], id = "") => this.scenario([makeSelection(relays)])
this.scenario([this.selection(id, relays)])
ForPubkey = (pubkey: string) => ForPubkey = (pubkey: string) =>
this.FromRelays(this.getRelaysForPubkey(pubkey, RelayMode.Read)) this.FromRelays(this.getRelaysForPubkey(pubkey, RelayMode.Read))
@@ -215,26 +172,26 @@ export class Router {
this.merge(pubkeys.map(pubkey => this.PubkeyInbox(pubkey))) this.merge(pubkeys.map(pubkey => this.PubkeyInbox(pubkey)))
Event = (event: TrustedEvent) => Event = (event: TrustedEvent) =>
this.FromRelays(this.getRelaysForPubkey(event.pubkey, RelayMode.Write), event.id) this.FromRelays(this.getRelaysForPubkey(event.pubkey, RelayMode.Write))
EventChildren = (event: TrustedEvent) => EventChildren = (event: TrustedEvent) =>
this.FromRelays(this.getRelaysForPubkey(event.pubkey, RelayMode.Read), event.id) this.FromRelays(this.getRelaysForPubkey(event.pubkey, RelayMode.Read))
EventAncestors = (event: TrustedEvent, type: "mentions" | "replies" | "roots") => { EventAncestors = (event: TrustedEvent, type: "mentions" | "replies" | "roots") => {
return this.scenario( return this.scenario(
getAncestorTags(event.tags)[type].flatMap( getAncestorTags(event.tags)[type].flatMap(
([_, value, relay, pubkey]) => { ([_, value, relay, pubkey]) => {
const tagScenarios = [this.selection(value, this.ForUser().getUrls(), 0.5)] const selections = [makeSelection(this.ForUser().getUrls(), 0.5)]
if (pubkey) { if (pubkey) {
tagScenarios.push(this.selection(value, this.FromPubkey(pubkey).getUrls())) selections.push(makeSelection(this.FromPubkey(pubkey).getUrls()))
} }
if (relay) { if (relay) {
tagScenarios.push(this.selection(value, [relay], 0.9)) selections.push(makeSelection([relay], 0.9))
} }
return tagScenarios return selections
} }
) )
) )
@@ -249,9 +206,9 @@ export class Router {
PublishEvent = (event: TrustedEvent) => { PublishEvent = (event: TrustedEvent) => {
const pubkeys = getPubkeyTagValues(event.tags) const pubkeys = getPubkeyTagValues(event.tags)
return this.scenario([ return this.merge([
this.selection(event.id, this.FromPubkey(event.pubkey).getUrls()), this.FromPubkey(event.pubkey),
...pubkeys.map(pubkey => this.selection(event.id, this.ForPubkey(event.pubkey).getUrls(), 0.5)), ...pubkeys.map(pubkey => this.ForPubkey(pubkey).weight(0.5)),
]) ])
} }
} }
@@ -259,114 +216,75 @@ export class Router {
// Router Scenario // Router Scenario
export type RouterScenarioOptions = { export type RouterScenarioOptions = {
redundancy?: number
policy?: FallbackPolicy policy?: FallbackPolicy
limit?: number limit?: number
} }
export class RouterScenario { export class RouterScenario {
constructor( constructor(readonly router: Router, readonly selections: Selection[], readonly options: RouterScenarioOptions = {}) {}
readonly router: Router,
readonly selections: ValueRelays[],
readonly options: RouterScenarioOptions = {}
) {}
clone = (options: RouterScenarioOptions) => clone = (options: RouterScenarioOptions) =>
new RouterScenario(this.router, this.selections, {...this.options, ...options}) new RouterScenario(this.router, this.selections, {...this.options, ...options})
filter = (f: (selection: ValueRelays) => boolean) => filter = (f: (selection: Selection) => boolean) =>
new RouterScenario( new RouterScenario(this.router, this.selections.filter(selection => f(selection)), this.options)
this.router,
this.selections.filter(selection => f(selection)),
this.options
)
update = (f: (selection: ValueRelays) => ValueRelays) => update = (f: (selection: Selection) => Selection) =>
new RouterScenario( new RouterScenario(this.router, this.selections.map(selection => f(selection)), this.options)
this.router,
this.selections.map(selection => f(selection)),
this.options
)
redundancy = (redundancy: number) => this.clone({redundancy})
policy = (policy: FallbackPolicy) => this.clone({policy}) policy = (policy: FallbackPolicy) => this.clone({policy})
limit = (limit: number) => this.clone({limit}) limit = (limit: number) => this.clone({limit})
getRedundancy = () => this.options.redundancy || this.router.options.getRedundancy?.() || 3 weight = (scale: number) =>
this.update(selection => ({...selection, weight: selection.weight * scale}))
getPolicy = () => this.options.policy || addMaximalFallbacks getPolicy = () => this.options.policy || addMaximalFallbacks
getLimit = () => this.options.limit || this.router.options.getLimit?.() || 10 getLimit = () => this.options.limit || this.router.options.getLimit?.() || 10
getSelections = () => { getUrls = () => {
const allValues = new Set() const limit = this.getLimit()
const valuesByRelay: ValuesByRelay = new Map() const fallbackPolicy = this.getPolicy()
for (const {value, relays} of this.selections) { const relayWeights = new Map<string, number>()
allValues.add(value)
for (const {weight, relays} of this.selections) {
for (const relay of relays) { for (const relay of relays) {
if (isShareableRelayUrl(relay)) { if (!isShareableRelayUrl(relay)) {
pushToMapKey(valuesByRelay, relay, value) continue
} }
relayWeights.set(relay, add(weight, relayWeights.get(relay)))
} }
} }
// Adjust redundancy by limit, since if we're looking for very specific values odds const scoreRelay = (relay: string) => {
// are we're less tolerant of failure. Add more redundancy to fill our relay limit. const quality = this.router.options.getRelayQuality?.(relay) || 1
const limit = this.getLimit() const weight = relayWeights.get(relay)!
const redundancy = this.getRedundancy()
const adjustedRedundancy = Math.max( return -(quality * weight)
redundancy, }
redundancy * (limit / (allValues.size * redundancy))
const relays = take(
limit,
sortBy(
scoreRelay,
Array.from(relayWeights.keys())
.filter(scoreRelay)
)
) )
const seen = new Map<string, number>() const fallbacksNeeded = fallbackPolicy(relays.length, limit)
const result: ValuesByRelay = new Map() const allFallbackRelays = this.router.options.getFallbackRelays()
const relaySelections = this.router.relaySelectionsFromMap(valuesByRelay) const fallbackRelays = shuffle(allFallbackRelays).slice(0, fallbacksNeeded)
for (const {relay} of this.router.sortRelaySelections(relaySelections)) {
const values = new Set<string>()
for (const value of uniq(valuesByRelay.get(relay) || [])) {
const timesSeen = seen.get(value) || 0
if (timesSeen < adjustedRedundancy) { for (const fallbackRelay of fallbackRelays) {
seen.set(value, timesSeen + 1) relays.push(fallbackRelay)
values.add(value)
}
}
if (values.size > 0) {
result.set(relay, Array.from(values))
}
} }
const fallbacks = shuffle(this.router.options.getFallbackRelays()) return relays
const fallbackPolicy = this.getPolicy()
for (const {value} of this.selections) {
const timesSeen = seen.get(value) || 0
const fallbacksNeeded = fallbackPolicy(timesSeen, adjustedRedundancy)
if (fallbacksNeeded > 0) {
for (const relay of fallbacks.slice(0, fallbacksNeeded)) {
pushToMapKey(result, relay, value)
}
}
}
const [keep, discard] = splitAt(limit, this.router.relaySelectionsFromMap(result))
for (const target of keep.slice(0, redundancy)) {
target.values = uniq(
discard.concat(target).flatMap((selection: RelayValues) => selection.values)
)
}
return keep
} }
getUrls = () => this.getSelections().map((selection: RelayValues) => selection.relay)
getUrl = () => first(this.getUrls()) getUrl = () => first(this.getUrls())
} }
@@ -459,100 +377,63 @@ export const makeRouter = (options: Partial<RouterOptions> = {}) =>
getSearchRelays, getSearchRelays,
getRelayQuality, getRelayQuality,
getUserPubkey: () => pubkey.get(), getUserPubkey: () => pubkey.get(),
getRedundancy: () => 2,
getLimit: () => 5, getLimit: () => 5,
...options, ...options,
}) })
// Infer relay selections from filters // Infer relay selections from filters
export type FilterSelection = { type FilterScenario = {filter: Filter, scenario: RouterScenario}
id: string
filter: Filter
scenario: RouterScenario
}
type FilterSelectionRuleState = { type FilterSelectionRule = (filter: Filter) => FilterScenario[]
filter: Filter
selections: FilterSelection[]
}
type FilterSelectionRule = (state: FilterSelectionRuleState) => boolean export const getFilterSelectionsForLocalRelay = (filter: Filter) =>
[{filter, scenario: ctx.app.router.FromRelays([LOCAL_RELAY_URL])}]
export const makeFilterSelection = (id: string, filter: Filter, scenario: RouterScenario) => ({ export const getFilterSelectionsForSearch = (filter: Filter) => {
id, if (!filter.search) return []
filter,
scenario,
})
export const getFilterSelectionsForLocalRelay = (state: FilterSelectionRuleState) => {
const id = getFilterId(state.filter)
const scenario = ctx.app.router.FromRelays([LOCAL_RELAY_URL], id)
state.selections.push(makeFilterSelection(id, state.filter, scenario))
return false
}
export const getFilterSelectionsForSearch = (state: FilterSelectionRuleState) => {
if (!state.filter.search) return false
const id = getFilterId(state.filter)
const relays = ctx.app.router.options.getSearchRelays?.() || [] const relays = ctx.app.router.options.getSearchRelays?.() || []
const scenario = ctx.app.router.FromRelays(relays, id)
state.selections.push(makeFilterSelection(id, state.filter, scenario)) return [{filter, scenario: ctx.app.router.FromRelays(relays).weight(10)}]
return true
} }
export const getFilterSelectionsForWraps = (state: FilterSelectionRuleState) => { export const getFilterSelectionsForWraps = (filter: Filter) => {
if (!state.filter.kinds?.includes(WRAP) || state.filter.authors) return false if (!filter.kinds?.includes(WRAP) || filter.authors) return []
const id = getFilterId({...state.filter, kinds: [WRAP]}) return [{
const scenario = ctx.app.router.UserInbox().update(assoc('value', id)) filter: {...filter, kinds: [WRAP]},
scenario: ctx.app.router.UserInbox(),
state.selections.push(makeFilterSelection(id, state.filter, scenario)) }]
return false
} }
export const getFilterSelectionsForIndexedKinds = (state: FilterSelectionRuleState) => { export const getFilterSelectionsForIndexedKinds = (filter: Filter) => {
const kinds = intersection(INDEXED_KINDS, state.filter.kinds || []) const kinds = intersection(INDEXED_KINDS, filter.kinds || [])
if (kinds.length === 0) return false if (kinds.length === 0) return []
const id = getFilterId({...state.filter, kinds})
const relays = ctx.app.router.options.getIndexerRelays?.() || [] const relays = ctx.app.router.options.getIndexerRelays?.() || []
const scenario = ctx.app.router.FromRelays(relays, id)
state.selections.push(makeFilterSelection(id, state.filter, scenario)) return [{
filter: {...filter, kinds},
return false scenario: ctx.app.router.FromRelays(relays),
}]
} }
export const getFilterSelectionsForAuthors = (state: FilterSelectionRuleState) => { export const getFilterSelectionsForAuthors = (filter: Filter) => {
// If we have a ton of authors, just use our indexers if (!filter.authors) return []
if (!state.filter.authors) return false
const id = getFilterId(state.filter) const chunkCount = clamp([1, 4], Math.round(filter.authors.length / 50))
const pubkeys = sample(50, state.filter.authors!)
const scenario = ctx.app.router.FromPubkeys(pubkeys).update(assoc("value", id))
state.selections.push(makeFilterSelection(id, state.filter, scenario)) return chunks(chunkCount, filter.authors)
.map(authors => ({
return false filter: {...filter, authors},
scenario: ctx.app.router.FromPubkeys(authors),
}))
} }
export const getFilterSelectionsForUser = (state: FilterSelectionRuleState) => { export const getFilterSelectionsForUser = (filter: Filter) =>
const id = getFilterId(state.filter) [{filter, scenario: ctx.app.router.ForUser().weight(0.5)}]
const relays = ctx.app.router.ForUser().getUrls()
const scenario = ctx.app.router.FromRelays(relays, id)
state.selections.push(makeFilterSelection(id, state.filter, scenario))
return false
}
export const defaultFilterSelectionRules = [ export const defaultFilterSelectionRules = [
getFilterSelectionsForLocalRelay, getFilterSelectionsForLocalRelay,
@@ -563,48 +444,23 @@ export const defaultFilterSelectionRules = [
getFilterSelectionsForUser, getFilterSelectionsForUser,
] ]
export const getFilterSelections = ( export function* getFilterSelections(filters: Filter[], rules: FilterSelectionRule[] = defaultFilterSelectionRules) {
filters: Filter[],
rules: FilterSelectionRule[] = defaultFilterSelectionRules
): RelaysAndFilters[] => {
const scenarios: RouterScenario[] = []
const filtersById = new Map<string, Filter>() const filtersById = new Map<string, Filter>()
const scenariosById = new Map<string, RouterScenario[]>()
for (const filter of filters) { for (const filter of filters) {
const state: FilterSelectionRuleState = {filter, selections: []} for (const filterScenario of rules.flatMap(rule => rule(filter))) {
const id = getFilterId(filterScenario.filter)
for (const rule of rules) { filtersById.set(id, filterScenario.filter)
const done = rule(state) pushToMapKey(scenariosById, id, filterScenario.scenario)
if (done) {
break
}
}
for (const {id, filter, scenario} of state.selections) {
filtersById.set(id, filter)
scenarios.push(scenario.policy(addNoFallbacks))
} }
} }
// Use low redundancy because filters will be very low cardinality for (const [id, filter] of filtersById.entries()) {
const selections = ctx.app.router const scenario = ctx.app.router.merge(scenariosById.get(id) || [])
.merge(scenarios) const result = {filters: [filter], relays: scenario.getUrls()} as RelaysAndFilters
.redundancy(1)
.getSelections()
.map(({values, relay}) => ({
filters: values.map(id => filtersById.get(id)!),
relays: [relay],
}))
// Pubkey-based selections can get really big. Use the most popular relays for the long tail yield result
const limit = ctx.app.router.options.getLimit?.() || 8
const redundancy = ctx.app.router.options.getRedundancy?.() || 3
const [keep, discard] = splitAt(limit, selections)
for (const target of keep.slice(0, redundancy)) {
target.filters = unionFilters([...discard, target].flatMap(s => s.filters))
} }
return keep
} }
packages/app/tsc-multi.json
+1
View File
@@ -1,5 +1,6 @@
{ {
"targets": [ "targets": [
{"extname": ".cjs", "module": "commonjs"},
{"extname": ".mjs", "module": "esnext", "moduleResolution": "node"} {"extname": ".mjs", "module": "esnext", "moduleResolution": "node"}
], ],
"projects": ["tsconfig.json"] "projects": ["tsconfig.json"]
packages/lib/src/Tools.ts
+10 -2
View File
@@ -23,9 +23,17 @@ export const identity = <T>(x: T, ...args: unknown[]) => x
export const always = <T>(x: T, ...args: unknown[]) => () => x export const always = <T>(x: T, ...args: unknown[]) => () => x
export const inc = (x: number | Nil) => (x || 0) + 1 export const add = (x: number | Nil, y: number | Nil) => (x || 0) + (y || 0)
export const dec = (x: number | Nil) => (x || 0) - 1 export const sub = (x: number | Nil, y: number | Nil) => (x || 0) - (y || 0)
export const mul = (x: number | Nil, y: number | Nil) => (x || 0) * (y || 0)
export const div = (x: number | Nil, y: number) => (x || 0) / y
export const inc = (x: number | Nil) => add(x, 1)
export const dec = (x: number | Nil) => sub(x, 1)
export const lt = (x: number | Nil, y: number | Nil) => (x || 0) < (y || 0) export const lt = (x: number | Nil, y: number | Nil) => (x || 0) < (y || 0)