From eb1a219ca57c783101669466a726d63d1ad61d47 Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Wed, 4 Sep 2024 16:16:14 -0700 Subject: [PATCH] Speed up loading, improve relay selection from filters --- packages/app/src/follows.ts | 4 +- packages/app/src/mutes.ts | 4 +- packages/app/src/profiles.ts | 4 +- packages/app/src/relaySelections.ts | 6 +- packages/app/src/router.ts | 144 +++++++++++++--------------- 5 files changed, 78 insertions(+), 84 deletions(-) diff --git a/packages/app/src/follows.ts b/packages/app/src/follows.ts index 15028f8..1664cac 100644 --- a/packages/app/src/follows.ts +++ b/packages/app/src/follows.ts @@ -2,7 +2,7 @@ import {FOLLOWS, asDecryptedEvent, readList} from '@welshman/util' import {type TrustedEvent, type PublishedList} from '@welshman/util' import {type SubscribeRequest} from "@welshman/net" import {deriveEventsMapped, withGetter} from '@welshman/store' -import {repository, load} from './core' +import {repository, loadOne} from './core' import {collection} from './collection' import {ensurePlaintext} from './plaintext' @@ -28,5 +28,5 @@ export const { store: follows, getKey: follows => follows.event.pubkey, load: (pubkey: string, request: Partial = {}) => - load({...request, filters: [{kinds: [FOLLOWS], authors: [pubkey]}]}), + loadOne({...request, filters: [{kinds: [FOLLOWS], authors: [pubkey]}]}), }) diff --git a/packages/app/src/mutes.ts b/packages/app/src/mutes.ts index 61e353b..61b3767 100644 --- a/packages/app/src/mutes.ts +++ b/packages/app/src/mutes.ts @@ -2,7 +2,7 @@ import {MUTES, asDecryptedEvent, readList} from '@welshman/util' import {type TrustedEvent, type PublishedList} from '@welshman/util' import {type SubscribeRequest} from "@welshman/net" import {deriveEventsMapped, withGetter} from '@welshman/store' -import {repository, load} from './core' +import {repository, loadOne} from './core' import {collection} from './collection' import {ensurePlaintext} from './plaintext' @@ -28,5 +28,5 @@ export const { store: mutes, getKey: mute => mute.event.pubkey, load: (pubkey: string, request: Partial = {}) => - load({...request, filters: [{kinds: [MUTES], authors: [pubkey]}]}), + loadOne({...request, filters: [{kinds: [MUTES], authors: [pubkey]}]}), }) diff --git a/packages/app/src/profiles.ts b/packages/app/src/profiles.ts index 6204b9b..eced5bd 100644 --- a/packages/app/src/profiles.ts +++ b/packages/app/src/profiles.ts @@ -3,7 +3,7 @@ import {readProfile, displayProfile, displayPubkey, PROFILE} from '@welshman/uti import {type SubscribeRequest} from "@welshman/net" import {type PublishedProfile} from "@welshman/util" import {deriveEventsMapped, withGetter} from '@welshman/store' -import {repository, load} from './core' +import {repository, loadOne} from './core' import {createSearch} from './util' import {collection} from './collection' @@ -24,7 +24,7 @@ export const { store: profiles, getKey: profile => profile.event.pubkey, load: (pubkey: string, request: Partial = {}) => - load({...request, filters: [{kinds: [PROFILE], authors: [pubkey]}]}), + loadOne({...request, filters: [{kinds: [PROFILE], authors: [pubkey]}]}), }) export const profileSearch = derived(profiles, $profiles => diff --git a/packages/app/src/relaySelections.ts b/packages/app/src/relaySelections.ts index f49e451..9ebc412 100644 --- a/packages/app/src/relaySelections.ts +++ b/packages/app/src/relaySelections.ts @@ -1,7 +1,7 @@ import {INBOX_RELAYS, RELAYS, getRelayTags, normalizeRelayUrl, type TrustedEvent} from '@welshman/util' import {type SubscribeRequest} from "@welshman/net" import {deriveEvents, withGetter} from '@welshman/store' -import {load, repository} from './core' +import {loadOne, repository} from './core' import {collection} from './collection' export const getRelayUrls = (event?: TrustedEvent): string[] => @@ -29,7 +29,7 @@ export const { store: relaySelections, getKey: relaySelections => relaySelections.pubkey, load: (pubkey: string, request: Partial = {}) => - load({...request, filters: [{kinds: [RELAYS], authors: [pubkey]}]}), + loadOne({...request, filters: [{kinds: [RELAYS], authors: [pubkey]}]}), }) export const inboxRelaySelections = withGetter(deriveEvents(repository, {filters: [{kinds: [RELAYS]}]})) @@ -43,5 +43,5 @@ export const { store: inboxRelaySelections, getKey: inboxRelaySelections => inboxRelaySelections.pubkey, load: (pubkey: string, request: Partial = {}) => - load({...request, filters: [{kinds: [INBOX_RELAYS], authors: [pubkey]}]}), + loadOne({...request, filters: [{kinds: [INBOX_RELAYS], authors: [pubkey]}]}), }) diff --git a/packages/app/src/router.ts b/packages/app/src/router.ts index 39098bf..b84e846 100644 --- a/packages/app/src/router.ts +++ b/packages/app/src/router.ts @@ -1,6 +1,6 @@ import { intersection, first, switcher, throttleWithValue, clamp, last, splitAt, identity, sortBy, uniq, shuffle, - pushToMapKey, now, + pushToMapKey, now, assoc, } from '@welshman/lib' import { Tags, getFilterId, unionFilters, isShareableRelayUrl, isCommunityAddress, isGroupAddress, isContextAddress, @@ -303,8 +303,11 @@ export class RouterScenario { clone = (options: RouterScenarioOptions) => new RouterScenario(this.router, this.selections, {...this.options, ...options}) - select = (f: (selection: string) => boolean) => - new RouterScenario(this.router, this.selections.filter(({value}) => f(value)), this.options) + filter = (f: (selection: ValueRelays) => boolean) => + new RouterScenario(this.router, this.selections.filter(selection => f(selection)), this.options) + + update = (f: (selection: ValueRelays) => ValueRelays) => + new RouterScenario(this.router, this.selections.map(selection => f(selection)), this.options) redundancy = (redundancy: number) => this.clone({redundancy}) @@ -475,120 +478,111 @@ export type FilterSelection = { scenario: RouterScenario } +type FilterSelectionRuleState = { + filter: Filter, + selections: FilterSelection[] +} + +type FilterSelectionRule = (state: FilterSelectionRuleState) => boolean + export const makeFilterSelection = (id: string, filter: Filter, scenario: RouterScenario) => ({id, filter, scenario}) -export const getFilterSelectionsForSearch = (filter: Filter) => { - const id = getFilterId(filter) +export const getFilterSelectionsForSearch = (state: FilterSelectionRuleState) => { + if (!state.filter.search) return false + + const id = getFilterId(state.filter) const relays = AppContext.router.options.getSearchRelays?.() || [] const scenario = AppContext.router.product([id], relays) - return [makeFilterSelection(id, filter, scenario)] + state.selections.push(makeFilterSelection(id, state.filter, scenario)) + + return true } -export const getFilterSelectionsForIndexedKinds = (filter: Filter) => { - const kinds = intersection(INDEXED_KINDS, filter.kinds!) - const id = getFilterId({...filter, kinds}) - const relays = AppContext.router.options.getIndexerRelays?.() || [] - const scenario = AppContext.router.product([id], relays) +export const getFilterSelectionsForContext = (state: FilterSelectionRuleState) => { + const contexts = state.filter["#a"]?.filter(isContextAddress) || [] - return [makeFilterSelection(id, filter, scenario)] -} + if (contexts.length === 0) return false -export const getFilterSelectionsForContext = (filter: Filter) => { - const filterSelections = [] - const contexts = filter["#a"].filter(isContextAddress) const scenario = AppContext.router.WithinMultipleContexts(contexts) for (const {relay, values} of scenario.getSelections()) { - const contextFilter = {...filter, "#a": Array.from(values)} + const contextFilter = {...state.filter, "#a": Array.from(values)} const id = getFilterId(contextFilter) const scenario = AppContext.router.product([id], [relay]) - filterSelections.push( - makeFilterSelection(id, contextFilter, scenario) - ) + state.selections.push(makeFilterSelection(id, contextFilter, scenario)) } - return filterSelections + return true } -export const getFilterSelectionsForAuthors = (filter: Filter) => { - const filterSelections = [] - const scenario = AppContext.router.FromPubkeys(filter.authors!) +export const getFilterSelectionsForIndexedKinds = (state: FilterSelectionRuleState) => { + const kinds = intersection(INDEXED_KINDS, state.filter.kinds || []) - for (const {relay, values} of scenario.getSelections()) { - const authorsFilter = {...filter, authors: Array.from(values)} - const id = getFilterId(authorsFilter) + if (kinds.length === 0) return false - filterSelections.push( - makeFilterSelection(id, authorsFilter, AppContext.router.product([id], [relay])) - ) - } + const id = getFilterId({...state.filter, kinds}) + const relays = AppContext.router.options.getIndexerRelays?.() || [] + const scenario = AppContext.router.product([id], relays) - return filterSelections + state.selections.push(makeFilterSelection(id, state.filter, scenario)) + + return false } -export const getFilterSelectionsForMentions = (filter: Filter) => { - const filterSelections = [] - const scenario = AppContext.router.ForPubkeys(filter['#p']!) +export const getFilterSelectionsForAuthors = (state: FilterSelectionRuleState) => { + if (!state.filter.authors) return false - for (const {relay, values} of scenario.getSelections()) { - const mentionsFilter = {...filter, '#p': Array.from(values)} - const id = getFilterId(mentionsFilter) + const id = getFilterId(state.filter) + const scenario = AppContext.router.FromPubkeys(state.filter.authors!).update(assoc('value', id)) - filterSelections.push( - makeFilterSelection(id, mentionsFilter, AppContext.router.product([id], [relay])) - ) - } + state.selections.push(makeFilterSelection(id, state.filter, scenario)) - return filterSelections + return false } -export const getFilterSelectionsForUser = (filter: Filter) => { - const id = getFilterId(filter) - const scenario = AppContext.router.ReadRelays() +export const getFilterSelectionsForUser = (state: FilterSelectionRuleState) => { + const id = getFilterId(state.filter) + const relays = AppContext.router.ReadRelays().getUrls() + const scenario = AppContext.router.product([id], relays) - return [makeFilterSelection(id, filter, AppContext.router.product([id], scenario.getUrls()))] + state.selections.push(makeFilterSelection(id, state.filter, scenario)) + + return false } -export const getFilterSelections = (filters: Filter[]): RelayFilters[] => { +export const defaultFilterSelectionRules = [ + getFilterSelectionsForSearch, + getFilterSelectionsForContext, + getFilterSelectionsForIndexedKinds, + getFilterSelectionsForAuthors, + getFilterSelectionsForUser, +] + +export const getFilterSelections = (filters: Filter[], rules: FilterSelectionRule[] = defaultFilterSelectionRules): RelayFilters[] => { const scenarios: RouterScenario[] = [] const filtersById = new Map() - const addSelections = (selections: FilterSelection[]) => { - for (const {id, filter, scenario} of selections) { - filtersById.set(id, filter) - scenarios.push(scenario) - } - } - for (const filter of filters) { - if (filter.search) { - addSelections(getFilterSelectionsForSearch(filter)) + const state: FilterSelectionRuleState = {filter, selections: []} + + for (const rule of rules) { + const done = rule(state) + + if (done) { + break + } } - if (filter.kinds?.some(k => INDEXED_KINDS.includes(k))) { - addSelections(getFilterSelectionsForIndexedKinds(filter)) - } - - if (filter["#a"]?.some(isContextAddress)) { - addSelections(getFilterSelectionsForContext(filter)) - } - - if (filter.authors) { - addSelections(getFilterSelectionsForAuthors(filter)) - } - - if (filter['#p']) { - addSelections(getFilterSelectionsForMentions(filter)) - } - - if (scenarios.length === 0) { - addSelections(getFilterSelectionsForUser(filter)) + for (const {id, filter, scenario} of state.selections) { + filtersById.set(id, filter) + scenarios.push(scenario.policy(addNoFallbacks)) } } + // Use low redundancy because filters will be very low cardinality const selections = AppContext.router .merge(scenarios)