coracle/welshman
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to compiler strategy for custom feeds

Browse Source
This commit is contained in:
Jon Staab
2024-04-12 11:25:44 -07:00
parent d3604ed9f2
commit 9530ab4cf4
1 changed files with 57 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/custom-feeds/index.ts
+57 -156
View File
@@ -2,17 +2,8 @@ import {inc, now, isNil} from '@coracle.social/lib'
import type {Rumor, Filter} from '@coracle.social/util'
import {Tags, getIdFilters, mergeFilters} from '@coracle.social/util'
// TODO:
// - if one of the feeds in a union is a filter, don't execute it,
// use it to filter down results from other feeds
// - if multiple feeds in a composite are or result in filters (like lists, lols),
// merge them before executing the request
export enum FeedType {
Difference = "\\",
Intersection = "∩",
SymmetricDifference = "Δ",
Union = "∪",
Union = "union",
Filter = "filter",
List = "list",
LOL = "lol",
@@ -20,220 +11,122 @@ export enum FeedType {
}
export enum Scope {
User = "user",
Self = "self",
Follows = "follows",
Followers = "followers",
}
export type WotRange = 0 | 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 | 0.7 | 0.8 | 0.9 | 1
export type DynamicFilter = Filter & {
scopes?: Scope[]
min_wot?: WotRange
max_wot?: WotRange
min_wot?: number
max_wot?: number
until_ago?: number
since_ago?: number
}
export type DVMRequest = {
kind: number
input?: string
pubkey?: string
tags?: string[][]
}
export type DifferenceFeed = [FeedType.Difference, ...Feed[]]
export type IntersectionFeed = [FeedType.Intersection, ...Feed[]]
export type SymmetricDifferenceFeed = [FeedType.SymmetricDifference, ...Feed[]]
export type UnionFeed = [FeedType.Union, ...Feed[]]
export type FilterFeed = [FeedType.Filter, ...DynamicFilter[]]
export type DVMFeed = [FeedType.DVM, ...DVMRequest[]]
export type ListFeed = [FeedType.List, ...string[]]
export type LOLFeed = [FeedType.LOL, ...string[]]
export type DVMFeed = [FeedType.DVM, ...DVMRequest[]]
export type Feed = DifferenceFeed | IntersectionFeed | SymmetricDifferenceFeed | UnionFeed | FilterFeed | ListFeed | LOLFeed | DVMFeed
export type Feed = UnionFeed | FilterFeed | ListFeed | LOLFeed | DVMFeed
export const difference = (...feeds: Feed[]) => [FeedType.Difference, ...feeds]
export const intersection = (...feeds: Feed[]) => [FeedType.Intersection, ...feeds]
export const symmetricDifference = (...feeds: Feed[]) => [FeedType.SymmetricDifference, ...feeds]
export const union = (...feeds: Feed[]) => [FeedType.Union, ...feeds]
export const filter = (...filters: DynamicFilter[]) => [FeedType.Filter, ...filters]
export const list = (...addresses: string[]) => [FeedType.List, ...addresses]
export const lol = (...addresses: string[]) => [FeedType.LOL, ...addresses]
export const dvm = (...requests: DVMRequest[]) => [FeedType.DVM, ...requests]
export type InterpreterOpts = {
reqDvm: (request: DVMRequest, opts: ExecuteOpts) => void
reqFilters: (filters: Filter[], opts: ExecuteOpts) => void
getPubkeysForScope: (scope: Scope) => string[]
getPubkeysForWotRange: (minWot: number, maxWot: number) => string[]
}
export const union = (...feeds: Feed[]) =>
[FeedType.Union, ...feeds] as UnionFeed
export const filter = (...filters: DynamicFilter[]) =>
[FeedType.Filter, ...filters] as FilterFeed
export const list = (...addresses: string[]) =>
[FeedType.List, ...addresses] as ListFeed
export const lol = (...addresses: string[]) =>
[FeedType.LOL, ...addresses] as LOLFeed
export const dvm = (...requests: DVMRequest[]) =>
[FeedType.DVM, ...requests] as DVMFeed
export type ExecuteOpts = {
onEvent: (event: Rumor) => void
onComplete?: () => void
}
export class Interpreter {
constructor(readonly opts: InterpreterOpts) {}
export type FeedCompilerOpts = {
reqDvm: (request: DVMRequest, opts: ExecuteOpts) => void
reqFilters: (filters: Filter[], opts: ExecuteOpts) => void
getPubkeysForScope: (scope: Scope) => string[]
getPubkeysForWotRange: (minWot: number, maxWot: number) => string[]
}
export class FeedCompiler {
constructor(readonly opts: FeedCompilerOpts) {}
// Dispatch to different types of feed
execute([type, ...feed]: Feed, opts: ExecuteOpts) {
switch(type) {
case FeedType.Difference:
return this._executeDifference(feed as Feed[], opts)
case FeedType.Intersection:
return this._executeIntersection(feed as Feed[], opts)
case FeedType.SymmetricDifference:
return this._executeSymmetricDifference(feed as Feed[], opts)
default:
return this._feedToFilters([type, ...feed] as Feed).then(filters =>
this.opts.reqFilters(filters.map(this._compileFilter), opts)
)
}
execute(feed: Feed, opts: ExecuteOpts) {
return this.compile(feed).then(filters =>
this.opts.reqFilters(filters, opts)
)
}
async _feedToFilters([type, ...feed]: Feed) {
async compile([type, ...feed]: Feed) {
switch(type) {
case FeedType.Union:
return await this._unionToFilters(feed as Feed[])
return await this._compileUnion(feed as Feed[])
case FeedType.List:
return await this._listsToFilters(feed as string[])
return await this._compileLists(feed as string[])
case FeedType.LOL:
return await this._lolsToFilters(feed as string[])
return await this._compileLols(feed as string[])
case FeedType.DVM:
return await this._dvmsToFilters(feed as DVMRequest[])
return await this._compileDvms(feed as DVMRequest[])
case FeedType.Filter:
return feed as Filter[]
return (feed as DynamicFilter[]).map(filter => this._compileFilter(filter))
default:
throw new Error(`Unable to convert feed of type ${type} to filters`)
}
}
async _feedsToFilters(feeds: Feed[]) {
// Everything can be compiled to filters
async _compileUnion(feeds: Feed[]): Promise<Filter[]> {
const filters: Filter[] = []
await Promise.all(
feeds.map(async feed => {
for (const filter of await this._feedToFilters(feed)) {
filters.push(this._compileFilter(filter))
for (const filter of await this.compile(feed)) {
filters.push(filter)
}
})
)
return filters
return mergeFilters(filters)
}
// Special-case executors for set operations we can't infer filters for
async _executeDifference(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) {
const skip = new Set<string>()
const events: Rumor[] = []
feeds.forEach((subFeed: Feed, i: number) => {
this.execute(subFeed, {
onEvent: (event: Rumor) => {
if (i === 0) {
events.push(event)
} else {
skip.add(event.id)
}
},
onComplete: () => {
for (const event of events) {
if (!skip.has(event.id)) {
onEvent(event)
}
onComplete?.()
}
},
})
})
}
async _executeIntersection(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) {
const counts = new Map<string, number>()
const events = new Map<string, Rumor>()
feeds.forEach((subFeed: Feed, i: number) => {
this.execute(subFeed, {
onEvent: (event: Rumor) => {
events.set(event.id, event)
counts.set(event.id, inc(counts.get(event.id)))
},
onComplete: () => {
for (const event of events.values()) {
if (counts.get(event.id) === feeds.length) {
onEvent(event)
}
onComplete?.()
}
},
})
})
}
async _executeSymmetricDifference(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) {
const counts = new Map<string, number>()
const events = new Map<string, Rumor>()
feeds.forEach((subFeed: Feed, i: number) => {
this.execute(subFeed, {
onEvent: (event: Rumor) => {
events.set(event.id, event)
counts.set(event.id, inc(counts.get(event.id)))
},
onComplete: () => {
for (const event of events.values()) {
if (counts.get(event.id) === 1) {
onEvent(event)
}
onComplete?.()
}
},
})
})
}
// Everything else can be compiled to filters
async _unionToFilters(feeds: Feed[]): Promise<Filter[]> {
return mergeFilters(await this._feedsToFilters(feeds))
}
async _listsToFilters(addresses: string[]): Promise<Filter[]> {
async _compileLists(addresses: string[]): Promise<Filter[]> {
return new Promise(resolve => {
const events: Rumor[] = []
this.opts.reqFilters(getIdFilters(addresses), {
onEvent: (event: Rumor) => events.push(event),
onEvent: (event: Rumor) =>events.push(event),
onComplete: () => resolve(this._getFiltersFromTags(Tags.fromEvents(events))),
})
})
}
async _lolsToFilters(addresses: string[]): Promise<Filter[]> {
async _compileLols(addresses: string[]): Promise<Filter[]> {
return new Promise(resolve => {
const events: Rumor[] = []
this.opts.reqFilters(getIdFilters(addresses), {
onEvent: (event: Rumor) => events.push(event),
onComplete: () => resolve(this._listsToFilters(Tags.fromEvents(events).values("a").valueOf())),
onComplete: () => resolve(this._compileLists(Tags.fromEvents(events).values("a").valueOf())),
})
})
}
async _dvmsToFilters(requests: DVMRequest[]): Promise<Filter[]> {
async _compileDvms(requests: DVMRequest[]): Promise<Filter[]> {
const events: Rumor[] = []
await Promise.all(
@@ -273,8 +166,16 @@ export class Interpreter {
filter.authors = scopes.flatMap(scope => this.opts.getPubkeysForScope(scope))
}
if ((!isNil(min_wot) || !isNil(max_wot)) && !filter.authors) {
filter.authors = this.opts.getPubkeysForWotRange(min_wot || 1, max_wot || 1)
if ((!isNil(min_wot) || !isNil(max_wot))) {
const authors = this.opts.getPubkeysForWotRange(min_wot || 0, max_wot || 1)
if (filter.authors) {
const authorsSet = new Set(authors)
filter.authors = filter.authors.filter(pubkey => authorsSet.has(pubkey))
} else {
filter.authors = authors
}
}
if (!isNil(until_ago)) {