diff --git a/packages/custom-feeds/index.ts b/packages/custom-feeds/index.ts index dcd57f9..4fa1a4d 100644 --- a/packages/custom-feeds/index.ts +++ b/packages/custom-feeds/index.ts @@ -2,17 +2,8 @@ import {inc, now, isNil} from '@coracle.social/lib' import type {Rumor, Filter} from '@coracle.social/util' import {Tags, getIdFilters, mergeFilters} from '@coracle.social/util' -// TODO: -// - if one of the feeds in a union is a filter, don't execute it, -// use it to filter down results from other feeds -// - if multiple feeds in a composite are or result in filters (like lists, lols), -// merge them before executing the request - export enum FeedType { - Difference = "\\", - Intersection = "∩", - SymmetricDifference = "Δ", - Union = "∪", + Union = "union", Filter = "filter", List = "list", LOL = "lol", @@ -20,220 +11,122 @@ export enum FeedType { } export enum Scope { - User = "user", + Self = "self", Follows = "follows", Followers = "followers", } -export type WotRange = 0 | 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 | 0.7 | 0.8 | 0.9 | 1 - export type DynamicFilter = Filter & { scopes?: Scope[] - min_wot?: WotRange - max_wot?: WotRange + min_wot?: number + max_wot?: number until_ago?: number since_ago?: number } export type DVMRequest = { kind: number - input?: string - pubkey?: string + tags?: string[][] } -export type DifferenceFeed = [FeedType.Difference, ...Feed[]] -export type IntersectionFeed = [FeedType.Intersection, ...Feed[]] -export type SymmetricDifferenceFeed = [FeedType.SymmetricDifference, ...Feed[]] export type UnionFeed = [FeedType.Union, ...Feed[]] export type FilterFeed = [FeedType.Filter, ...DynamicFilter[]] -export type DVMFeed = [FeedType.DVM, ...DVMRequest[]] export type ListFeed = [FeedType.List, ...string[]] export type LOLFeed = [FeedType.LOL, ...string[]] +export type DVMFeed = [FeedType.DVM, ...DVMRequest[]] -export type Feed = DifferenceFeed | IntersectionFeed | SymmetricDifferenceFeed | UnionFeed | FilterFeed | ListFeed | LOLFeed | DVMFeed +export type Feed = UnionFeed | FilterFeed | ListFeed | LOLFeed | DVMFeed -export const difference = (...feeds: Feed[]) => [FeedType.Difference, ...feeds] - -export const intersection = (...feeds: Feed[]) => [FeedType.Intersection, ...feeds] - -export const symmetricDifference = (...feeds: Feed[]) => [FeedType.SymmetricDifference, ...feeds] - -export const union = (...feeds: Feed[]) => [FeedType.Union, ...feeds] - -export const filter = (...filters: DynamicFilter[]) => [FeedType.Filter, ...filters] - -export const list = (...addresses: string[]) => [FeedType.List, ...addresses] - -export const lol = (...addresses: string[]) => [FeedType.LOL, ...addresses] - -export const dvm = (...requests: DVMRequest[]) => [FeedType.DVM, ...requests] - -export type InterpreterOpts = { - reqDvm: (request: DVMRequest, opts: ExecuteOpts) => void - reqFilters: (filters: Filter[], opts: ExecuteOpts) => void - getPubkeysForScope: (scope: Scope) => string[] - getPubkeysForWotRange: (minWot: number, maxWot: number) => string[] -} +export const union = (...feeds: Feed[]) => + [FeedType.Union, ...feeds] as UnionFeed +export const filter = (...filters: DynamicFilter[]) => + [FeedType.Filter, ...filters] as FilterFeed +export const list = (...addresses: string[]) => + [FeedType.List, ...addresses] as ListFeed +export const lol = (...addresses: string[]) => + [FeedType.LOL, ...addresses] as LOLFeed +export const dvm = (...requests: DVMRequest[]) => + [FeedType.DVM, ...requests] as DVMFeed export type ExecuteOpts = { onEvent: (event: Rumor) => void onComplete?: () => void } -export class Interpreter { - constructor(readonly opts: InterpreterOpts) {} +export type FeedCompilerOpts = { + reqDvm: (request: DVMRequest, opts: ExecuteOpts) => void + reqFilters: (filters: Filter[], opts: ExecuteOpts) => void + getPubkeysForScope: (scope: Scope) => string[] + getPubkeysForWotRange: (minWot: number, maxWot: number) => string[] +} + +export class FeedCompiler { + constructor(readonly opts: FeedCompilerOpts) {} // Dispatch to different types of feed - execute([type, ...feed]: Feed, opts: ExecuteOpts) { - switch(type) { - case FeedType.Difference: - return this._executeDifference(feed as Feed[], opts) - case FeedType.Intersection: - return this._executeIntersection(feed as Feed[], opts) - case FeedType.SymmetricDifference: - return this._executeSymmetricDifference(feed as Feed[], opts) - default: - return this._feedToFilters([type, ...feed] as Feed).then(filters => - this.opts.reqFilters(filters.map(this._compileFilter), opts) - ) - } + execute(feed: Feed, opts: ExecuteOpts) { + return this.compile(feed).then(filters => + this.opts.reqFilters(filters, opts) + ) } - async _feedToFilters([type, ...feed]: Feed) { + async compile([type, ...feed]: Feed) { switch(type) { case FeedType.Union: - return await this._unionToFilters(feed as Feed[]) + return await this._compileUnion(feed as Feed[]) case FeedType.List: - return await this._listsToFilters(feed as string[]) + return await this._compileLists(feed as string[]) case FeedType.LOL: - return await this._lolsToFilters(feed as string[]) + return await this._compileLols(feed as string[]) case FeedType.DVM: - return await this._dvmsToFilters(feed as DVMRequest[]) + return await this._compileDvms(feed as DVMRequest[]) case FeedType.Filter: - return feed as Filter[] + return (feed as DynamicFilter[]).map(filter => this._compileFilter(filter)) default: throw new Error(`Unable to convert feed of type ${type} to filters`) } } - async _feedsToFilters(feeds: Feed[]) { + // Everything can be compiled to filters + + async _compileUnion(feeds: Feed[]): Promise { const filters: Filter[] = [] await Promise.all( feeds.map(async feed => { - for (const filter of await this._feedToFilters(feed)) { - filters.push(this._compileFilter(filter)) + for (const filter of await this.compile(feed)) { + filters.push(filter) } }) ) - return filters + return mergeFilters(filters) } - // Special-case executors for set operations we can't infer filters for - - async _executeDifference(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) { - const skip = new Set() - const events: Rumor[] = [] - - feeds.forEach((subFeed: Feed, i: number) => { - this.execute(subFeed, { - onEvent: (event: Rumor) => { - if (i === 0) { - events.push(event) - } else { - skip.add(event.id) - } - }, - onComplete: () => { - for (const event of events) { - if (!skip.has(event.id)) { - onEvent(event) - } - - onComplete?.() - } - }, - }) - }) - } - - async _executeIntersection(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) { - const counts = new Map() - const events = new Map() - - feeds.forEach((subFeed: Feed, i: number) => { - this.execute(subFeed, { - onEvent: (event: Rumor) => { - events.set(event.id, event) - counts.set(event.id, inc(counts.get(event.id))) - }, - onComplete: () => { - for (const event of events.values()) { - if (counts.get(event.id) === feeds.length) { - onEvent(event) - } - - onComplete?.() - } - }, - }) - }) - } - - async _executeSymmetricDifference(feeds: Feed[], {onEvent, onComplete}: ExecuteOpts) { - const counts = new Map() - const events = new Map() - - feeds.forEach((subFeed: Feed, i: number) => { - this.execute(subFeed, { - onEvent: (event: Rumor) => { - events.set(event.id, event) - counts.set(event.id, inc(counts.get(event.id))) - }, - onComplete: () => { - for (const event of events.values()) { - if (counts.get(event.id) === 1) { - onEvent(event) - } - - onComplete?.() - } - }, - }) - }) - } - - // Everything else can be compiled to filters - - async _unionToFilters(feeds: Feed[]): Promise { - return mergeFilters(await this._feedsToFilters(feeds)) - } - - async _listsToFilters(addresses: string[]): Promise { + async _compileLists(addresses: string[]): Promise { return new Promise(resolve => { const events: Rumor[] = [] this.opts.reqFilters(getIdFilters(addresses), { - onEvent: (event: Rumor) => events.push(event), + onEvent: (event: Rumor) =>events.push(event), onComplete: () => resolve(this._getFiltersFromTags(Tags.fromEvents(events))), }) }) } - async _lolsToFilters(addresses: string[]): Promise { + async _compileLols(addresses: string[]): Promise { return new Promise(resolve => { const events: Rumor[] = [] this.opts.reqFilters(getIdFilters(addresses), { onEvent: (event: Rumor) => events.push(event), - onComplete: () => resolve(this._listsToFilters(Tags.fromEvents(events).values("a").valueOf())), + onComplete: () => resolve(this._compileLists(Tags.fromEvents(events).values("a").valueOf())), }) }) } - async _dvmsToFilters(requests: DVMRequest[]): Promise { + async _compileDvms(requests: DVMRequest[]): Promise { const events: Rumor[] = [] await Promise.all( @@ -273,8 +166,16 @@ export class Interpreter { filter.authors = scopes.flatMap(scope => this.opts.getPubkeysForScope(scope)) } - if ((!isNil(min_wot) || !isNil(max_wot)) && !filter.authors) { - filter.authors = this.opts.getPubkeysForWotRange(min_wot || 1, max_wot || 1) + if ((!isNil(min_wot) || !isNil(max_wot))) { + const authors = this.opts.getPubkeysForWotRange(min_wot || 0, max_wot || 1) + + if (filter.authors) { + const authorsSet = new Set(authors) + + filter.authors = filter.authors.filter(pubkey => authorsSet.has(pubkey)) + } else { + filter.authors = authors + } } if (!isNil(until_ago)) {