Re-work feeds

build_and_link.sh

@@ -3,6 +3,7 @@
 upstream=$1
 
 npm run build -w @welshman/$upstream
+npm run lint -w @welshman/$upstream
 
 for downstream in $(ls packages); do
   n=@welshman/$upstream

packages/feeds/README.md

@@ -14,7 +14,7 @@ const loader = new FeedLoader({
 })
 
 // Define a feed using set operations
-const feed = intersectionFeed(
+const feed = intersectFeed(
   unionFeed(
     dvmFeed({
       kind: 5300,

packages/feeds/compiler.ts

@@ -1,7 +1,7 @@
-import {uniq, tryCatch, now, isNil} from '@welshman/lib'
+import {uniq, flatten, pushToMapKey, intersection, ensureNumber, tryCatch, now} from '@welshman/lib'
 import type {Rumor, Filter} from '@welshman/util'
-import {Tags, intersectFilters, BOGUS_RELAY_URL, getIdFilters, unionFilters} from '@welshman/util'
+import {Tags, intersectFilters, getAddress, getIdFilters, unionFilters} from '@welshman/util'
-import type {RequestItem, DVMItem, Scope, Feed, DynamicFilter, FeedOptions} from './core'
+import type {RequestItem, TagFilterMapping, ListItem, DVMItem, Scope, Feed, FeedOptions} from './core'
 import {FeedType, getSubFeeds} from './core'
 
 export class FeedCompiler<E extends Rumor> {
@@ -17,184 +17,232 @@ export class FeedCompiler<E extends Rumor> {
 
   canCompile([type, ...feed]: Feed): boolean {
     switch(type) {
-      case FeedType.Relay:
       case FeedType.Union:
       case FeedType.Intersection:
         return getSubFeeds([type, ...feed] as Feed).every(this.canCompile)
-      case FeedType.Filter:
+      case FeedType.Address:
-      case FeedType.List:
+      case FeedType.Author:
       case FeedType.DVM:
+      case FeedType.ID:
+      case FeedType.Kind:
+      case FeedType.List:
+      case FeedType.Relay:
+      case FeedType.Scope:
+      case FeedType.Since:
+      case FeedType.SinceAgo:
+      case FeedType.ATag:
+      case FeedType.ETag:
+      case FeedType.PTag:
+      case FeedType.TTag:
+      case FeedType.Until:
+      case FeedType.UntilAgo:
+      case FeedType.WOT:
         return true
       default:
         return false
     }
   }
 
-  async compile([type, ...feed]: Feed) {
+  async compile([type, ...feed]: Feed): Promise<RequestItem[]> {
     switch(type) {
-      case FeedType.Union:
+      case FeedType.Address:      return this._compileAddresses(feed as string[])
-        return await this._compileUnion(feed as Feed[])
+      case FeedType.Author:       return this._compileFilter("authors", feed as string[])
-      case FeedType.Intersection:
+      case FeedType.DVM:          return await this._compileDvms(feed as DVMItem[])
-        return await this._compileIntersection(feed as Feed[])
+      case FeedType.ID:           return this._compileFilter("ids", feed as string[])
-      case FeedType.Relay:
+      case FeedType.Intersection: return await this._compileIntersection(feed as Feed[])
-        /* eslint no-case-declarations: 0 */
+      case FeedType.Kind:         return this._compileFilter("kinds", feed as number[])
-        const {relays, filters} = await this._compileUnion(feed.slice(1) as Feed[])
+      case FeedType.List:         return await this._compileLists(feed as ListItem[])
-
+      case FeedType.Relay:        return [{relays: feed as string[]}]
-        return {relays: relays.concat(feed[0] as string[]), filters}
+      case FeedType.Scope:        return this._compileScopes(feed as Scope[])
-      case FeedType.Filter:
+      case FeedType.Since:        return this._compileFilter("since", feed[0] as number)
-        return {
+      case FeedType.SinceAgo:     return this._compileFilter("since", now() - (feed[0] as number))
-          relays: [],
+      case FeedType.ATag:         return this._compileFilter("#a", feed as string[])
-          filters: (feed as DynamicFilter[]).map(filter => this._compileFilter(filter)),
+      case FeedType.ETag:         return this._compileFilter("#e", feed as string[])
-        }
+      case FeedType.PTag:         return this._compileFilter("#p", feed as string[])
-      case FeedType.List:
+      case FeedType.TTag:         return this._compileFilter("#t", feed as string[])
-        return await this._compileLists(feed as string[])
+      case FeedType.Until:        return this._compileFilter("until", feed[0] as number)
-      case FeedType.DVM:
+      case FeedType.UntilAgo:     return this._compileFilter("until", now() - (feed[0] as number))
-        return await this._compileDvms(feed as DVMItem[])
+      case FeedType.Union:        return await this._compileUnion(feed as Feed[])
+      case FeedType.WOT:          return this._compileWot(feed[0] as number, feed[1] as number)
       default:
         throw new Error(`Unable to convert feed of type ${type} to filters`)
     }
   }
 
-  async _compileUnion(feeds: Feed[]): Promise<RequestItem> {
+  _compileAddresses(addresses: string[]) {
-    const relays: string[] = []
+    return [{filters: getIdFilters(addresses)}]
+  }
+
+  _compileFilter(key: string, value: any) {
+    return [{filters: [{[key]: value} as Filter]}]
+  }
+
+  _compileScopes(scopes: Scope[]) {
+    return [{filters: [{authors: uniq(scopes.flatMap(this.options.getPubkeysForScope))}]}]
+  }
+
+  _compileWot(min_wot: number, max_wot: number) {
+    return [{filters: [{authors: this.options.getPubkeysForWotRange(min_wot, max_wot)}]}]
+  }
+
+  async _compileDvms(items: DVMItem[]): Promise<RequestItem[]> {
     const filters: Filter[] = []
 
     await Promise.all(
-      feeds.map(async feed => {
+      items.map(({mappings, ...request}) =>
-        const item = await this.compile(feed)
-
-        for (const relay of item.relays) {
-          relays.push(relay)
-        }
-
-        for (const filter of item.filters) {
-          filters.push(filter)
-        }
-      })
-    )
-
-    return {
-      relays: uniq(relays),
-      filters: unionFilters(filters),
-    }
-  }
-
-  async _compileIntersection(feeds: Feed[]): Promise<RequestItem> {
-    const items = await Promise.all(feeds.map(this.compile))
-    const filters = intersectFilters(items.map(item => item.filters))
-
-    let relays = uniq(items.flatMap(item => item.relays))
-    let hasRelays = relays.length > 0
-
-    items.forEach((item, i) => {
-      if (item.relays.length > 0) {
-        relays = relays.filter(relay => item.relays.includes(relay))
-      }
-    })
-
-    if (hasRelays && relays.length === 0) {
-      relays.push(BOGUS_RELAY_URL)
-    }
-
-    return {relays, filters}
-  }
-
-  async _compileLists(addresses: string[]): Promise<RequestItem> {
-    const events: E[] = []
-
-    await this.options.request({
-      relays: [],
-      filters: getIdFilters(addresses),
-      onEvent: (e: E) => events.push(e),
-    })
-
-
-    return {
-      relays: [],
-      filters: this._getFiltersFromTags(Tags.fromEvents(events)),
-    }
-  }
-
-  async _compileDvms(requests: DVMItem[]): Promise<RequestItem> {
-    const responseTags: Tags[] = []
-
-    await Promise.all(
-      requests.map(request =>
         this.options.requestDvm({
           ...request,
           onEvent: async (e: E) => {
             const tags = Tags.fromEvent(e)
-            const {id, pubkey} = await tryCatch(() => JSON.parse(tags.get("request")?.value())) || {}
+            const request = await tryCatch(() => JSON.parse(tags.get("request")?.value()))
+            const responseTags = tags.rejectByValue([request?.id, request?.pubkey])
 
-            responseTags.push(tags.filterByKey(["t", "p", "e", "a"]).rejectByValue([id, pubkey]))
+            for (const filter of await this._getFiltersFromTags(responseTags, mappings)) {
+              filters.push(filter)
+            }
           },
         })
       )
     )
 
-    const mergedTags = Tags.from(responseTags.flatMap(tags => tags.valueOf()))
+    return [{filters: unionFilters(filters)}]
-
-    return {
-      relays: mergedTags.relays().valueOf(),
-      filters: this._getFiltersFromTags(mergedTags),
   }
+
+  async _compileIntersection(feeds: Feed[]): Promise<RequestItem[]> {
+    const [head, ...tail] = await Promise.all(feeds.map(this.compile))
+
+    const result = []
+
+    for (let {filters, relays} of head) {
+      const matchingGroups = tail.map(
+        items => items.filter(
+          it => (
+            (!relays || !it.relays || intersection(relays, it.relays).length > 0) &&
+            (!filters || !it.filters || intersectFilters([filters, it.filters]).length > 0)
+          )
+        )
+      ).filter(
+        items => items.length > 0
+      )
+
+      if (matchingGroups.length < tail.length) {
+        continue
+      }
+
+      for (const items of matchingGroups) {
+        for (const item of items) {
+          if (relays && item.relays) {
+            relays = relays.filter(r => item.relays!.includes(r))
+          } else if (item.relays) {
+            relays = item.relays
+          }
+
+          if (filters && item.filters) {
+            filters = intersectFilters([filters, item.filters])
+          } else if (item.filters) {
+            filters = item.filters
+          }
+        }
+      }
+
+      result.push({relays, filters})
+    }
+
+    return result
+  }
+
+  async _compileUnion(feeds: Feed[]): Promise<RequestItem[]> {
+    const filtersByRelay = new Map<string, Filter[]>()
+    const filtersWithoutRelay: Filter[] = []
+    const relaysWithoutFilter: string[] = []
+
+    await Promise.all(
+      feeds.map(async feed => {
+        for (const item of await this.compile(feed)) {
+          if (item.relays) {
+            for (const relay of item.relays) {
+              if (item.filters) {
+                for (const filter of item.filters) {
+                  pushToMapKey(filtersByRelay, relay, filter)
+                }
+              } else {
+                relaysWithoutFilter.push(relay)
+              }
+            }
+          } else if (item.filters) {
+            for (const filter of item.filters) {
+              filtersWithoutRelay.push(filter)
+            }
+          }
+        }
+      })
+    )
+
+    const items: RequestItem[] = []
+
+    for (const [relay, filters] of filtersByRelay.entries()) {
+      items.push({
+        relays: [relay],
+        filters: unionFilters(filters),
+      })
+    }
+
+    if (filtersWithoutRelay.length > 0) {
+      items.push({filters: unionFilters(filtersWithoutRelay)})
+    }
+
+    if (relaysWithoutFilter.length > 0) {
+      items.push({relays: uniq(relaysWithoutFilter)})
+    }
+
+    return items
+  }
+
+  async _compileLists(listItems: ListItem[]): Promise<RequestItem[]> {
+    const addresses = uniq(listItems.map(({address}) => address))
+    const eventsByAddress = new Map<string, E>()
+
+    await this.options.request({
+      filters: getIdFilters(addresses),
+      onEvent: (e: E) => eventsByAddress.set(getAddress(e), e),
+    })
+
+    const filters = flatten(
+      await Promise.all(
+        listItems.map(({address, mappings}) => {
+          const event = eventsByAddress.get(address)
+
+          return event ? this._getFiltersFromTags(Tags.fromEvent(event), mappings) : []
+        })
+      )
+    )
+
+    return [{filters: unionFilters(filters)}]
   }
 
   // Utilities
 
-  _compileFilter({scopes, min_wot, max_wot, until_ago, since_ago, ...filter}: DynamicFilter) {
+  async _getFiltersFromTags(tags: Tags, mappings: TagFilterMapping[]) {
-    if (scopes && !filter.authors) {
+    const filters = []
-      filter.authors = scopes.flatMap((scope: Scope) => this.options.getPubkeysForScope(scope))
+
+    for (const [tagName, feedType] of mappings) {
+      const filterTags = tags.whereKey(tagName)
+
+      if (filterTags.exists()) {
+        let values: string[] | number[] = filterTags.values().valueOf()
+
+        if (feedType === FeedType.Kind) {
+          values = values.map(ensureNumber) as number[]
         }
 
-    if ((!isNil(min_wot) || !isNil(max_wot))) {
+        for (const item of await this.compile([feedType, ...values] as Feed)) {
-      const authors = this.options.getPubkeysForWotRange(min_wot || 0, max_wot || 1)
+          for (const filter of item.filters || []) {
-
-      if (filter.authors) {
-        const authorsSet = new Set(authors)
-
-        filter.authors = filter.authors.filter(pubkey => authorsSet.has(pubkey))
-      } else {
-        filter.authors = authors
-      }
-    }
-
-    if (!isNil(until_ago)) {
-      filter.until = now() - until_ago!
-    }
-
-    if (!isNil(since_ago)) {
-      filter.since = now() - since_ago!
-    }
-
-    return filter as Filter
-  }
-
-  _getFiltersFromTags(tags: Tags) {
-    const ttags = tags.values("t")
-    const ptags = tags.values("p")
-    const eatags = tags.filterByKey(["e", "a"]).values()
-    const filters: Filter[] = []
-
-    if (ttags.exists()) {
-      filters.push({"#t": ttags.valueOf()})
-    }
-
-    if (ptags.exists()) {
-      filters.push({authors: ptags.valueOf()})
-    }
-
-    if (eatags.exists()) {
-      for (const filter of getIdFilters(eatags.valueOf())) {
             filters.push(filter)
           }
         }
-
+      }
-    // If we don't have any filters, return nothing instead of everything
-    if (filters.length === 0) {
-      filters.push({authors: []})
     }
 
-    return filters
+    return unionFilters(filters)
   }
 }

packages/feeds/core.ts

@@ -1,14 +1,27 @@
 import type {Filter} from '@welshman/util'
 
 export enum FeedType {
-  Difference = "difference",
+  Address = "address",
-  Intersection = "intersection",
+  Author = "author",
-  SymmetricDifference = "symdiff",
-  Union = "union",
-  Filter = "filter",
-  Relay = "relay",
-  List = "list",
   DVM = "dvm",
+  Difference = "difference",
+  ID = "id",
+  Intersection = "intersection",
+  Kind = "kind",
+  List = "list",
+  WOT = "wot",
+  Relay = "relay",
+  Scope = "scope",
+  Since = "since",
+  SinceAgo = "since_ago",
+  SymmetricDifference = "symmetric_difference",
+  ATag = "#a",
+  ETag = "#e",
+  PTag = "#p",
+  TTag = "#t",
+  Union = "union",
+  Until = "until",
+  UntilAgo = "until_ago",
 }
 
 export enum Scope {
@@ -18,82 +31,154 @@ export enum Scope {
   Self = "self",
 }
 
-export type DynamicFilter = Filter & {
+export type TagFeedType =
-  scopes?: Scope[]
+  FeedType.ATag |
-  min_wot?: number
+  FeedType.ETag |
-  max_wot?: number
+  FeedType.PTag |
-  until_ago?: number
+  FeedType.TTag
-  since_ago?: number
+
+
+export type FilterFeedType =
+  FeedType.ID |
+  FeedType.Address |
+  FeedType.Author |
+  FeedType.Kind |
+  FeedType.Relay |
+  TagFeedType
+
+export type TagFilterMapping = [string, FilterFeedType]
+
+export type DVMItem = {
+  kind: number,
+  mappings: TagFilterMapping[],
+  tags?: string[][],
+  relays?: string[],
 }
 
-export type RelayFeed = [FeedType.Relay, string[], ...Feed[]]
+export type ListItem = {
-export type DifferenceFeed = [FeedType.Difference, ...Feed[]]
+  address: string,
-export type IntersectionFeed = [FeedType.Intersection, ...Feed[]]
+  mappings: TagFilterMapping[],
-export type SymmetricDifferenceFeed = [FeedType.SymmetricDifference, ...Feed[]]
+}
-export type UnionFeed = [FeedType.Union, ...Feed[]]
+
-export type FilterFeed = [FeedType.Filter, ...DynamicFilter[]]
+export type WOTItem = {
-export type ListFeed = [FeedType.List, ...string[]]
+  min: number,
-export type DVMFeed = [FeedType.DVM, ...DVMItem[]]
+  max: number,
+}
+
+export type AddressFeed = [type: FeedType.Address, ...addresses: string[]]
+export type AuthorFeed = [type: FeedType.Author, ...pubkeys: string[]]
+export type DVMFeed = [type: FeedType.DVM, ...items: DVMItem[]]
+export type DifferenceFeed = [type: FeedType.Difference, ...feeds: Feed[]]
+export type IDFeed = [type: FeedType.ID, ...ids: string[]]
+export type IntersectionFeed = [type: FeedType.Intersection, ...feeds: Feed[]]
+export type KindFeed = [type: FeedType.Kind, ...kinds: number[]]
+export type ListFeed = [type: FeedType.List, ...items: ListItem[]]
+export type WOTFeed = [type: FeedType.WOT, ...items: WOTItem[]]
+export type RelayFeed = [type: FeedType.Relay, ...urls: string[]]
+export type ScopeFeed = [type: FeedType.Scope, ...scopes: Scope[]]
+export type SinceAgoFeed = [type: FeedType.SinceAgo, since_ago: number]
+export type SinceFeed = [type: FeedType.Since, since: number]
+export type SymmetricDifferenceFeed = [type: FeedType.SymmetricDifference, ...feeds: Feed[]]
+export type ATagFeed = [type: FeedType.ATag, ...addresses: string[]]
+export type ETagFeed = [type: FeedType.ETag, ...ids: string[]]
+export type PTagFeed = [type: FeedType.PTag, ...pubkeys: string[]]
+export type TTagFeed = [type: FeedType.TTag, ...topics: string[]]
+export type UnionFeed = [type: FeedType.Union, ...feeds: Feed[]]
+export type UntilAgoFeed = [type: FeedType.UntilAgo, until_ago: number]
+export type UntilFeed = [type: FeedType.Until, until: number]
 
 export type Feed =
-  RelayFeed |
+  AddressFeed |
+  AuthorFeed |
+  DVMFeed |
   DifferenceFeed |
+  IDFeed |
   IntersectionFeed |
-  SymmetricDifferenceFeed |
+  KindFeed |
-  UnionFeed |
-  FilterFeed |
-  RelayFeed |
   ListFeed |
-  DVMFeed
+  WOTFeed |
+  RelayFeed |
+  ScopeFeed |
+  SinceAgoFeed |
+  SinceFeed |
+  SymmetricDifferenceFeed |
+  ATagFeed |
+  ETagFeed |
+  PTagFeed |
+  TTagFeed |
+  UnionFeed |
+  UntilAgoFeed |
+  UntilFeed
 
-export const relayFeed = (relays: string[], ...feeds: Feed[]) => [FeedType.Relay, relays, ...feeds] as Feed
+export const addressFeed = (...addresses: string[]): AddressFeed => [FeedType.Address, ...addresses]
-export const differenceFeed = (...feeds: Feed[]) => [FeedType.Difference, ...feeds] as Feed
+export const authorFeed = (...pubkeys: string[]): AuthorFeed => [FeedType.Author, ...pubkeys]
-export const intersectionFeed = (...feeds: Feed[]) => [FeedType.Intersection, ...feeds] as Feed
+export const dvmFeed = (...items: DVMItem[]): DVMFeed => [FeedType.DVM, ...items]
-export const symmetricDifferenceFeed = (...feeds: Feed[]) => [FeedType.SymmetricDifference, ...feeds] as Feed
+export const differenceFeed = (...feeds: Feed[]): DifferenceFeed => [FeedType.Difference, ...feeds]
-export const unionFeed = (...feeds: Feed[]) => [FeedType.Union, ...feeds] as Feed
+export const idFeed = (...ids: string[]): IDFeed => [FeedType.ID, ...ids]
-export const filterFeed = (...filters: DynamicFilter[]) => [FeedType.Filter, ...filters] as Feed
+export const intersectionFeed = (...feeds: Feed[]): IntersectionFeed => [FeedType.Intersection, ...feeds]
-export const listFeed = (...addresses: string[]) => [FeedType.List, ...addresses] as Feed
+export const kindFeed = (...kinds: number[]): KindFeed => [FeedType.Kind, ...kinds]
-export const dvmFeed = (...requests: DVMItem[]) => [FeedType.DVM, ...requests] as Feed
+export const listFeed = (...items: ListItem[]): ListFeed => [FeedType.List, ...items]
+export const wotFeed = (...items: WOTItem[]): WOTFeed => [FeedType.WOT, ...items]
+export const relayFeed = (...urls: string[]): RelayFeed => [FeedType.Relay, ...urls]
+export const scopeFeed = (...scopes: Scope[]): ScopeFeed => [FeedType.Scope, ...scopes]
+export const sinceAgoFeed = (since_ago: number): SinceAgoFeed => [FeedType.SinceAgo, since_ago]
+export const sinceFeed = (since: number): SinceFeed => [FeedType.Since, since]
+export const symmetricDifferenceFeed = (...feeds: Feed[]): SymmetricDifferenceFeed => [FeedType.SymmetricDifference, ...feeds]
+export const aTagFeed = (...values: string[]): ATagFeed => [FeedType.ATag, ...values]
+export const eTagFeed = (...values: string[]): ETagFeed => [FeedType.ETag, ...values]
+export const pTagFeed = (...values: string[]): PTagFeed => [FeedType.PTag, ...values]
+export const tTagFeed = (...values: string[]): TTagFeed => [FeedType.TTag, ...values]
+export const unionFeed = (...feeds: Feed[]): UnionFeed => [FeedType.Union, ...feeds]
+export const untilAgoFeed = (until_ago: number): UntilAgoFeed => [FeedType.UntilAgo, until_ago]
+export const untilFeed = (until: number): UntilFeed => [FeedType.Until, until]
 
-export const hasSubFeeds = ([type]: Feed) =>
+export const feedsFromFilter = (filter: Filter) => {
+  const feeds = []
+
+  for (const [k, v] of Object.entries(filter)) {
+    if (k === 'ids') feeds.push(idFeed(...v as string[]))
+    else if (k === 'kinds') feeds.push(kindFeed(...v as number[]))
+    else if (k === 'authors') feeds.push(authorFeed(...v as string[]))
+    else if (k === 'since') feeds.push(sinceFeed(v as number))
+    else if (k === 'until') feeds.push(untilFeed(v as number))
+    else if (k === "#a") feeds.push(aTagFeed(...v as string[]))
+    else if (k === "#e") feeds.push(eTagFeed(...v as string[]))
+    else if (k === "#p") feeds.push(pTagFeed(...v as string[]))
+    else if (k === "#t") feeds.push(tTagFeed(...v as string[]))
+    else throw new Error(`Unable to create feed from filter ${k}: ${v}`)
+  }
+
+  return feeds
+}
+
+export const feedFromFilter = (filter: Filter) => intersectionFeed(...feedsFromFilter(filter))
+
+export const hasSubFeeds = ([type]: [FeedType]) =>
   [
     FeedType.Union,
     FeedType.Intersection,
     FeedType.Difference,
     FeedType.SymmetricDifference,
-    FeedType.Relay,
   ].includes(type)
 
-export const getSubFeeds = ([type, ...feed]: Feed): Feed[] => {
+export const getSubFeeds = ([type, ...feeds]: Feed): Feed[] => hasSubFeeds([type]) ? feeds as Feed[] : []
-  switch (type) {
-    case FeedType.Relay:
-      return feed.slice(1) as Feed[]
-    case FeedType.Difference:
-    case FeedType.Intersection:
-    case FeedType.SymmetricDifference:
-    case FeedType.Union:
-      return feed as Feed[]
-    default:
-      return []
-  }
-}
 
 export type RequestItem = {
-  relays: string[]
+  relays?: string[]
-  filters: Filter[]
+  filters?: Filter[]
 }
 
 export type RequestOpts<E> = RequestItem & {
   onEvent: (event: E) => void
 }
 
-export type DVMItem = {
+export type DVMRequest = {
-  kind: number
+  kind: number,
-  tags?: string[][]
+  tags?: string[][],
-  relays?: string[]
+  relays?: string[],
 }
 
-export type DVMOpts<E> = DVMItem & {
+export type DVMOpts<E> = DVMRequest & {
   onEvent: (event: E) => void
 }
 

packages/feeds/loader.ts

@@ -21,7 +21,7 @@ export class FeedLoader<E extends Rumor> {
 
   async getLoader([type, ...feed]: Feed, loadOpts: LoadOpts<E>) {
     if (this.compiler.canCompile([type, ...feed] as Feed)) {
-      return this._getRequestLoader(await this.compiler.compile([type, ...feed] as Feed), loadOpts)
+      return this._getRequestsLoader(await this.compiler.compile([type, ...feed] as Feed), loadOpts)
     }
 
     switch(type) {
@@ -38,9 +38,35 @@ export class FeedLoader<E extends Rumor> {
     }
   }
 
+  async _getRequestsLoader(requests: RequestItem[], {onEvent, onExhausted}: LoadOpts<E>) {
+    const seen = new Set()
+    const exhausted = new Set()
+    const loaders = await Promise.all(
+      requests.map(
+        request => this._getRequestLoader(request, {
+          onExhausted: () => exhausted.add(request),
+          onEvent: e => {
+            if (!seen.has(e.id)) {
+              onEvent?.(e)
+              seen.add(e.id)
+            }
+          },
+        })
+      )
+    )
+
+    return async (limit: number) => {
+      await Promise.all(loaders.map(loader => loader(limit)))
+
+      if (exhausted.size === requests.length) {
+        onExhausted?.()
+      }
+    }
+  }
+
   async _getRequestLoader({relays, filters}: RequestItem, {onEvent, onExhausted}: LoadOpts<E>) {
     // Make sure we have some kind of filter to send if we've been given an empty one, as happens with relay feeds
-    if (filters.length === 0) {
+    if (!filters || filters.length === 0) {
       filters = [{}]
     }
 
@@ -55,7 +81,7 @@ export class FeedLoader<E extends Rumor> {
     let until = maxUntil
 
     return async (limit: number) => {
-      const requestFilters = filters
+      const requestFilters = filters!
         // Remove filters that don't fit our window
         .filter((filter: Filter) => {
           const filterSince = filter.since || EPOCH

packages/feeds/tsc-multi.json

@@ -1,6 +1,5 @@
 {
   "targets": [
-    {"extname": ".cjs", "module": "commonjs"},
     {"extname": ".mjs", "module": "esnext", "moduleResolution": "node"}
   ],
   "projects": ["tsconfig.json"]

packages/lib/Tools.ts

@@ -73,6 +73,20 @@ export const concat = <T>(...xs: (T | Nil)[][]) => xs.flatMap(x => x || [])
 
 export const append = <T>(xs: (T | Nil)[], x: T) => concat(xs, [x])
 
+export const union = <T>(a: T[], b: T[]) => uniq([...a, ...b])
+
+export const intersection = <T>(a: T[], b: T[]) => {
+  const s = new Set(b)
+
+  return a.filter(x => s.has(x))
+}
+
+export const difference = <T>(a: T[], b: T[]) => {
+  const s = new Set(b)
+
+  return a.filter(x => !s.has(x))
+}
+
 export const clamp = ([min, max]: [number, number], n: number) => Math.min(max, Math.max(min, n))
 
 export const tryCatch = async <T>(f: () => Promise<T | void> | T | void, onError?: (e: Error) => void): Promise<T | void> => {
@@ -114,7 +128,9 @@ export const toIterable = (x: any) => isIterable(x) ? x : [x]
 
 export const ensurePlural = <T>(x: T | T[]) => (x instanceof Array ? x : [x])
 
-export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
+export const ensureNumber = (x: number | string) => parseFloat(x as string)
+
+export const flatten = <T>(xs: T[][]) => xs.flatMap(identity)
 
 export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))