Re-work feeds
This commit is contained in:
Jon Staab
@@ -14,7 +14,7 @@ const loader = new FeedLoader({
|
||||
})
|
||||
|
||||
// Define a feed using set operations
|
||||
const feed = intersectionFeed(
|
||||
const feed = intersectFeed(
|
||||
unionFeed(
|
||||
dvmFeed({
|
||||
kind: 5300,
|
||||
|
||||
+189
-141
@@ -1,7 +1,7 @@
|
||||
import {uniq, tryCatch, now, isNil} from '@welshman/lib'
|
||||
import {uniq, flatten, pushToMapKey, intersection, ensureNumber, tryCatch, now} from '@welshman/lib'
|
||||
import type {Rumor, Filter} from '@welshman/util'
|
||||
import {Tags, intersectFilters, BOGUS_RELAY_URL, getIdFilters, unionFilters} from '@welshman/util'
|
||||
import type {RequestItem, DVMItem, Scope, Feed, DynamicFilter, FeedOptions} from './core'
|
||||
import {Tags, intersectFilters, getAddress, getIdFilters, unionFilters} from '@welshman/util'
|
||||
import type {RequestItem, TagFilterMapping, ListItem, DVMItem, Scope, Feed, FeedOptions} from './core'
|
||||
import {FeedType, getSubFeeds} from './core'
|
||||
|
||||
export class FeedCompiler<E extends Rumor> {
|
||||
@@ -17,184 +17,232 @@ export class FeedCompiler<E extends Rumor> {
|
||||
|
||||
canCompile([type, ...feed]: Feed): boolean {
|
||||
switch(type) {
|
||||
case FeedType.Relay:
|
||||
case FeedType.Union:
|
||||
case FeedType.Intersection:
|
||||
return getSubFeeds([type, ...feed] as Feed).every(this.canCompile)
|
||||
case FeedType.Filter:
|
||||
case FeedType.List:
|
||||
case FeedType.Address:
|
||||
case FeedType.Author:
|
||||
case FeedType.DVM:
|
||||
case FeedType.ID:
|
||||
case FeedType.Kind:
|
||||
case FeedType.List:
|
||||
case FeedType.Relay:
|
||||
case FeedType.Scope:
|
||||
case FeedType.Since:
|
||||
case FeedType.SinceAgo:
|
||||
case FeedType.ATag:
|
||||
case FeedType.ETag:
|
||||
case FeedType.PTag:
|
||||
case FeedType.TTag:
|
||||
case FeedType.Until:
|
||||
case FeedType.UntilAgo:
|
||||
case FeedType.WOT:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async compile([type, ...feed]: Feed) {
|
||||
async compile([type, ...feed]: Feed): Promise<RequestItem[]> {
|
||||
switch(type) {
|
||||
case FeedType.Union:
|
||||
return await this._compileUnion(feed as Feed[])
|
||||
case FeedType.Intersection:
|
||||
return await this._compileIntersection(feed as Feed[])
|
||||
case FeedType.Relay:
|
||||
/* eslint no-case-declarations: 0 */
|
||||
const {relays, filters} = await this._compileUnion(feed.slice(1) as Feed[])
|
||||
|
||||
return {relays: relays.concat(feed[0] as string[]), filters}
|
||||
case FeedType.Filter:
|
||||
return {
|
||||
relays: [],
|
||||
filters: (feed as DynamicFilter[]).map(filter => this._compileFilter(filter)),
|
||||
}
|
||||
case FeedType.List:
|
||||
return await this._compileLists(feed as string[])
|
||||
case FeedType.DVM:
|
||||
return await this._compileDvms(feed as DVMItem[])
|
||||
case FeedType.Address: return this._compileAddresses(feed as string[])
|
||||
case FeedType.Author: return this._compileFilter("authors", feed as string[])
|
||||
case FeedType.DVM: return await this._compileDvms(feed as DVMItem[])
|
||||
case FeedType.ID: return this._compileFilter("ids", feed as string[])
|
||||
case FeedType.Intersection: return await this._compileIntersection(feed as Feed[])
|
||||
case FeedType.Kind: return this._compileFilter("kinds", feed as number[])
|
||||
case FeedType.List: return await this._compileLists(feed as ListItem[])
|
||||
case FeedType.Relay: return [{relays: feed as string[]}]
|
||||
case FeedType.Scope: return this._compileScopes(feed as Scope[])
|
||||
case FeedType.Since: return this._compileFilter("since", feed[0] as number)
|
||||
case FeedType.SinceAgo: return this._compileFilter("since", now() - (feed[0] as number))
|
||||
case FeedType.ATag: return this._compileFilter("#a", feed as string[])
|
||||
case FeedType.ETag: return this._compileFilter("#e", feed as string[])
|
||||
case FeedType.PTag: return this._compileFilter("#p", feed as string[])
|
||||
case FeedType.TTag: return this._compileFilter("#t", feed as string[])
|
||||
case FeedType.Until: return this._compileFilter("until", feed[0] as number)
|
||||
case FeedType.UntilAgo: return this._compileFilter("until", now() - (feed[0] as number))
|
||||
case FeedType.Union: return await this._compileUnion(feed as Feed[])
|
||||
case FeedType.WOT: return this._compileWot(feed[0] as number, feed[1] as number)
|
||||
default:
|
||||
throw new Error(`Unable to convert feed of type ${type} to filters`)
|
||||
}
|
||||
}
|
||||
|
||||
async _compileUnion(feeds: Feed[]): Promise<RequestItem> {
|
||||
const relays: string[] = []
|
||||
_compileAddresses(addresses: string[]) {
|
||||
return [{filters: getIdFilters(addresses)}]
|
||||
}
|
||||
|
||||
_compileFilter(key: string, value: any) {
|
||||
return [{filters: [{[key]: value} as Filter]}]
|
||||
}
|
||||
|
||||
_compileScopes(scopes: Scope[]) {
|
||||
return [{filters: [{authors: uniq(scopes.flatMap(this.options.getPubkeysForScope))}]}]
|
||||
}
|
||||
|
||||
_compileWot(min_wot: number, max_wot: number) {
|
||||
return [{filters: [{authors: this.options.getPubkeysForWotRange(min_wot, max_wot)}]}]
|
||||
}
|
||||
|
||||
async _compileDvms(items: DVMItem[]): Promise<RequestItem[]> {
|
||||
const filters: Filter[] = []
|
||||
|
||||
await Promise.all(
|
||||
feeds.map(async feed => {
|
||||
const item = await this.compile(feed)
|
||||
|
||||
for (const relay of item.relays) {
|
||||
relays.push(relay)
|
||||
}
|
||||
|
||||
for (const filter of item.filters) {
|
||||
filters.push(filter)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
return {
|
||||
relays: uniq(relays),
|
||||
filters: unionFilters(filters),
|
||||
}
|
||||
}
|
||||
|
||||
async _compileIntersection(feeds: Feed[]): Promise<RequestItem> {
|
||||
const items = await Promise.all(feeds.map(this.compile))
|
||||
const filters = intersectFilters(items.map(item => item.filters))
|
||||
|
||||
let relays = uniq(items.flatMap(item => item.relays))
|
||||
let hasRelays = relays.length > 0
|
||||
|
||||
items.forEach((item, i) => {
|
||||
if (item.relays.length > 0) {
|
||||
relays = relays.filter(relay => item.relays.includes(relay))
|
||||
}
|
||||
})
|
||||
|
||||
if (hasRelays && relays.length === 0) {
|
||||
relays.push(BOGUS_RELAY_URL)
|
||||
}
|
||||
|
||||
return {relays, filters}
|
||||
}
|
||||
|
||||
async _compileLists(addresses: string[]): Promise<RequestItem> {
|
||||
const events: E[] = []
|
||||
|
||||
await this.options.request({
|
||||
relays: [],
|
||||
filters: getIdFilters(addresses),
|
||||
onEvent: (e: E) => events.push(e),
|
||||
})
|
||||
|
||||
|
||||
return {
|
||||
relays: [],
|
||||
filters: this._getFiltersFromTags(Tags.fromEvents(events)),
|
||||
}
|
||||
}
|
||||
|
||||
async _compileDvms(requests: DVMItem[]): Promise<RequestItem> {
|
||||
const responseTags: Tags[] = []
|
||||
|
||||
await Promise.all(
|
||||
requests.map(request =>
|
||||
items.map(({mappings, ...request}) =>
|
||||
this.options.requestDvm({
|
||||
...request,
|
||||
onEvent: async (e: E) => {
|
||||
const tags = Tags.fromEvent(e)
|
||||
const {id, pubkey} = await tryCatch(() => JSON.parse(tags.get("request")?.value())) || {}
|
||||
const request = await tryCatch(() => JSON.parse(tags.get("request")?.value()))
|
||||
const responseTags = tags.rejectByValue([request?.id, request?.pubkey])
|
||||
|
||||
responseTags.push(tags.filterByKey(["t", "p", "e", "a"]).rejectByValue([id, pubkey]))
|
||||
for (const filter of await this._getFiltersFromTags(responseTags, mappings)) {
|
||||
filters.push(filter)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
const mergedTags = Tags.from(responseTags.flatMap(tags => tags.valueOf()))
|
||||
return [{filters: unionFilters(filters)}]
|
||||
}
|
||||
|
||||
return {
|
||||
relays: mergedTags.relays().valueOf(),
|
||||
filters: this._getFiltersFromTags(mergedTags),
|
||||
async _compileIntersection(feeds: Feed[]): Promise<RequestItem[]> {
|
||||
const [head, ...tail] = await Promise.all(feeds.map(this.compile))
|
||||
|
||||
const result = []
|
||||
|
||||
for (let {filters, relays} of head) {
|
||||
const matchingGroups = tail.map(
|
||||
items => items.filter(
|
||||
it => (
|
||||
(!relays || !it.relays || intersection(relays, it.relays).length > 0) &&
|
||||
(!filters || !it.filters || intersectFilters([filters, it.filters]).length > 0)
|
||||
)
|
||||
)
|
||||
).filter(
|
||||
items => items.length > 0
|
||||
)
|
||||
|
||||
if (matchingGroups.length < tail.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (const items of matchingGroups) {
|
||||
for (const item of items) {
|
||||
if (relays && item.relays) {
|
||||
relays = relays.filter(r => item.relays!.includes(r))
|
||||
} else if (item.relays) {
|
||||
relays = item.relays
|
||||
}
|
||||
|
||||
if (filters && item.filters) {
|
||||
filters = intersectFilters([filters, item.filters])
|
||||
} else if (item.filters) {
|
||||
filters = item.filters
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.push({relays, filters})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async _compileUnion(feeds: Feed[]): Promise<RequestItem[]> {
|
||||
const filtersByRelay = new Map<string, Filter[]>()
|
||||
const filtersWithoutRelay: Filter[] = []
|
||||
const relaysWithoutFilter: string[] = []
|
||||
|
||||
await Promise.all(
|
||||
feeds.map(async feed => {
|
||||
for (const item of await this.compile(feed)) {
|
||||
if (item.relays) {
|
||||
for (const relay of item.relays) {
|
||||
if (item.filters) {
|
||||
for (const filter of item.filters) {
|
||||
pushToMapKey(filtersByRelay, relay, filter)
|
||||
}
|
||||
} else {
|
||||
relaysWithoutFilter.push(relay)
|
||||
}
|
||||
}
|
||||
} else if (item.filters) {
|
||||
for (const filter of item.filters) {
|
||||
filtersWithoutRelay.push(filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const items: RequestItem[] = []
|
||||
|
||||
for (const [relay, filters] of filtersByRelay.entries()) {
|
||||
items.push({
|
||||
relays: [relay],
|
||||
filters: unionFilters(filters),
|
||||
})
|
||||
}
|
||||
|
||||
if (filtersWithoutRelay.length > 0) {
|
||||
items.push({filters: unionFilters(filtersWithoutRelay)})
|
||||
}
|
||||
|
||||
if (relaysWithoutFilter.length > 0) {
|
||||
items.push({relays: uniq(relaysWithoutFilter)})
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
async _compileLists(listItems: ListItem[]): Promise<RequestItem[]> {
|
||||
const addresses = uniq(listItems.map(({address}) => address))
|
||||
const eventsByAddress = new Map<string, E>()
|
||||
|
||||
await this.options.request({
|
||||
filters: getIdFilters(addresses),
|
||||
onEvent: (e: E) => eventsByAddress.set(getAddress(e), e),
|
||||
})
|
||||
|
||||
const filters = flatten(
|
||||
await Promise.all(
|
||||
listItems.map(({address, mappings}) => {
|
||||
const event = eventsByAddress.get(address)
|
||||
|
||||
return event ? this._getFiltersFromTags(Tags.fromEvent(event), mappings) : []
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return [{filters: unionFilters(filters)}]
|
||||
}
|
||||
|
||||
// Utilities
|
||||
|
||||
_compileFilter({scopes, min_wot, max_wot, until_ago, since_ago, ...filter}: DynamicFilter) {
|
||||
if (scopes && !filter.authors) {
|
||||
filter.authors = scopes.flatMap((scope: Scope) => this.options.getPubkeysForScope(scope))
|
||||
}
|
||||
async _getFiltersFromTags(tags: Tags, mappings: TagFilterMapping[]) {
|
||||
const filters = []
|
||||
|
||||
if ((!isNil(min_wot) || !isNil(max_wot))) {
|
||||
const authors = this.options.getPubkeysForWotRange(min_wot || 0, max_wot || 1)
|
||||
for (const [tagName, feedType] of mappings) {
|
||||
const filterTags = tags.whereKey(tagName)
|
||||
|
||||
if (filter.authors) {
|
||||
const authorsSet = new Set(authors)
|
||||
if (filterTags.exists()) {
|
||||
let values: string[] | number[] = filterTags.values().valueOf()
|
||||
|
||||
filter.authors = filter.authors.filter(pubkey => authorsSet.has(pubkey))
|
||||
} else {
|
||||
filter.authors = authors
|
||||
if (feedType === FeedType.Kind) {
|
||||
values = values.map(ensureNumber) as number[]
|
||||
}
|
||||
|
||||
for (const item of await this.compile([feedType, ...values] as Feed)) {
|
||||
for (const filter of item.filters || []) {
|
||||
filters.push(filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!isNil(until_ago)) {
|
||||
filter.until = now() - until_ago!
|
||||
}
|
||||
|
||||
if (!isNil(since_ago)) {
|
||||
filter.since = now() - since_ago!
|
||||
}
|
||||
|
||||
return filter as Filter
|
||||
}
|
||||
|
||||
_getFiltersFromTags(tags: Tags) {
|
||||
const ttags = tags.values("t")
|
||||
const ptags = tags.values("p")
|
||||
const eatags = tags.filterByKey(["e", "a"]).values()
|
||||
const filters: Filter[] = []
|
||||
|
||||
if (ttags.exists()) {
|
||||
filters.push({"#t": ttags.valueOf()})
|
||||
}
|
||||
|
||||
if (ptags.exists()) {
|
||||
filters.push({authors: ptags.valueOf()})
|
||||
}
|
||||
|
||||
if (eatags.exists()) {
|
||||
for (const filter of getIdFilters(eatags.valueOf())) {
|
||||
filters.push(filter)
|
||||
}
|
||||
}
|
||||
|
||||
// If we don't have any filters, return nothing instead of everything
|
||||
if (filters.length === 0) {
|
||||
filters.push({authors: []})
|
||||
}
|
||||
|
||||
return filters
|
||||
return unionFilters(filters)
|
||||
}
|
||||
}
|
||||
|
||||
+142
-57
@@ -1,14 +1,27 @@
|
||||
import type {Filter} from '@welshman/util'
|
||||
|
||||
export enum FeedType {
|
||||
Difference = "difference",
|
||||
Intersection = "intersection",
|
||||
SymmetricDifference = "symdiff",
|
||||
Union = "union",
|
||||
Filter = "filter",
|
||||
Relay = "relay",
|
||||
List = "list",
|
||||
Address = "address",
|
||||
Author = "author",
|
||||
DVM = "dvm",
|
||||
Difference = "difference",
|
||||
ID = "id",
|
||||
Intersection = "intersection",
|
||||
Kind = "kind",
|
||||
List = "list",
|
||||
WOT = "wot",
|
||||
Relay = "relay",
|
||||
Scope = "scope",
|
||||
Since = "since",
|
||||
SinceAgo = "since_ago",
|
||||
SymmetricDifference = "symmetric_difference",
|
||||
ATag = "#a",
|
||||
ETag = "#e",
|
||||
PTag = "#p",
|
||||
TTag = "#t",
|
||||
Union = "union",
|
||||
Until = "until",
|
||||
UntilAgo = "until_ago",
|
||||
}
|
||||
|
||||
export enum Scope {
|
||||
@@ -18,82 +31,154 @@ export enum Scope {
|
||||
Self = "self",
|
||||
}
|
||||
|
||||
export type DynamicFilter = Filter & {
|
||||
scopes?: Scope[]
|
||||
min_wot?: number
|
||||
max_wot?: number
|
||||
until_ago?: number
|
||||
since_ago?: number
|
||||
export type TagFeedType =
|
||||
FeedType.ATag |
|
||||
FeedType.ETag |
|
||||
FeedType.PTag |
|
||||
FeedType.TTag
|
||||
|
||||
|
||||
export type FilterFeedType =
|
||||
FeedType.ID |
|
||||
FeedType.Address |
|
||||
FeedType.Author |
|
||||
FeedType.Kind |
|
||||
FeedType.Relay |
|
||||
TagFeedType
|
||||
|
||||
export type TagFilterMapping = [string, FilterFeedType]
|
||||
|
||||
export type DVMItem = {
|
||||
kind: number,
|
||||
mappings: TagFilterMapping[],
|
||||
tags?: string[][],
|
||||
relays?: string[],
|
||||
}
|
||||
|
||||
export type RelayFeed = [FeedType.Relay, string[], ...Feed[]]
|
||||
export type DifferenceFeed = [FeedType.Difference, ...Feed[]]
|
||||
export type IntersectionFeed = [FeedType.Intersection, ...Feed[]]
|
||||
export type SymmetricDifferenceFeed = [FeedType.SymmetricDifference, ...Feed[]]
|
||||
export type UnionFeed = [FeedType.Union, ...Feed[]]
|
||||
export type FilterFeed = [FeedType.Filter, ...DynamicFilter[]]
|
||||
export type ListFeed = [FeedType.List, ...string[]]
|
||||
export type DVMFeed = [FeedType.DVM, ...DVMItem[]]
|
||||
export type ListItem = {
|
||||
address: string,
|
||||
mappings: TagFilterMapping[],
|
||||
}
|
||||
|
||||
export type WOTItem = {
|
||||
min: number,
|
||||
max: number,
|
||||
}
|
||||
|
||||
export type AddressFeed = [type: FeedType.Address, ...addresses: string[]]
|
||||
export type AuthorFeed = [type: FeedType.Author, ...pubkeys: string[]]
|
||||
export type DVMFeed = [type: FeedType.DVM, ...items: DVMItem[]]
|
||||
export type DifferenceFeed = [type: FeedType.Difference, ...feeds: Feed[]]
|
||||
export type IDFeed = [type: FeedType.ID, ...ids: string[]]
|
||||
export type IntersectionFeed = [type: FeedType.Intersection, ...feeds: Feed[]]
|
||||
export type KindFeed = [type: FeedType.Kind, ...kinds: number[]]
|
||||
export type ListFeed = [type: FeedType.List, ...items: ListItem[]]
|
||||
export type WOTFeed = [type: FeedType.WOT, ...items: WOTItem[]]
|
||||
export type RelayFeed = [type: FeedType.Relay, ...urls: string[]]
|
||||
export type ScopeFeed = [type: FeedType.Scope, ...scopes: Scope[]]
|
||||
export type SinceAgoFeed = [type: FeedType.SinceAgo, since_ago: number]
|
||||
export type SinceFeed = [type: FeedType.Since, since: number]
|
||||
export type SymmetricDifferenceFeed = [type: FeedType.SymmetricDifference, ...feeds: Feed[]]
|
||||
export type ATagFeed = [type: FeedType.ATag, ...addresses: string[]]
|
||||
export type ETagFeed = [type: FeedType.ETag, ...ids: string[]]
|
||||
export type PTagFeed = [type: FeedType.PTag, ...pubkeys: string[]]
|
||||
export type TTagFeed = [type: FeedType.TTag, ...topics: string[]]
|
||||
export type UnionFeed = [type: FeedType.Union, ...feeds: Feed[]]
|
||||
export type UntilAgoFeed = [type: FeedType.UntilAgo, until_ago: number]
|
||||
export type UntilFeed = [type: FeedType.Until, until: number]
|
||||
|
||||
export type Feed =
|
||||
RelayFeed |
|
||||
AddressFeed |
|
||||
AuthorFeed |
|
||||
DVMFeed |
|
||||
DifferenceFeed |
|
||||
IDFeed |
|
||||
IntersectionFeed |
|
||||
SymmetricDifferenceFeed |
|
||||
UnionFeed |
|
||||
FilterFeed |
|
||||
RelayFeed |
|
||||
KindFeed |
|
||||
ListFeed |
|
||||
DVMFeed
|
||||
WOTFeed |
|
||||
RelayFeed |
|
||||
ScopeFeed |
|
||||
SinceAgoFeed |
|
||||
SinceFeed |
|
||||
SymmetricDifferenceFeed |
|
||||
ATagFeed |
|
||||
ETagFeed |
|
||||
PTagFeed |
|
||||
TTagFeed |
|
||||
UnionFeed |
|
||||
UntilAgoFeed |
|
||||
UntilFeed
|
||||
|
||||
export const relayFeed = (relays: string[], ...feeds: Feed[]) => [FeedType.Relay, relays, ...feeds] as Feed
|
||||
export const differenceFeed = (...feeds: Feed[]) => [FeedType.Difference, ...feeds] as Feed
|
||||
export const intersectionFeed = (...feeds: Feed[]) => [FeedType.Intersection, ...feeds] as Feed
|
||||
export const symmetricDifferenceFeed = (...feeds: Feed[]) => [FeedType.SymmetricDifference, ...feeds] as Feed
|
||||
export const unionFeed = (...feeds: Feed[]) => [FeedType.Union, ...feeds] as Feed
|
||||
export const filterFeed = (...filters: DynamicFilter[]) => [FeedType.Filter, ...filters] as Feed
|
||||
export const listFeed = (...addresses: string[]) => [FeedType.List, ...addresses] as Feed
|
||||
export const dvmFeed = (...requests: DVMItem[]) => [FeedType.DVM, ...requests] as Feed
|
||||
export const addressFeed = (...addresses: string[]): AddressFeed => [FeedType.Address, ...addresses]
|
||||
export const authorFeed = (...pubkeys: string[]): AuthorFeed => [FeedType.Author, ...pubkeys]
|
||||
export const dvmFeed = (...items: DVMItem[]): DVMFeed => [FeedType.DVM, ...items]
|
||||
export const differenceFeed = (...feeds: Feed[]): DifferenceFeed => [FeedType.Difference, ...feeds]
|
||||
export const idFeed = (...ids: string[]): IDFeed => [FeedType.ID, ...ids]
|
||||
export const intersectionFeed = (...feeds: Feed[]): IntersectionFeed => [FeedType.Intersection, ...feeds]
|
||||
export const kindFeed = (...kinds: number[]): KindFeed => [FeedType.Kind, ...kinds]
|
||||
export const listFeed = (...items: ListItem[]): ListFeed => [FeedType.List, ...items]
|
||||
export const wotFeed = (...items: WOTItem[]): WOTFeed => [FeedType.WOT, ...items]
|
||||
export const relayFeed = (...urls: string[]): RelayFeed => [FeedType.Relay, ...urls]
|
||||
export const scopeFeed = (...scopes: Scope[]): ScopeFeed => [FeedType.Scope, ...scopes]
|
||||
export const sinceAgoFeed = (since_ago: number): SinceAgoFeed => [FeedType.SinceAgo, since_ago]
|
||||
export const sinceFeed = (since: number): SinceFeed => [FeedType.Since, since]
|
||||
export const symmetricDifferenceFeed = (...feeds: Feed[]): SymmetricDifferenceFeed => [FeedType.SymmetricDifference, ...feeds]
|
||||
export const aTagFeed = (...values: string[]): ATagFeed => [FeedType.ATag, ...values]
|
||||
export const eTagFeed = (...values: string[]): ETagFeed => [FeedType.ETag, ...values]
|
||||
export const pTagFeed = (...values: string[]): PTagFeed => [FeedType.PTag, ...values]
|
||||
export const tTagFeed = (...values: string[]): TTagFeed => [FeedType.TTag, ...values]
|
||||
export const unionFeed = (...feeds: Feed[]): UnionFeed => [FeedType.Union, ...feeds]
|
||||
export const untilAgoFeed = (until_ago: number): UntilAgoFeed => [FeedType.UntilAgo, until_ago]
|
||||
export const untilFeed = (until: number): UntilFeed => [FeedType.Until, until]
|
||||
|
||||
export const hasSubFeeds = ([type]: Feed) =>
|
||||
export const feedsFromFilter = (filter: Filter) => {
|
||||
const feeds = []
|
||||
|
||||
for (const [k, v] of Object.entries(filter)) {
|
||||
if (k === 'ids') feeds.push(idFeed(...v as string[]))
|
||||
else if (k === 'kinds') feeds.push(kindFeed(...v as number[]))
|
||||
else if (k === 'authors') feeds.push(authorFeed(...v as string[]))
|
||||
else if (k === 'since') feeds.push(sinceFeed(v as number))
|
||||
else if (k === 'until') feeds.push(untilFeed(v as number))
|
||||
else if (k === "#a") feeds.push(aTagFeed(...v as string[]))
|
||||
else if (k === "#e") feeds.push(eTagFeed(...v as string[]))
|
||||
else if (k === "#p") feeds.push(pTagFeed(...v as string[]))
|
||||
else if (k === "#t") feeds.push(tTagFeed(...v as string[]))
|
||||
else throw new Error(`Unable to create feed from filter ${k}: ${v}`)
|
||||
}
|
||||
|
||||
return feeds
|
||||
}
|
||||
|
||||
export const feedFromFilter = (filter: Filter) => intersectionFeed(...feedsFromFilter(filter))
|
||||
|
||||
export const hasSubFeeds = ([type]: [FeedType]) =>
|
||||
[
|
||||
FeedType.Union,
|
||||
FeedType.Intersection,
|
||||
FeedType.Difference,
|
||||
FeedType.SymmetricDifference,
|
||||
FeedType.Relay,
|
||||
].includes(type)
|
||||
|
||||
export const getSubFeeds = ([type, ...feed]: Feed): Feed[] => {
|
||||
switch (type) {
|
||||
case FeedType.Relay:
|
||||
return feed.slice(1) as Feed[]
|
||||
case FeedType.Difference:
|
||||
case FeedType.Intersection:
|
||||
case FeedType.SymmetricDifference:
|
||||
case FeedType.Union:
|
||||
return feed as Feed[]
|
||||
default:
|
||||
return []
|
||||
}
|
||||
}
|
||||
export const getSubFeeds = ([type, ...feeds]: Feed): Feed[] => hasSubFeeds([type]) ? feeds as Feed[] : []
|
||||
|
||||
export type RequestItem = {
|
||||
relays: string[]
|
||||
filters: Filter[]
|
||||
relays?: string[]
|
||||
filters?: Filter[]
|
||||
}
|
||||
|
||||
export type RequestOpts<E> = RequestItem & {
|
||||
onEvent: (event: E) => void
|
||||
}
|
||||
|
||||
export type DVMItem = {
|
||||
kind: number
|
||||
tags?: string[][]
|
||||
relays?: string[]
|
||||
export type DVMRequest = {
|
||||
kind: number,
|
||||
tags?: string[][],
|
||||
relays?: string[],
|
||||
}
|
||||
|
||||
export type DVMOpts<E> = DVMItem & {
|
||||
export type DVMOpts<E> = DVMRequest & {
|
||||
onEvent: (event: E) => void
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ export class FeedLoader<E extends Rumor> {
|
||||
|
||||
async getLoader([type, ...feed]: Feed, loadOpts: LoadOpts<E>) {
|
||||
if (this.compiler.canCompile([type, ...feed] as Feed)) {
|
||||
return this._getRequestLoader(await this.compiler.compile([type, ...feed] as Feed), loadOpts)
|
||||
return this._getRequestsLoader(await this.compiler.compile([type, ...feed] as Feed), loadOpts)
|
||||
}
|
||||
|
||||
switch(type) {
|
||||
@@ -38,9 +38,35 @@ export class FeedLoader<E extends Rumor> {
|
||||
}
|
||||
}
|
||||
|
||||
async _getRequestsLoader(requests: RequestItem[], {onEvent, onExhausted}: LoadOpts<E>) {
|
||||
const seen = new Set()
|
||||
const exhausted = new Set()
|
||||
const loaders = await Promise.all(
|
||||
requests.map(
|
||||
request => this._getRequestLoader(request, {
|
||||
onExhausted: () => exhausted.add(request),
|
||||
onEvent: e => {
|
||||
if (!seen.has(e.id)) {
|
||||
onEvent?.(e)
|
||||
seen.add(e.id)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return async (limit: number) => {
|
||||
await Promise.all(loaders.map(loader => loader(limit)))
|
||||
|
||||
if (exhausted.size === requests.length) {
|
||||
onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _getRequestLoader({relays, filters}: RequestItem, {onEvent, onExhausted}: LoadOpts<E>) {
|
||||
// Make sure we have some kind of filter to send if we've been given an empty one, as happens with relay feeds
|
||||
if (filters.length === 0) {
|
||||
if (!filters || filters.length === 0) {
|
||||
filters = [{}]
|
||||
}
|
||||
|
||||
@@ -55,7 +81,7 @@ export class FeedLoader<E extends Rumor> {
|
||||
let until = maxUntil
|
||||
|
||||
return async (limit: number) => {
|
||||
const requestFilters = filters
|
||||
const requestFilters = filters!
|
||||
// Remove filters that don't fit our window
|
||||
.filter((filter: Filter) => {
|
||||
const filterSince = filter.since || EPOCH
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"targets": [
|
||||
{"extname": ".cjs", "module": "commonjs"},
|
||||
{"extname": ".mjs", "module": "esnext", "moduleResolution": "node"}
|
||||
],
|
||||
"projects": ["tsconfig.json"]
|
||||
|
||||
+17
-1
@@ -73,6 +73,20 @@ export const concat = <T>(...xs: (T | Nil)[][]) => xs.flatMap(x => x || [])
|
||||
|
||||
export const append = <T>(xs: (T | Nil)[], x: T) => concat(xs, [x])
|
||||
|
||||
export const union = <T>(a: T[], b: T[]) => uniq([...a, ...b])
|
||||
|
||||
export const intersection = <T>(a: T[], b: T[]) => {
|
||||
const s = new Set(b)
|
||||
|
||||
return a.filter(x => s.has(x))
|
||||
}
|
||||
|
||||
export const difference = <T>(a: T[], b: T[]) => {
|
||||
const s = new Set(b)
|
||||
|
||||
return a.filter(x => !s.has(x))
|
||||
}
|
||||
|
||||
export const clamp = ([min, max]: [number, number], n: number) => Math.min(max, Math.max(min, n))
|
||||
|
||||
export const tryCatch = async <T>(f: () => Promise<T | void> | T | void, onError?: (e: Error) => void): Promise<T | void> => {
|
||||
@@ -114,7 +128,9 @@ export const toIterable = (x: any) => isIterable(x) ? x : [x]
|
||||
|
||||
export const ensurePlural = <T>(x: T | T[]) => (x instanceof Array ? x : [x])
|
||||
|
||||
export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
|
||||
export const ensureNumber = (x: number | string) => parseFloat(x as string)
|
||||
|
||||
export const flatten = <T>(xs: T[][]) => xs.flatMap(identity)
|
||||
|
||||
export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user