Put everything in src directories
This commit is contained in:
Jon Staab
@@ -0,0 +1,279 @@
|
||||
import {uniq, identity, flatten, pushToMapKey, intersection, tryCatch, now} from '@welshman/lib'
|
||||
import type {TrustedEvent, Filter} from '@welshman/util'
|
||||
import {Tags, intersectFilters, matchFilter, getAddress, getIdFilters, unionFilters} from '@welshman/util'
|
||||
import type {CreatedAtItem, RequestItem, ListItem, LabelItem, WOTItem, DVMItem, Scope, Feed, FeedOptions} from './core'
|
||||
import {getFeedArgs, feedsFromTags} from './utils'
|
||||
import {FeedType} from './core'
|
||||
|
||||
export class FeedCompiler<E extends TrustedEvent> {
|
||||
constructor(readonly options: FeedOptions<E>) {}
|
||||
|
||||
canCompile(feed: Feed): boolean {
|
||||
switch(feed[0]) {
|
||||
case FeedType.Union:
|
||||
case FeedType.Intersection:
|
||||
return getFeedArgs(feed).every(f => this.canCompile(f))
|
||||
case FeedType.Address:
|
||||
case FeedType.Author:
|
||||
case FeedType.CreatedAt:
|
||||
case FeedType.DVM:
|
||||
case FeedType.ID:
|
||||
case FeedType.Kind:
|
||||
case FeedType.List:
|
||||
case FeedType.Label:
|
||||
case FeedType.Relay:
|
||||
case FeedType.Scope:
|
||||
case FeedType.Search:
|
||||
case FeedType.Tag:
|
||||
case FeedType.WOT:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async compile(feed: Feed): Promise<RequestItem[]> {
|
||||
switch(feed[0]) {
|
||||
case FeedType.ID: return this._compileFilter('ids', getFeedArgs(feed))
|
||||
case FeedType.Kind: return this._compileFilter('kinds', getFeedArgs(feed))
|
||||
case FeedType.Author: return this._compileFilter('authors', getFeedArgs(feed))
|
||||
case FeedType.DVM: return await this._compileDvms(getFeedArgs(feed))
|
||||
case FeedType.Intersection: return await this._compileIntersection(getFeedArgs(feed))
|
||||
case FeedType.List: return await this._compileLists(getFeedArgs(feed))
|
||||
case FeedType.Label: return await this._compileLabels(getFeedArgs(feed))
|
||||
case FeedType.Union: return await this._compileUnion(getFeedArgs(feed))
|
||||
case FeedType.Address: return this._compileAddresses(getFeedArgs(feed))
|
||||
case FeedType.CreatedAt: return this._compileCreatedAt(getFeedArgs(feed))
|
||||
case FeedType.Scope: return this._compileScopes(getFeedArgs(feed))
|
||||
case FeedType.Search: return this._compileSearches(getFeedArgs(feed))
|
||||
case FeedType.WOT: return this._compileWot(getFeedArgs(feed))
|
||||
case FeedType.Relay: return [{relays: getFeedArgs(feed)}]
|
||||
case FeedType.Tag: {
|
||||
const [key, ...value] = getFeedArgs(feed)
|
||||
|
||||
return this._compileFilter(key, value)
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unable to convert feed of type ${feed[0]} to filters`)
|
||||
}
|
||||
}
|
||||
|
||||
_compileAddresses(addresses: string[]) {
|
||||
return [{filters: getIdFilters(addresses)}]
|
||||
}
|
||||
|
||||
_compileFilter(key: string, value: any) {
|
||||
return [{filters: [{[key]: value} as Filter]}]
|
||||
}
|
||||
|
||||
_compileCreatedAt(items: CreatedAtItem[]) {
|
||||
const filters = items
|
||||
.map(({since, until, relative = []}) => {
|
||||
if (since && relative.includes("since")) {
|
||||
since = now() - since
|
||||
}
|
||||
|
||||
if (until && relative.includes("until")) {
|
||||
until = now() - until
|
||||
}
|
||||
|
||||
if (since && until) return {since, until}
|
||||
if (since) return {since}
|
||||
if (until) return {until}
|
||||
|
||||
return null
|
||||
})
|
||||
.filter(identity)
|
||||
|
||||
return [{filters: filters as Filter[]}]
|
||||
}
|
||||
|
||||
_compileScopes(scopes: Scope[]) {
|
||||
return [{filters: [{authors: uniq(scopes.flatMap(this.options.getPubkeysForScope))}]}]
|
||||
}
|
||||
|
||||
_compileSearches(searches: string[]) {
|
||||
return [{filters: searches.map(search => ({search}))}]
|
||||
}
|
||||
|
||||
_compileWot(wotItems: WOTItem[]) {
|
||||
return [{filters: wotItems.map(({min = 0, max = 1}) => ({authors: this.options.getPubkeysForWOTRange(min, max)}))}]
|
||||
}
|
||||
|
||||
async _compileDvms(items: DVMItem[]): Promise<RequestItem[]> {
|
||||
const feeds: Feed[] = []
|
||||
|
||||
await Promise.all(
|
||||
items.map(({mappings, ...request}) =>
|
||||
this.options.requestDVM({
|
||||
...request,
|
||||
onEvent: async (e: E) => {
|
||||
const tags = Tags.wrap(await tryCatch(() => JSON.parse(e.content)) || [])
|
||||
|
||||
for (const feed of feedsFromTags(tags, mappings)) {
|
||||
feeds.push(feed)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return await this._compileUnion(feeds)
|
||||
}
|
||||
|
||||
async _compileIntersection(feeds: Feed[]): Promise<RequestItem[]> {
|
||||
const [head, ...tail] = await Promise.all(feeds.map(f => this.compile(f)))
|
||||
|
||||
const result = []
|
||||
|
||||
for (let {filters, relays} of head || []) {
|
||||
const matchingGroups = tail.map(
|
||||
items => items.filter(
|
||||
it => (
|
||||
(!relays || !it.relays || intersection(relays, it.relays).length > 0) &&
|
||||
(!filters || !it.filters || intersectFilters([filters, it.filters]).length > 0)
|
||||
)
|
||||
)
|
||||
).filter(
|
||||
items => items.length > 0
|
||||
)
|
||||
|
||||
if (matchingGroups.length < tail.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (const items of matchingGroups) {
|
||||
for (const item of items) {
|
||||
if (relays && item.relays) {
|
||||
relays = relays.filter(r => item.relays!.includes(r))
|
||||
} else if (item.relays) {
|
||||
relays = item.relays
|
||||
}
|
||||
|
||||
if (filters && item.filters) {
|
||||
filters = intersectFilters([filters, item.filters])
|
||||
} else if (item.filters) {
|
||||
filters = item.filters
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.push({relays, filters})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
async _compileUnion(feeds: Feed[]): Promise<RequestItem[]> {
|
||||
const filtersByRelay = new Map<string, Filter[]>()
|
||||
const filtersWithoutRelay: Filter[] = []
|
||||
const relaysWithoutFilter: string[] = []
|
||||
|
||||
await Promise.all(
|
||||
feeds.map(async feed => {
|
||||
for (const item of await this.compile(feed)) {
|
||||
if (item.relays) {
|
||||
for (const relay of item.relays) {
|
||||
if (item.filters) {
|
||||
for (const filter of item.filters) {
|
||||
pushToMapKey(filtersByRelay, relay, filter)
|
||||
}
|
||||
} else {
|
||||
relaysWithoutFilter.push(relay)
|
||||
}
|
||||
}
|
||||
} else if (item.filters) {
|
||||
for (const filter of item.filters) {
|
||||
filtersWithoutRelay.push(filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const items: RequestItem[] = []
|
||||
|
||||
for (const [relay, filters] of filtersByRelay.entries()) {
|
||||
items.push({
|
||||
relays: [relay],
|
||||
filters: unionFilters(filters),
|
||||
})
|
||||
}
|
||||
|
||||
if (filtersWithoutRelay.length > 0) {
|
||||
items.push({filters: unionFilters(filtersWithoutRelay)})
|
||||
}
|
||||
|
||||
if (relaysWithoutFilter.length > 0) {
|
||||
items.push({relays: uniq(relaysWithoutFilter)})
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
async _compileLists(listItems: ListItem[]): Promise<RequestItem[]> {
|
||||
const addresses = uniq(listItems.flatMap(({addresses}) => addresses))
|
||||
const eventsByAddress = new Map<string, E>()
|
||||
|
||||
await this.options.request({
|
||||
filters: getIdFilters(addresses),
|
||||
onEvent: (e: E) => eventsByAddress.set(getAddress(e), e),
|
||||
})
|
||||
|
||||
const feeds = flatten(
|
||||
await Promise.all(
|
||||
listItems.map(({addresses, mappings}) => {
|
||||
const feeds: Feed[] = []
|
||||
|
||||
for (const address of addresses) {
|
||||
const event = eventsByAddress.get(address)
|
||||
|
||||
if (event) {
|
||||
for (const feed of feedsFromTags(Tags.fromEvent(event), mappings)) {
|
||||
feeds.push(feed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return feeds
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return this._compileUnion(feeds)
|
||||
}
|
||||
|
||||
async _compileLabels(labelItems: LabelItem[]): Promise<RequestItem[]> {
|
||||
const events: E[] = []
|
||||
|
||||
await Promise.all(
|
||||
labelItems.map(({mappings, relays, ...filter}) =>
|
||||
this.options.request({
|
||||
relays,
|
||||
filters: [{kinds: [1985], ...filter}],
|
||||
onEvent: (e: E) => events.push(e),
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
const feeds = flatten(
|
||||
await Promise.all(
|
||||
labelItems.map(({mappings, relays, ...filter}) => {
|
||||
const tags: string[][] = []
|
||||
|
||||
for (const event of events) {
|
||||
if (matchFilter(filter, event)) {
|
||||
for (const tag of event.tags) {
|
||||
tags.push(tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return feedsFromTags(Tags.wrap(tags), mappings)
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return this._compileUnion(feeds)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type {Filter} from '@welshman/util'
|
||||
|
||||
export enum FeedType {
|
||||
Address = "address",
|
||||
Author = "author",
|
||||
CreatedAt = "created_at",
|
||||
DVM = "dvm",
|
||||
Difference = "difference",
|
||||
ID = "id",
|
||||
Intersection = "intersection",
|
||||
Kind = "kind",
|
||||
List = "list",
|
||||
Label = "label",
|
||||
WOT = "wot",
|
||||
Relay = "relay",
|
||||
Scope = "scope",
|
||||
Search = "search",
|
||||
Tag = "tag",
|
||||
Union = "union",
|
||||
}
|
||||
|
||||
export enum Scope {
|
||||
Followers = "followers",
|
||||
Follows = "follows",
|
||||
Network = "network",
|
||||
Self = "self",
|
||||
}
|
||||
|
||||
export type FilterFeedType =
|
||||
FeedType.ID |
|
||||
FeedType.Address |
|
||||
FeedType.Author |
|
||||
FeedType.Kind |
|
||||
FeedType.Relay |
|
||||
FeedType.Tag
|
||||
|
||||
export type TagFeedMapping = [string, Feed]
|
||||
|
||||
export type DVMItem = {
|
||||
kind: number,
|
||||
tags?: string[][],
|
||||
relays?: string[],
|
||||
mappings?: TagFeedMapping[],
|
||||
}
|
||||
|
||||
export type ListItem = {
|
||||
addresses: string[],
|
||||
mappings?: TagFeedMapping[],
|
||||
}
|
||||
|
||||
export type LabelItem = {
|
||||
relays?: string[],
|
||||
authors?: string[]
|
||||
[key: `#${string}`]: string[]
|
||||
mappings?: TagFeedMapping[],
|
||||
}
|
||||
|
||||
export type WOTItem = {
|
||||
min?: number,
|
||||
max?: number,
|
||||
}
|
||||
|
||||
export type CreatedAtItem = {
|
||||
since?: number,
|
||||
until?: number,
|
||||
relative?: string[],
|
||||
}
|
||||
|
||||
export type AddressFeed = [type: FeedType.Address, ...addresses: string[]]
|
||||
export type AuthorFeed = [type: FeedType.Author, ...pubkeys: string[]]
|
||||
export type CreatedAtFeed = [type: FeedType.CreatedAt, ...items: CreatedAtItem[]]
|
||||
export type DVMFeed = [type: FeedType.DVM, ...items: DVMItem[]]
|
||||
export type DifferenceFeed = [type: FeedType.Difference, ...feeds: Feed[]]
|
||||
export type IDFeed = [type: FeedType.ID, ...ids: string[]]
|
||||
export type IntersectionFeed = [type: FeedType.Intersection, ...feeds: Feed[]]
|
||||
export type KindFeed = [type: FeedType.Kind, ...kinds: number[]]
|
||||
export type ListFeed = [type: FeedType.List, ...items: ListItem[]]
|
||||
export type LabelFeed = [type: FeedType.Label, ...items: LabelItem[]]
|
||||
export type WOTFeed = [type: FeedType.WOT, ...items: WOTItem[]]
|
||||
export type RelayFeed = [type: FeedType.Relay, ...urls: string[]]
|
||||
export type ScopeFeed = [type: FeedType.Scope, ...scopes: Scope[]]
|
||||
export type SearchFeed = [type: FeedType.Search, ...searches: string[]]
|
||||
export type TagFeed = [type: FeedType.Tag, key: string, ...values: string[]]
|
||||
export type UnionFeed = [type: FeedType.Union, ...feeds: Feed[]]
|
||||
|
||||
export type Feed =
|
||||
AddressFeed |
|
||||
AuthorFeed |
|
||||
CreatedAtFeed |
|
||||
DVMFeed |
|
||||
DifferenceFeed |
|
||||
IDFeed |
|
||||
IntersectionFeed |
|
||||
KindFeed |
|
||||
ListFeed |
|
||||
LabelFeed |
|
||||
WOTFeed |
|
||||
RelayFeed |
|
||||
ScopeFeed |
|
||||
SearchFeed |
|
||||
TagFeed |
|
||||
UnionFeed
|
||||
|
||||
export type RequestItem = {
|
||||
relays?: string[]
|
||||
filters?: Filter[]
|
||||
}
|
||||
|
||||
export type RequestOpts<E> = RequestItem & {
|
||||
onEvent: (event: E) => void
|
||||
}
|
||||
|
||||
export type DVMRequest = {
|
||||
kind: number,
|
||||
tags?: string[][],
|
||||
relays?: string[],
|
||||
}
|
||||
|
||||
export type DVMOpts<E> = DVMRequest & {
|
||||
onEvent: (event: E) => void
|
||||
}
|
||||
|
||||
export type FeedOptions<E> = {
|
||||
request: (opts: RequestOpts<E>) => Promise<void>
|
||||
requestDVM: (opts: DVMOpts<E>) => Promise<void>
|
||||
getPubkeysForScope: (scope: Scope) => string[]
|
||||
getPubkeysForWOTRange: (minWOT: number, maxWOT: number) => string[]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from './core'
|
||||
export * from './compiler'
|
||||
export * from './loader'
|
||||
export * from './utils'
|
||||
@@ -0,0 +1,251 @@
|
||||
import {inc, max, min, now} from '@welshman/lib'
|
||||
import type {TrustedEvent, Filter} from '@welshman/util'
|
||||
import {EPOCH, trimFilters, guessFilterDelta} from '@welshman/util'
|
||||
import type {Feed, RequestItem, FeedOptions} from './core'
|
||||
import {FeedType} from './core'
|
||||
import {FeedCompiler} from './compiler'
|
||||
|
||||
export type LoadOpts<E> = {
|
||||
onEvent?: (event: E) => void
|
||||
onExhausted?: () => void
|
||||
useWindowing?: boolean
|
||||
}
|
||||
|
||||
export type Loader = (limit: number) => Promise<void>
|
||||
|
||||
export class FeedLoader<E extends TrustedEvent> {
|
||||
compiler: FeedCompiler<E>
|
||||
|
||||
constructor(readonly options: FeedOptions<E>) {
|
||||
this.compiler = new FeedCompiler(options)
|
||||
}
|
||||
|
||||
async getLoader([type, ...feed]: Feed, loadOpts: LoadOpts<E>) {
|
||||
if (this.compiler.canCompile([type, ...feed] as Feed)) {
|
||||
return this.getRequestsLoader(await this.compiler.compile([type, ...feed] as Feed), loadOpts)
|
||||
}
|
||||
|
||||
switch(type) {
|
||||
case FeedType.Difference:
|
||||
return this._getDifferenceLoader(feed as Feed[], loadOpts)
|
||||
case FeedType.Intersection:
|
||||
return this._getIntersectionLoader(feed as Feed[], loadOpts)
|
||||
case FeedType.Union:
|
||||
return this._getUnionLoader(feed as Feed[], loadOpts)
|
||||
default:
|
||||
throw new Error(`Unable to convert feed of type ${type} to loader`)
|
||||
}
|
||||
}
|
||||
|
||||
async getRequestsLoader(requests: RequestItem[], loadOpts: LoadOpts<E>) {
|
||||
const seen = new Set()
|
||||
const exhausted = new Set()
|
||||
const loaders = await Promise.all(
|
||||
requests.map(
|
||||
request => this._getRequestLoader(request, {
|
||||
...loadOpts,
|
||||
onExhausted: () => exhausted.add(request),
|
||||
onEvent: e => {
|
||||
if (!seen.has(e.id)) {
|
||||
loadOpts.onEvent?.(e)
|
||||
seen.add(e.id)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return async (limit: number) => {
|
||||
await Promise.all(loaders.map(loader => loader(limit)))
|
||||
|
||||
if (exhausted.size === requests.length) {
|
||||
loadOpts.onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _getRequestLoader({relays, filters}: RequestItem, {useWindowing = true, onEvent, onExhausted}: LoadOpts<E>) {
|
||||
// Make sure we have some kind of filter to send if we've been given an empty one, as happens with relay feeds
|
||||
if (!filters || filters.length === 0) {
|
||||
filters = [{}]
|
||||
}
|
||||
|
||||
const untils = filters.flatMap((filter: Filter) => filter.until ? [filter.until] : [])
|
||||
const sinces = filters.flatMap((filter: Filter) => filter.since ? [filter.since] : [])
|
||||
const maxUntil = untils.length === filters.length ? max(untils) : now()
|
||||
const minSince = sinces.length === filters.length ? min(sinces) : EPOCH
|
||||
const initialDelta = guessFilterDelta(filters)
|
||||
|
||||
let delta = initialDelta
|
||||
let since = useWindowing ? maxUntil - delta : minSince
|
||||
let until = maxUntil
|
||||
|
||||
return async (limit: number) => {
|
||||
const requestFilters = filters!
|
||||
// Remove filters that don't fit our window
|
||||
.filter((filter: Filter) => {
|
||||
const filterSince = filter.since || minSince
|
||||
const filterUntil = filter.until || maxUntil
|
||||
|
||||
return filterSince < until && filterUntil > since
|
||||
})
|
||||
// Modify the filters to define our window
|
||||
.map((filter: Filter) => ({...filter, until, limit, since}))
|
||||
|
||||
if (requestFilters.length === 0) {
|
||||
return onExhausted?.()
|
||||
}
|
||||
|
||||
let count = 0
|
||||
|
||||
await this.options.request({
|
||||
relays,
|
||||
filters: trimFilters(requestFilters),
|
||||
onEvent: (event: E) => {
|
||||
count += 1
|
||||
until = Math.min(until, event.created_at - 1)
|
||||
onEvent?.(event)
|
||||
},
|
||||
})
|
||||
|
||||
if (useWindowing) {
|
||||
if (since === minSince) {
|
||||
onExhausted?.()
|
||||
}
|
||||
|
||||
// Relays can't be relied upon to return events in descending order, do exponential
|
||||
// windowing to ensure we get the most recent stuff on first load, but eventually find it all
|
||||
if (count < limit) {
|
||||
delta = delta * Math.round(10 - 9 * (Math.log(count + 1) / Math.log(limit + 1)))
|
||||
until = since
|
||||
}
|
||||
|
||||
since = Math.max(minSince, until - delta)
|
||||
} else if (count === 0) {
|
||||
onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _getDifferenceLoader(feeds: Feed[], {onEvent, onExhausted}: LoadOpts<E>) {
|
||||
const exhausted = new Set<number>()
|
||||
const skip = new Set<string>()
|
||||
const events: E[] = []
|
||||
const seen = new Set()
|
||||
|
||||
const loaders = await Promise.all(
|
||||
feeds.map((feed: Feed, i: number) =>
|
||||
this.getLoader(feed, {
|
||||
onExhausted: () => exhausted.add(i),
|
||||
onEvent: (event: E) => {
|
||||
if (i === 0) {
|
||||
events.push(event)
|
||||
} else {
|
||||
skip.add(event.id)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return async (limit: number) => {
|
||||
await Promise.all(
|
||||
loaders.map(async (loader: Loader, i: number) => {
|
||||
if (exhausted.has(i)) {
|
||||
return
|
||||
}
|
||||
|
||||
await loader(limit)
|
||||
})
|
||||
)
|
||||
|
||||
for (const event of events.splice(0)) {
|
||||
if (!skip.has(event.id) && !seen.has(event.id)) {
|
||||
onEvent?.(event)
|
||||
seen.add(event.id)
|
||||
}
|
||||
}
|
||||
|
||||
if (exhausted.size === loaders.length) {
|
||||
onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _getIntersectionLoader(feeds: Feed[], {onEvent, onExhausted}: LoadOpts<E>) {
|
||||
const exhausted = new Set<number>()
|
||||
const counts = new Map<string, number>()
|
||||
const events: E[] = []
|
||||
const seen = new Set()
|
||||
|
||||
const loaders = await Promise.all(
|
||||
feeds.map((feed: Feed, i: number) =>
|
||||
this.getLoader(feed, {
|
||||
onExhausted: () => exhausted.add(i),
|
||||
onEvent: (event: E) => {
|
||||
events.push(event)
|
||||
counts.set(event.id, inc(counts.get(event.id)))
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return async (limit: number) => {
|
||||
await Promise.all(
|
||||
loaders.map(async (loader: Loader, i: number) => {
|
||||
if (exhausted.has(i)) {
|
||||
return
|
||||
}
|
||||
|
||||
await loader(limit)
|
||||
})
|
||||
)
|
||||
|
||||
for (const event of events.splice(0)) {
|
||||
if (counts.get(event.id) === loaders.length && !seen.has(event.id)) {
|
||||
onEvent?.(event)
|
||||
seen.add(event.id)
|
||||
}
|
||||
}
|
||||
|
||||
if (exhausted.size === loaders.length) {
|
||||
onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _getUnionLoader(feeds: Feed[], {onEvent, onExhausted}: LoadOpts<E>) {
|
||||
const exhausted = new Set<number>()
|
||||
const seen = new Set()
|
||||
|
||||
const loaders = await Promise.all(
|
||||
feeds.map((feed: Feed, i: number) =>
|
||||
this.getLoader(feed, {
|
||||
onExhausted: () => exhausted.add(i),
|
||||
onEvent: (event: E) => {
|
||||
if (!seen.has(event.id)) {
|
||||
onEvent?.(event)
|
||||
seen.add(event.id)
|
||||
}
|
||||
},
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
return async (limit: number) => {
|
||||
await Promise.all(
|
||||
loaders.map(async (loader: Loader, i: number) => {
|
||||
if (exhausted.has(i)) {
|
||||
return
|
||||
}
|
||||
|
||||
await loader(limit)
|
||||
})
|
||||
)
|
||||
|
||||
if (exhausted.size === loaders.length) {
|
||||
onExhausted?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import {ensureNumber} from '@welshman/lib'
|
||||
import type {Filter} from '@welshman/util'
|
||||
import {Tags} from '@welshman/util'
|
||||
import {
|
||||
FeedType,
|
||||
Feed,
|
||||
AddressFeed,
|
||||
AuthorFeed,
|
||||
CreatedAtFeed,
|
||||
DVMFeed,
|
||||
DifferenceFeed,
|
||||
IDFeed,
|
||||
IntersectionFeed,
|
||||
KindFeed,
|
||||
ListFeed,
|
||||
LabelFeed,
|
||||
WOTFeed,
|
||||
RelayFeed,
|
||||
ScopeFeed,
|
||||
SearchFeed,
|
||||
TagFeed,
|
||||
UnionFeed,
|
||||
TagFeedMapping,
|
||||
Scope,
|
||||
WOTItem,
|
||||
DVMItem,
|
||||
ListItem,
|
||||
LabelItem,
|
||||
CreatedAtItem,
|
||||
} from './core'
|
||||
|
||||
export const makeAddressFeed = (...addresses: string[]): AddressFeed => [FeedType.Address, ...addresses]
|
||||
export const makeAuthorFeed = (...pubkeys: string[]): AuthorFeed => [FeedType.Author, ...pubkeys]
|
||||
export const makeCreatedAtFeed = (...items: CreatedAtItem[]): CreatedAtFeed => [FeedType.CreatedAt, ...items]
|
||||
export const makeDVMFeed = (...items: DVMItem[]): DVMFeed => [FeedType.DVM, ...items]
|
||||
export const makeDifferenceFeed = (...feeds: Feed[]): DifferenceFeed => [FeedType.Difference, ...feeds]
|
||||
export const makeIDFeed = (...ids: string[]): IDFeed => [FeedType.ID, ...ids]
|
||||
export const makeIntersectionFeed = (...feeds: Feed[]): IntersectionFeed => [FeedType.Intersection, ...feeds]
|
||||
export const makeKindFeed = (...kinds: number[]): KindFeed => [FeedType.Kind, ...kinds]
|
||||
export const makeListFeed = (...items: ListItem[]): ListFeed => [FeedType.List, ...items]
|
||||
export const makeLabelFeed = (...items: LabelItem[]): LabelFeed => [FeedType.Label, ...items]
|
||||
export const makeWOTFeed = (...items: WOTItem[]): WOTFeed => [FeedType.WOT, ...items]
|
||||
export const makeRelayFeed = (...urls: string[]): RelayFeed => [FeedType.Relay, ...urls]
|
||||
export const makeScopeFeed = (...scopes: Scope[]): ScopeFeed => [FeedType.Scope, ...scopes]
|
||||
export const makeSearchFeed = (...searches: string[]): SearchFeed => [FeedType.Search, ...searches]
|
||||
export const makeTagFeed = (key: string, ...values: string[]): TagFeed => [FeedType.Tag, key, ...values]
|
||||
export const makeUnionFeed = (...feeds: Feed[]): UnionFeed => [FeedType.Union, ...feeds]
|
||||
|
||||
export const isAddressFeed = (feed: Feed): feed is AddressFeed => feed[0] === FeedType.Address
|
||||
export const isAuthorFeed = (feed: Feed): feed is AuthorFeed => feed[0] === FeedType.Author
|
||||
export const isCreatedAtFeed = (feed: Feed): feed is CreatedAtFeed => feed[0] === FeedType.CreatedAt
|
||||
export const isDVMFeed = (feed: Feed): feed is DVMFeed => feed[0] === FeedType.DVM
|
||||
export const isDifferenceFeed = (feed: Feed): feed is DifferenceFeed => feed[0] === FeedType.Difference
|
||||
export const isIDFeed = (feed: Feed): feed is IDFeed => feed[0] === FeedType.ID
|
||||
export const isIntersectionFeed = (feed: Feed): feed is IntersectionFeed => feed[0] === FeedType.Intersection
|
||||
export const isKindFeed = (feed: Feed): feed is KindFeed => feed[0] === FeedType.Kind
|
||||
export const isListFeed = (feed: Feed): feed is ListFeed => feed[0] === FeedType.List
|
||||
export const isLabelFeed = (feed: Feed): feed is LabelFeed => feed[0] === FeedType.Label
|
||||
export const isWOTFeed = (feed: Feed): feed is WOTFeed => feed[0] === FeedType.WOT
|
||||
export const isRelayFeed = (feed: Feed): feed is RelayFeed => feed[0] === FeedType.Relay
|
||||
export const isScopeFeed = (feed: Feed): feed is ScopeFeed => feed[0] === FeedType.Scope
|
||||
export const isSearchFeed = (feed: Feed): feed is SearchFeed => feed[0] === FeedType.Search
|
||||
export const isTagFeed = (feed: Feed): feed is TagFeed => feed[0] === FeedType.Tag
|
||||
export const isUnionFeed = (feed: Feed): feed is UnionFeed => feed[0] === FeedType.Union
|
||||
|
||||
export function getFeedArgs(feed: IntersectionFeed | UnionFeed | DifferenceFeed): Feed[]
|
||||
export function getFeedArgs(feed: AddressFeed | AuthorFeed | IDFeed | RelayFeed | SearchFeed): string[]
|
||||
export function getFeedArgs(feed: CreatedAtFeed): CreatedAtItem[]
|
||||
export function getFeedArgs(feed: ListFeed): ListItem[]
|
||||
export function getFeedArgs(feed: LabelFeed): LabelItem[]
|
||||
export function getFeedArgs(feed: DVMFeed): DVMItem[]
|
||||
export function getFeedArgs(feed: WOTFeed): WOTItem[]
|
||||
export function getFeedArgs(feed: ScopeFeed): Scope[]
|
||||
export function getFeedArgs(feed: KindFeed): number[]
|
||||
export function getFeedArgs(feed: TagFeed): [string, ...string[]]
|
||||
export function getFeedArgs(feed: Feed) {
|
||||
switch (feed[0]) {
|
||||
case FeedType.Intersection: return feed.slice(1) as Feed[]
|
||||
case FeedType.Union: return feed.slice(1) as Feed[]
|
||||
case FeedType.Difference: return feed.slice(1) as Feed[]
|
||||
case FeedType.Address: return feed.slice(1) as string[]
|
||||
case FeedType.Author: return feed.slice(1) as string[]
|
||||
case FeedType.ID: return feed.slice(1) as string[]
|
||||
case FeedType.Relay: return feed.slice(1) as string[]
|
||||
case FeedType.Search: return feed.slice(1) as string[]
|
||||
case FeedType.Tag: return feed.slice(1) as [string, ...string[]]
|
||||
case FeedType.CreatedAt: return feed.slice(1) as CreatedAtItem[]
|
||||
case FeedType.List: return feed.slice(1) as ListItem[]
|
||||
case FeedType.Label: return feed.slice(1) as LabelItem[]
|
||||
case FeedType.DVM: return feed.slice(1) as DVMItem[]
|
||||
case FeedType.WOT: return feed.slice(1) as WOTItem[]
|
||||
case FeedType.Scope: return feed.slice(1) as Scope[]
|
||||
case FeedType.Kind: return feed.slice(1) as number[]
|
||||
}
|
||||
}
|
||||
|
||||
export const hasSubFeeds = (feed: Feed): feed is IntersectionFeed | UnionFeed | DifferenceFeed =>
|
||||
[FeedType.Union, FeedType.Intersection, FeedType.Difference].includes(feed[0])
|
||||
|
||||
export const defaultTagFeedMappings: TagFeedMapping[] = [
|
||||
['a', [FeedType.Address]],
|
||||
['e', [FeedType.ID]],
|
||||
['p', [FeedType.Author]],
|
||||
['r', [FeedType.Relay]],
|
||||
['t', [FeedType.Tag, '#t']],
|
||||
]
|
||||
|
||||
export const feedsFromTags = (tags: Tags, mappings?: TagFeedMapping[]) => {
|
||||
const feeds = []
|
||||
|
||||
for (const [tagName, templateFeed] of mappings || defaultTagFeedMappings) {
|
||||
const filterTags = tags.whereKey(tagName)
|
||||
|
||||
if (filterTags.exists()) {
|
||||
let values: string[] | number[] = filterTags.values().valueOf()
|
||||
|
||||
if (isKindFeed(templateFeed)) {
|
||||
values = values.map(ensureNumber) as number[]
|
||||
}
|
||||
|
||||
feeds.push([...templateFeed, ...values] as Feed)
|
||||
}
|
||||
}
|
||||
|
||||
return feeds
|
||||
}
|
||||
|
||||
export const feedFromTags = (tags: Tags, mappings?: TagFeedMapping[]) =>
|
||||
makeIntersectionFeed(...feedsFromTags(tags, mappings))
|
||||
|
||||
export const feedsFromFilter = ({since, until, ...filter}: Filter) => {
|
||||
const feeds = []
|
||||
|
||||
if (since && until) {
|
||||
feeds.push(makeCreatedAtFeed({since, until}))
|
||||
} else if (since) {
|
||||
feeds.push(makeCreatedAtFeed({since}))
|
||||
} else if (until) {
|
||||
feeds.push(makeCreatedAtFeed({until}))
|
||||
}
|
||||
|
||||
for (const [k, v] of Object.entries(filter)) {
|
||||
if (k === 'ids') feeds.push(makeIDFeed(...v as string[]))
|
||||
else if (k === 'kinds') feeds.push(makeKindFeed(...v as number[]))
|
||||
else if (k === 'authors') feeds.push(makeAuthorFeed(...v as string[]))
|
||||
else if (k.startsWith('#')) feeds.push(makeTagFeed(k as string, ...v as string[]))
|
||||
else throw new Error(`Unable to create feed from filter ${k}: ${v}`)
|
||||
}
|
||||
|
||||
return feeds
|
||||
}
|
||||
|
||||
export const feedFromFilter = (filter: Filter) =>
|
||||
makeIntersectionFeed(...feedsFromFilter(filter))
|
||||
|
||||
|
||||
export const walkFeed = (feed: Feed, visit: (feed: Feed) => void) => {
|
||||
visit(feed)
|
||||
|
||||
if (hasSubFeeds(feed)) {
|
||||
for (const subFeed of getFeedArgs(feed)) {
|
||||
walkFeed(subFeed, visit)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user