coracle/welshman
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move feed utils to its own file

Browse Source
This commit is contained in:
Jon Staab
2024-05-03 10:30:50 -07:00
parent a343f74241
commit d4cf63e906
10 changed files with 245 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/feeds/compiler.ts
+43 -62
View File
@@ -1,8 +1,9 @@
import {uniq, identity, flatten, pushToMapKey, intersection, ensureNumber, tryCatch, now} from '@welshman/lib'
import {uniq, identity, flatten, pushToMapKey, intersection, tryCatch, now} from '@welshman/lib'
import type {Rumor, Filter} from '@welshman/util'
import {Tags, intersectFilters, getAddress, getIdFilters, unionFilters} from '@welshman/util'
import type {WOTItem, CreatedAtItem, RequestItem, TagFilterMapping, ListItem, DVMItem, Scope, Feed, FeedOptions} from './core'
import {FeedType, getSubFeeds} from './core'
import type {CreatedAtItem, RequestItem, ListItem, WOTItem, DVMItem, Scope, Feed, FeedOptions} from './core'
import {hasSubFeeds, getFeedArgs, feedsFromTags} from './utils'
import {FeedType} from './core'
export class FeedCompiler<E extends Rumor> {
constructor(readonly options: FeedOptions<E>) {}
@@ -10,16 +11,18 @@ export class FeedCompiler<E extends Rumor> {
walk(feed: Feed, visit: (feed: Feed) => void) {
visit(feed)
for (const subFeed of getSubFeeds(feed)) {
this.walk(subFeed, visit)
if (hasSubFeeds(feed)) {
for (const subFeed of getFeedArgs(feed)) {
this.walk(subFeed, visit)
}
}
}
canCompile([type, ...feed]: Feed): boolean {
switch(type) {
canCompile(feed: Feed): boolean {
switch(feed[0]) {
case FeedType.Union:
case FeedType.Intersection:
return getSubFeeds([type, ...feed] as Feed).every(f => this.canCompile(f))
return getFeedArgs(feed).every(f => this.canCompile(f))
case FeedType.Address:
case FeedType.Author:
case FeedType.CreatedAt:
@@ -38,24 +41,28 @@ export class FeedCompiler<E extends Rumor> {
}
}
async compile([type, ...feed]: Feed): Promise<RequestItem[]> {
switch(type) {
case FeedType.Address: return this._compileAddresses(feed as string[])
case FeedType.CreatedAt: return this._compileCreatedAt(feed as CreatedAtItem[])
case FeedType.Author: return this._compileFilter("authors", feed as string[])
case FeedType.DVM: return await this._compileDvms(feed as DVMItem[])
case FeedType.ID: return this._compileFilter("ids", feed as string[])
case FeedType.Intersection: return await this._compileIntersection(feed as Feed[])
case FeedType.Kind: return this._compileFilter("kinds", feed as number[])
case FeedType.List: return await this._compileLists(feed as ListItem[])
case FeedType.Relay: return [{relays: feed as string[]}]
case FeedType.Scope: return this._compileScopes(feed as Scope[])
case FeedType.Search: return this._compileSearches(feed as string[])
case FeedType.Tag: return this._compileFilter(feed[0] as string, feed.slice(1) as string[])
case FeedType.Union: return await this._compileUnion(feed as Feed[])
case FeedType.WOT: return this._compileWot(feed as WOTItem)
async compile(feed: Feed): Promise<RequestItem[]> {
switch(feed[0]) {
case FeedType.ID: return this._compileFilter('ids', getFeedArgs(feed))
case FeedType.Kind: return this._compileFilter('kinds', getFeedArgs(feed))
case FeedType.Author: return this._compileFilter('authors', getFeedArgs(feed))
case FeedType.DVM: return await this._compileDvms(getFeedArgs(feed))
case FeedType.Intersection: return await this._compileIntersection(getFeedArgs(feed))
case FeedType.List: return await this._compileLists(getFeedArgs(feed))
case FeedType.Union: return await this._compileUnion(getFeedArgs(feed))
case FeedType.Address: return this._compileAddresses(getFeedArgs(feed))
case FeedType.CreatedAt: return this._compileCreatedAt(getFeedArgs(feed))
case FeedType.Scope: return this._compileScopes(getFeedArgs(feed))
case FeedType.Search: return this._compileSearches(getFeedArgs(feed))
case FeedType.WOT: return this._compileWot(getFeedArgs(feed))
case FeedType.Relay: return [{relays: getFeedArgs(feed)}]
case FeedType.Tag: {
const [key, ...value] = getFeedArgs(feed)
return this._compileFilter(key, value)
}
default:
throw new Error(`Unable to convert feed of type ${type} to filters`)
throw new Error(`Unable to convert feed of type ${feed[0]} to filters`)
}
}
@@ -99,31 +106,31 @@ export class FeedCompiler<E extends Rumor> {
return [{filters: searches.map(search => ({search}))}]
}
_compileWot({min = 0, max = 1}) {
return [{filters: [{authors: this.options.getPubkeysForWotRange(min, max)}]}]
_compileWot(wotItems: WOTItem[]) {
return [{filters: wotItems.map(({min = 0, max = 1}) => ({authors: this.options.getPubkeysForWOTRange(min, max)}))}]
}
async _compileDvms(items: DVMItem[]): Promise<RequestItem[]> {
const filters: Filter[] = []
const feeds: Feed[] = []
await Promise.all(
items.map(({mappings, ...request}) =>
this.options.requestDvm({
this.options.requestDVM({
...request,
onEvent: async (e: E) => {
const tags = Tags.fromEvent(e)
const request = await tryCatch(() => JSON.parse(tags.get("request")?.value()))
const responseTags = tags.rejectByValue([request?.id, request?.pubkey])
for (const filter of await this._getFiltersFromTags(responseTags, mappings)) {
filters.push(filter)
for (const feed of feedsFromTags(responseTags, mappings)) {
feeds.push(feed)
}
},
})
)
)
return [{filters: unionFilters(filters)}]
return this._compileUnion(feeds)
}
async _compileIntersection(feeds: Feed[]): Promise<RequestItem[]> {
@@ -131,7 +138,7 @@ export class FeedCompiler<E extends Rumor> {
const result = []
for (let {filters, relays} of head) {
for (let {filters, relays} of head || []) {
const matchingGroups = tail.map(
items => items.filter(
it => (
@@ -225,42 +232,16 @@ export class FeedCompiler<E extends Rumor> {
onEvent: (e: E) => eventsByAddress.set(getAddress(e), e),
})
const filters = flatten(
const feeds = flatten(
await Promise.all(
listItems.map(({address, mappings}) => {
const event = eventsByAddress.get(address)
return event ? this._getFiltersFromTags(Tags.fromEvent(event), mappings) : []
return event ? feedsFromTags(Tags.fromEvent(event), mappings) : []
})
)
)
return [{filters: unionFilters(filters)}]
}
// Utilities
async _getFiltersFromTags(tags: Tags, mappings: TagFilterMapping[]) {
const filters = []
for (const [tagName, feedType] of mappings) {
const filterTags = tags.whereKey(tagName)
if (filterTags.exists()) {
let values: string[] | number[] = filterTags.values().valueOf()
if (feedType === FeedType.Kind) {
values = values.map(ensureNumber) as number[]
}
for (const item of await this.compile([feedType, ...values] as Feed)) {
for (const filter of item.filters || []) {
filters.push(filter)
}
}
}
}
return unionFilters(filters)
return this._compileUnion(feeds)
}
}
packages/feeds/core.ts
+5 -73
View File
@@ -34,18 +34,18 @@ export type FilterFeedType =
FeedType.Relay |
FeedType.Tag
export type TagFilterMapping = [string, FilterFeedType]
export type TagFeedMapping = [string, Feed]
export type DVMItem = {
kind: number,
mappings: TagFilterMapping[],
tags?: string[][],
relays?: string[],
mappings?: TagFeedMapping[],
}
export type ListItem = {
address: string,
mappings: TagFilterMapping[],
mappings: TagFeedMapping[],
}
export type WOTItem = {
@@ -94,74 +94,6 @@ export type Feed =
TagFeed |
UnionFeed
export const addressFeed = (...addresses: string[]): AddressFeed => [FeedType.Address, ...addresses]
export const authorFeed = (...pubkeys: string[]): AuthorFeed => [FeedType.Author, ...pubkeys]
export const createdAtFeed = (...items: CreatedAtItem[]): CreatedAtFeed => [FeedType.CreatedAt, ...items]
export const dvmFeed = (...items: DVMItem[]): DVMFeed => [FeedType.DVM, ...items]
export const differenceFeed = (...feeds: Feed[]): DifferenceFeed => [FeedType.Difference, ...feeds]
export const idFeed = (...ids: string[]): IDFeed => [FeedType.ID, ...ids]
export const intersectionFeed = (...feeds: Feed[]): IntersectionFeed => [FeedType.Intersection, ...feeds]
export const kindFeed = (...kinds: number[]): KindFeed => [FeedType.Kind, ...kinds]
export const listFeed = (...items: ListItem[]): ListFeed => [FeedType.List, ...items]
export const wotFeed = (...items: WOTItem[]): WOTFeed => [FeedType.WOT, ...items]
export const relayFeed = (...urls: string[]): RelayFeed => [FeedType.Relay, ...urls]
export const scopeFeed = (...scopes: Scope[]): ScopeFeed => [FeedType.Scope, ...scopes]
export const searchFeed = (...searches: string[]): SearchFeed => [FeedType.Search, ...searches]
export const symmetricDifferenceFeed = (...feeds: Feed[]): SymmetricDifferenceFeed => [FeedType.SymmetricDifference, ...feeds]
export const tagFeed = (key: string, ...values: string[]): TagFeed => [FeedType.Tag, key, ...values]
export const unionFeed = (...feeds: Feed[]): UnionFeed => [FeedType.Union, ...feeds]
export const feedsFromFilter = ({since, until, ...filter}: Filter) => {
const feeds = []
if (since && until) {
feeds.push(createdAtFeed({since, until}))
} else if (since) {
feeds.push(createdAtFeed({since}))
} else if (until) {
feeds.push(createdAtFeed({until}))
}
for (const [k, v] of Object.entries(filter)) {
if (k === 'ids') feeds.push(idFeed(...v as string[]))
else if (k === 'kinds') feeds.push(kindFeed(...v as number[]))
else if (k === 'authors') feeds.push(authorFeed(...v as string[]))
else if (k.startsWith('#')) feeds.push(tagFeed(k as string, ...v as string[]))
else throw new Error(`Unable to create feed from filter ${k}: ${v}`)
}
return feeds
}
export const feedFromFilter = (filter: Filter) => intersectionFeed(...feedsFromFilter(filter))
export const isAddressFeed = (feed: Feed) => feed[0] === FeedType.Address
export const isAuthorFeed = (feed: Feed) => feed[0] === FeedType.Author
export const isCreatedAtFeed = (feed: Feed) => feed[0] === FeedType.CreatedAt
export const isDvmFeed = (feed: Feed) => feed[0] === FeedType.DVM
export const isDifferenceFeed = (feed: Feed) => feed[0] === FeedType.Difference
export const isIdFeed = (feed: Feed) => feed[0] === FeedType.ID
export const isIntersectionFeed = (feed: Feed) => feed[0] === FeedType.Intersection
export const isKindFeed = (feed: Feed) => feed[0] === FeedType.Kind
export const isListFeed = (feed: Feed) => feed[0] === FeedType.List
export const isWotFeed = (feed: Feed) => feed[0] === FeedType.WOT
export const isRelayFeed = (feed: Feed) => feed[0] === FeedType.Relay
export const isScopeFeed = (feed: Feed) => feed[0] === FeedType.Scope
export const isSearchFeed = (feed: Feed) => feed[0] === FeedType.Search
export const isSymmetricDifferenceFeed = (feed: Feed) => feed[0] === FeedType.SymmetricDifference
export const isTagFeed = (feed: Feed) => feed[0] === FeedType.Tag
export const isUnionFeed = (feed: Feed) => feed[0] === FeedType.Union
export const hasSubFeeds = ([type]: [FeedType, ...any[]]) =>
[
FeedType.Union,
FeedType.Intersection,
FeedType.Difference,
FeedType.SymmetricDifference,
].includes(type)
export const getSubFeeds = ([type, ...feeds]: Feed): Feed[] => hasSubFeeds([type]) ? feeds as Feed[] : []
export type RequestItem = {
relays?: string[]
filters?: Filter[]
@@ -183,7 +115,7 @@ export type DVMOpts<E> = DVMRequest & {
export type FeedOptions<E> = {
request: (opts: RequestOpts<E>) => Promise<void>
requestDvm: (opts: DVMOpts<E>) => Promise<void>
requestDVM: (opts: DVMOpts<E>) => Promise<void>
getPubkeysForScope: (scope: Scope) => string[]
getPubkeysForWotRange: (minWot: number, maxWot: number) => string[]
getPubkeysForWOTRange: (minWOT: number, maxWOT: number) => string[]
}
packages/feeds/index.ts
+1
View File
@@ -1,3 +1,4 @@
export * from './core'
export * from './compiler'
export * from './loader'
export * from './utils'
packages/feeds/loader.ts
+5
View File
@@ -39,6 +39,11 @@ export class FeedLoader<E extends Rumor> {
}
async _getRequestsLoader(requests: RequestItem[], {onEvent, onExhausted}: LoadOpts<E>) {
// Empty requests are not a no-op, they're a global feed
if (requests.length === 0) {
requests = [{}]
}
const seen = new Set()
const exhausted = new Set()
const loaders = await Promise.all(
packages/feeds/tsc-multi.json
+1
View File
@@ -1,5 +1,6 @@
{
"targets": [
{"extname": ".cjs", "module": "commonjs"},
{"extname": ".mjs", "module": "esnext", "moduleResolution": "node"}
],
"projects": ["tsconfig.json"]
packages/feeds/utils.ts
+153
View File
@@ -0,0 +1,153 @@
import {ensureNumber} from '@welshman/lib'
import type {Filter} from '@welshman/util'
import {Tags} from '@welshman/util'
import {
FeedType,
Feed,
AddressFeed,
AuthorFeed,
CreatedAtFeed,
DVMFeed,
DifferenceFeed,
IDFeed,
IntersectionFeed,
KindFeed,
ListFeed,
WOTFeed,
RelayFeed,
ScopeFeed,
SearchFeed,
SymmetricDifferenceFeed,
TagFeed,
UnionFeed,
TagFeedMapping,
Scope,
WOTItem,
DVMItem,
ListItem,
CreatedAtItem,
} from './core'
export const makeAddressFeed = (...addresses: string[]): AddressFeed => [FeedType.Address, ...addresses]
export const makeAuthorFeed = (...pubkeys: string[]): AuthorFeed => [FeedType.Author, ...pubkeys]
export const makeCreatedAtFeed = (...items: CreatedAtItem[]): CreatedAtFeed => [FeedType.CreatedAt, ...items]
export const makeDVMFeed = (...items: DVMItem[]): DVMFeed => [FeedType.DVM, ...items]
export const makeDifferenceFeed = (...feeds: Feed[]): DifferenceFeed => [FeedType.Difference, ...feeds]
export const makeIDFeed = (...ids: string[]): IDFeed => [FeedType.ID, ...ids]
export const makeIntersectionFeed = (...feeds: Feed[]): IntersectionFeed => [FeedType.Intersection, ...feeds]
export const makeKindFeed = (...kinds: number[]): KindFeed => [FeedType.Kind, ...kinds]
export const makeListFeed = (...items: ListItem[]): ListFeed => [FeedType.List, ...items]
export const makeWOTFeed = (...items: WOTItem[]): WOTFeed => [FeedType.WOT, ...items]
export const makeRelayFeed = (...urls: string[]): RelayFeed => [FeedType.Relay, ...urls]
export const makeScopeFeed = (...scopes: Scope[]): ScopeFeed => [FeedType.Scope, ...scopes]
export const makeSearchFeed = (...searches: string[]): SearchFeed => [FeedType.Search, ...searches]
export const makeSymmetricDifferenceFeed = (...feeds: Feed[]): SymmetricDifferenceFeed => [FeedType.SymmetricDifference, ...feeds]
export const makeTagFeed = (key: string, ...values: string[]): TagFeed => [FeedType.Tag, key, ...values]
export const makeUnionFeed = (...feeds: Feed[]): UnionFeed => [FeedType.Union, ...feeds]
export const isAddressFeed = (feed: Feed): feed is AddressFeed => feed[0] === FeedType.Address
export const isAuthorFeed = (feed: Feed): feed is AuthorFeed => feed[0] === FeedType.Author
export const isCreatedAtFeed = (feed: Feed): feed is CreatedAtFeed => feed[0] === FeedType.CreatedAt
export const isDVMFeed = (feed: Feed): feed is DVMFeed => feed[0] === FeedType.DVM
export const isDifferenceFeed = (feed: Feed): feed is DifferenceFeed => feed[0] === FeedType.Difference
export const isIDFeed = (feed: Feed): feed is IDFeed => feed[0] === FeedType.ID
export const isIntersectionFeed = (feed: Feed): feed is IntersectionFeed => feed[0] === FeedType.Intersection
export const isKindFeed = (feed: Feed): feed is KindFeed => feed[0] === FeedType.Kind
export const isListFeed = (feed: Feed): feed is ListFeed => feed[0] === FeedType.List
export const isWOTFeed = (feed: Feed): feed is WOTFeed => feed[0] === FeedType.WOT
export const isRelayFeed = (feed: Feed): feed is RelayFeed => feed[0] === FeedType.Relay
export const isScopeFeed = (feed: Feed): feed is ScopeFeed => feed[0] === FeedType.Scope
export const isSearchFeed = (feed: Feed): feed is SearchFeed => feed[0] === FeedType.Search
export const isSymmetricDifferenceFeed = (feed: Feed): feed is SymmetricDifferenceFeed => feed[0] === FeedType.SymmetricDifference
export const isTagFeed = (feed: Feed): feed is TagFeed => feed[0] === FeedType.Tag
export const isUnionFeed = (feed: Feed): feed is UnionFeed => feed[0] === FeedType.Union
export function getFeedArgs(feed: IntersectionFeed | UnionFeed | DifferenceFeed | SymmetricDifferenceFeed): Feed[]
export function getFeedArgs(feed: AddressFeed | AuthorFeed | IDFeed | RelayFeed | SearchFeed): string[]
export function getFeedArgs(feed: CreatedAtFeed): CreatedAtItem[]
export function getFeedArgs(feed: ListFeed): ListItem[]
export function getFeedArgs(feed: DVMFeed): DVMItem[]
export function getFeedArgs(feed: WOTFeed): WOTItem[]
export function getFeedArgs(feed: ScopeFeed): Scope[]
export function getFeedArgs(feed: KindFeed): number[]
export function getFeedArgs(feed: TagFeed): [string, ...string[]]
export function getFeedArgs(feed: Feed) {
switch (feed[0]) {
case FeedType.Intersection: return feed.slice(1) as Feed[]
case FeedType.Union: return feed.slice(1) as Feed[]
case FeedType.Difference: return feed.slice(1) as Feed[]
case FeedType.SymmetricDifference: return feed.slice(1) as Feed[]
case FeedType.Address: return feed.slice(1) as string[]
case FeedType.Author: return feed.slice(1) as string[]
case FeedType.ID: return feed.slice(1) as string[]
case FeedType.Relay: return feed.slice(1) as string[]
case FeedType.Search: return feed.slice(1) as string[]
case FeedType.Tag: return feed.slice(1) as [string, ...string[]]
case FeedType.CreatedAt: return feed.slice(1) as CreatedAtItem[]
case FeedType.List: return feed.slice(1) as ListItem[]
case FeedType.DVM: return feed.slice(1) as DVMItem[]
case FeedType.WOT: return feed.slice(1) as WOTItem[]
case FeedType.Scope: return feed.slice(1) as Scope[]
case FeedType.Kind: return feed.slice(1) as number[]
}
}
export const hasSubFeeds = (feed: Feed): feed is IntersectionFeed | UnionFeed | DifferenceFeed | SymmetricDifferenceFeed =>
[
FeedType.Union,
FeedType.Intersection,
FeedType.Difference,
FeedType.SymmetricDifference,
].includes(feed[0])
export const defaultTagFeedMappings: TagFeedMapping[] = [
['a', [FeedType.Address]],
['e', [FeedType.ID]],
['p', [FeedType.Author]],
['r', [FeedType.Relay]],
['t', [FeedType.Tag, '#t']],
]
export const feedsFromTags = (tags: Tags, mappings?: TagFeedMapping[]) => {
const feeds = []
for (const [tagName, templateFeed] of mappings || defaultTagFeedMappings) {
const filterTags = tags.whereKey(tagName)
if (filterTags.exists()) {
let values: string[] | number[] = filterTags.values().valueOf()
if (isKindFeed(templateFeed)) {
values = values.map(ensureNumber) as number[]
}
feeds.push([...templateFeed, ...values] as Feed)
}
}
return feeds
}
export const feedsFromFilter = ({since, until, ...filter}: Filter) => {
const feeds = []
if (since && until) {
feeds.push(makeCreatedAtFeed({since, until}))
} else if (since) {
feeds.push(makeCreatedAtFeed({since}))
} else if (until) {
feeds.push(makeCreatedAtFeed({until}))
}
for (const [k, v] of Object.entries(filter)) {
if (k === 'ids') feeds.push(makeIDFeed(...v as string[]))
else if (k === 'kinds') feeds.push(makeKindFeed(...v as number[]))
else if (k === 'authors') feeds.push(makeAuthorFeed(...v as string[]))
else if (k.startsWith('#')) feeds.push(makeTagFeed(k as string, ...v as string[]))
else throw new Error(`Unable to create feed from filter ${k}: ${v}`)
}
return feeds
}
export const feedFromFilter = (filter: Filter) => makeIntersectionFeed(...feedsFromFilter(filter))