Add custom feeds

install.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# npm i
+
+for package in $(ls packages); do
+  version=$(sed -nr 's/ +"version": "(.+)",/\1/p' packages/$package/package.json)
+
+  for downstream in $(ls packages); do
+    n=@coracle.social/$package
+    f=packages/$downstream/package.json
+    v=$(jq '.dependencies["'$n'"] // empty' $f)
+
+    if [[ ! -z $v ]]; then
+      jq '.dependencies["'$n'"]="'$version'"' $f
+    fi
+  done
+done

package-lock.json

@@ -125,6 +125,10 @@
         "node": ">=4"
       }
     },
+    "node_modules/@coracle.social/custom-feeds": {
+      "resolved": "packages/custom-feeds",
+      "link": true
+    },
     "node_modules/@coracle.social/lib": {
       "resolved": "packages/lib",
       "link": true
@@ -1296,6 +1300,14 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/execa": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
@@ -3332,12 +3344,40 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "packages/custom-feeds": {
+      "name": "@coracle.social/custom-feeds",
+      "version": "0.0.1",
+      "license": "MIT",
+      "dependencies": {
+        "@coracle.social/util": "*"
+      },
+      "devDependencies": {
+        "gts": "^5.0.1",
+        "tsc-multi": "^1.1.0",
+        "typescript": "~5.1.6"
+      }
+    },
+    "packages/feed": {
+      "name": "@coracle.social/feed",
+      "version": "0.0.1",
+      "extraneous": true,
+      "license": "MIT",
+      "dependencies": {
+        "@coracle.social/util": "^0.0.7"
+      },
+      "devDependencies": {
+        "gts": "^5.0.1",
+        "tsc-multi": "^1.1.0",
+        "typescript": "~5.1.6"
+      }
+    },
     "packages/lib": {
       "name": "@coracle.social/lib",
-      "version": "0.0.4",
+      "version": "0.0.5",
       "license": "MIT",
       "dependencies": {
         "@scure/base": "^1.1.6",
+        "events": "^3.3.0",
         "throttle-debounce": "^5.0.0"
       },
       "devDependencies": {
@@ -3361,8 +3401,8 @@
       "version": "0.0.7",
       "license": "MIT",
       "dependencies": {
-        "@coracle.social/lib": "^0.0.4",
+        "@coracle.social/lib": "*",
-        "@coracle.social/util": "^0.0.6",
+        "@coracle.social/util": "*",
         "isomorphic-ws": "^5.0.0",
         "ws": "^8.16.0"
       },
@@ -3373,13 +3413,39 @@
         "typescript": "~5.1.6"
       }
     },
-    "packages/util": {
+    "packages/network/node_modules/@coracle.social/lib": {
-      "name": "@coracle.social/util",
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/@coracle.social/lib/-/lib-0.0.4.tgz",
+      "integrity": "sha512-K4kYlXCZ90ZXPoHYdAhkzq8hjoe/vhVw2pB5XUI4Q/OCqI+h1/bJ4ubtLJV6j37figihjHPqkg3AnJZAYRlcfA==",
+      "dependencies": {
+        "@scure/base": "^1.1.6",
+        "throttle-debounce": "^5.0.0"
+      }
+    },
+    "packages/network/node_modules/@coracle.social/util": {
       "version": "0.0.6",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/@coracle.social/util/-/util-0.0.6.tgz",
+      "integrity": "sha512-LHr06XeHqXkwE0Kb/fMqgJnB4U47Wq7WD2fH5Ow4o+x4TWaPLlqrppBL9qN03KNfXB70uNvPsB1XB220wDlvkA==",
       "dependencies": {
         "@coracle.social/lib": "^0.0.4",
         "nostr-tools": "^2.3.2"
+      }
+    },
+    "packages/network/node_modules/@scure/base": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.6.tgz",
+      "integrity": "sha512-ok9AWwhcgYuGG3Zfhyqg+zwl+Wn5uE+dwC0NV/2qQkx4dABbb/bx96vWu8NSj+BNjjSjno+JRYRjle1jV08k3g==",
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
+    "packages/util": {
+      "name": "@coracle.social/util",
+      "version": "0.0.7",
+      "license": "MIT",
+      "dependencies": {
+        "@coracle.social/lib": "*",
+        "nostr-tools": "^2.3.2"
       },
       "devDependencies": {
         "@types/events": "^3.0.3",
@@ -3387,6 +3453,23 @@
         "tsc-multi": "^1.1.0",
         "typescript": "~5.1.6"
       }
+    },
+    "packages/util/node_modules/@coracle.social/lib": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/@coracle.social/lib/-/lib-0.0.4.tgz",
+      "integrity": "sha512-K4kYlXCZ90ZXPoHYdAhkzq8hjoe/vhVw2pB5XUI4Q/OCqI+h1/bJ4ubtLJV6j37figihjHPqkg3AnJZAYRlcfA==",
+      "dependencies": {
+        "@scure/base": "^1.1.6",
+        "throttle-debounce": "^5.0.0"
+      }
+    },
+    "packages/util/node_modules/@scure/base": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.6.tgz",
+      "integrity": "sha512-ok9AWwhcgYuGG3Zfhyqg+zwl+Wn5uE+dwC0NV/2qQkx4dABbb/bx96vWu8NSj+BNjjSjno+JRYRjle1jV08k3g==",
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
     }
   }
 }

packages/custom-feeds/.eslintignore

@@ -0,0 +1 @@
+build

packages/custom-feeds/index.ts

@@ -0,0 +1,318 @@
+import {inc, now, isNil} from '@coracle.social/lib'
+import type {Rumor, Filter} from '@coracle.social/util'
+import {Tags, getIdFilters} from '@coracle.social/util'
+
+// TODO:
+// - if one of the feeds in a union is a filter, don't execute it,
+//   use it to filter down results from other feeds
+// - if multiple feeds in a composite are or result in filters (like lists, lols),
+//   merge them before executing the request
+
+export enum Operator {
+  Difference = "\\",
+  Intersection = "∩",
+  SymmetricDifference = "Δ",
+  Union = "∪",
+}
+
+export enum FeedType {
+  Composite = "composite",
+  Filter = "filter",
+  List = "list",
+  LOL = "lol",
+  DVM = "dvm",
+}
+
+export enum Scope {
+  User = "user",
+  Follows = "follows",
+  Followers = "followers",
+}
+
+export type WotRange = 0 | 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 | 0.7 | 0.8 | 0.9 | 1
+
+export type DynamicFilter = Filter & {
+  scopes?: Scope[]
+  min_wot?: WotRange
+  max_wot?: WotRange
+  until_ago?: number
+  since_ago?: number
+}
+
+export type CompositeFeed = {
+  type: FeedType.Composite
+  operator: Operator
+  feeds: Feed[]
+}
+
+export type FilterFeed = {
+  type: FeedType.Filter
+  filter: DynamicFilter
+}
+
+export type ListFeed = {
+  type: FeedType.List
+  address: string
+}
+
+export type LOLFeed = {
+  type: FeedType.LOL
+  address: string
+}
+
+export type DVMFeed = {
+  type: FeedType.DVM
+  kind: number
+  input?: string
+  pubkey?: string
+}
+
+export type Feed = CompositeFeed | FilterFeed | ListFeed | LOLFeed | DVMFeed
+
+export const difference = (feeds: Feed[]) =>
+  ({type: FeedType.Composite as FeedType.Composite, operator: Operator.Difference, feeds})
+
+export const intersection = (feeds: Feed[]) =>
+  ({type: FeedType.Composite as FeedType.Composite, operator: Operator.Intersection, feeds})
+
+export const symmetricDifference = (feeds: Feed[]) =>
+  ({type: FeedType.Composite as FeedType.Composite, operator: Operator.SymmetricDifference, feeds})
+
+export const union = (feeds: Feed[]) =>
+  ({type: FeedType.Composite as FeedType.Composite, operator: Operator.Union, feeds})
+
+export const filter = (filter: DynamicFilter) =>
+  ({type: FeedType.Filter as FeedType.Filter, filter})
+
+export const list = (address: string) =>
+  ({type: FeedType.List as FeedType.List, address})
+
+export const lol = (address: string) =>
+  ({type: FeedType.LOL as FeedType.LOL, address})
+
+export const dvm = (kind: number, input?: string, pubkey?: string) =>
+  ({type: FeedType.DVM as FeedType.DVM, kind, input, pubkey})
+
+export type InterpreterOpts = {
+  reqDvm: (feed: DVMFeed, opts: ExecuteOpts) => void
+  reqFilters: (filters: Filter[], opts: ExecuteOpts) => void
+  getPubkeysForScope: (scope: Scope) => string[]
+  getPubkeysForWotRange: (minWot: number, maxWot: number) => string[]
+}
+
+export type ExecuteOpts = {
+  onEvent: (event: Rumor) => void
+  onComplete?: () => void
+}
+
+export class Interpreter {
+  constructor(readonly opts: InterpreterOpts) {}
+
+  execute(feed: Feed, opts: ExecuteOpts) {
+    switch (feed.type) {
+      case FeedType.Composite:
+        switch (feed.operator) {
+          case Operator.Difference: return this.executeDifference(feed, opts)
+          case Operator.Intersection: return this.executeIntersection(feed, opts)
+          case Operator.SymmetricDifference: return this.executeSymmetricDifference(feed, opts)
+          case Operator.Union: return this.executeUnion(feed, opts)
+        }
+      case FeedType.Filter: return this.executeFilter(feed, opts)
+      case FeedType.List: return this.executeList(feed, opts)
+      case FeedType.LOL: return this.executeLOL(feed, opts)
+      case FeedType.DVM: return this.executeDVM(feed, opts)
+    }
+  }
+
+  executeDifference(feed: CompositeFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const skip = new Set<string>()
+    const events: Rumor[] = []
+
+    feed.feeds.forEach((subFeed: Feed, i: number) => {
+      this.execute(subFeed, {
+        onEvent: (event: Rumor) => {
+          if (i === 0) {
+            events.push(event)
+          } else {
+            skip.add(event.id)
+          }
+        },
+        onComplete: () => {
+          for (const event of events) {
+            if (!skip.has(event.id)) {
+              onEvent(event)
+            }
+
+            onComplete?.()
+          }
+        },
+      })
+    })
+  }
+
+  executeIntersection(feed: CompositeFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const counts = new Map<string, number>()
+    const events = new Map<string, Rumor>()
+
+    feed.feeds.forEach((subFeed: Feed, i: number) => {
+      this.execute(subFeed, {
+        onEvent: (event: Rumor) => {
+          events.set(event.id, event)
+          counts.set(event.id, inc(counts.get(event.id)))
+        },
+        onComplete: () => {
+          for (const event of events.values()) {
+            if (counts.get(event.id) === feed.feeds.length) {
+              onEvent(event)
+            }
+
+            onComplete?.()
+          }
+        },
+      })
+    })
+  }
+
+  executeSymmetricDifference(feed: CompositeFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const counts = new Map<string, number>()
+    const events = new Map<string, Rumor>()
+
+    feed.feeds.forEach((subFeed: Feed, i: number) => {
+      this.execute(subFeed, {
+        onEvent: (event: Rumor) => {
+          events.set(event.id, event)
+          counts.set(event.id, inc(counts.get(event.id)))
+        },
+        onComplete: () => {
+          for (const event of events.values()) {
+            if (counts.get(event.id) === 1) {
+              onEvent(event)
+            }
+
+            onComplete?.()
+          }
+        },
+      })
+    })
+  }
+
+  executeUnion(feed: CompositeFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const getOnComplete = this.#getCompletionTracker(onComplete)
+
+    for (const subFeed of feed.feeds) {
+      this.execute(subFeed, {onEvent, onComplete: getOnComplete})
+    }
+  }
+
+  executeFilter(feed: FilterFeed, opts: ExecuteOpts) {
+    this.opts.reqFilters([this.#compileFilter(feed.filter)], opts)
+  }
+
+  executeList(feed: ListFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const getOnComplete = this.#getCompletionTracker(onComplete)
+
+    this.opts.reqFilters(getIdFilters([feed.address]), {
+      onComplete: getOnComplete(),
+      onEvent: (event: Rumor) => {
+        this.opts.reqFilters(this.#getFiltersFromTags(Tags.fromEvent(event)), {
+          onEvent,
+          onComplete: getOnComplete(),
+        })
+      },
+    })
+  }
+
+  executeLOL(feed: LOLFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const getOnComplete = this.#getCompletionTracker(onComplete)
+
+    this.opts.reqFilters(getIdFilters([feed.address]), {
+      onComplete: getOnComplete(),
+      onEvent: (event: Rumor) => {
+        const addresses = Tags.fromEvent(event).values("a").valueOf()
+
+        this.opts.reqFilters(getIdFilters(addresses), {
+          onComplete: getOnComplete(),
+          onEvent: (event: Rumor) => {
+            this.opts.reqFilters(this.#getFiltersFromTags(Tags.fromEvent(event)), {
+              onEvent,
+              onComplete: getOnComplete(),
+            })
+          },
+        })
+      },
+    })
+  }
+
+  executeDVM(feed: DVMFeed, {onEvent, onComplete}: ExecuteOpts) {
+    const getOnComplete = this.#getCompletionTracker(onComplete)
+
+    this.opts.reqDvm(feed, {
+      onComplete: getOnComplete(),
+      onEvent: (event: Rumor) => {
+        this.opts.reqFilters(this.#getFiltersFromTags(Tags.fromEvent(event)), {
+          onEvent,
+          onComplete: getOnComplete(),
+        })
+      },
+    })
+  }
+
+  #getCompletionTracker(onComplete?: () => void) {
+    let pending = 0
+
+    return () => {
+      pending += 1
+
+      return () => {
+        pending -= 1
+
+        if (pending === 0) {
+          onComplete?.()
+        }
+      }
+    }
+  }
+
+  #compileFilter({scopes, min_wot, max_wot, until_ago, since_ago, ...filter}: DynamicFilter) {
+    if (scopes && !filter.authors) {
+      filter.authors = scopes.flatMap(scope => this.opts.getPubkeysForScope(scope))
+    }
+
+    if ((!isNil(min_wot) || !isNil(max_wot)) && !filter.authors) {
+      filter.authors = this.opts.getPubkeysForWotRange(min_wot || 1, max_wot || 1)
+    }
+
+    if (!isNil(until_ago)) {
+      filter.until = now() - until_ago!
+    }
+
+    if (!isNil(since_ago)) {
+      filter.since = now() - since_ago!
+    }
+
+    return filter
+  }
+
+  #getFiltersFromTags(tags: Tags) {
+    const ttags = tags.values("t")
+    const ptags = tags.values("p")
+    const eatags = tags.filterByKey(["e", "a"]).values()
+    const filters: Filter[] = []
+
+    if (ttags.exists()) {
+      filters.push({"#t": ttags.valueOf()})
+    }
+
+    if (ptags.exists()) {
+      filters.push({authors: ptags.valueOf()})
+    }
+
+    if (eatags.exists()) {
+      for (const filter of getIdFilters(eatags.valueOf())) {
+        filters.push(filter)
+      }
+    }
+
+    return filters
+  }
+}

packages/custom-feeds/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@coracle.social/custom-feeds",
+  "version": "0.0.1",
+  "author": "hodlbod",
+  "license": "MIT",
+  "description": "Utilities for building dynamic nostr feeds.",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "files": [
+    "build"
+  ],
+  "types": "./build/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./build/index.d.ts",
+      "import": "./build/index.mjs",
+      "require": "./build/index.cjs"
+    }
+  },
+  "scripts": {
+    "pub": "npm run lint && npm run build && npm publish",
+    "build": "gts clean && tsc-multi",
+    "lint": "gts lint",
+    "fix": "gts fix"
+  },
+  "devDependencies": {
+    "gts": "^5.0.1",
+    "tsc-multi": "^1.1.0",
+    "typescript": "~5.1.6"
+  },
+  "dependencies": {
+    "@coracle.social/util": "*"
+  }
+}

packages/custom-feeds/tsc-multi.json

@@ -0,0 +1,7 @@
+{
+  "targets": [
+    {"extname": ".cjs", "module": "commonjs"},
+    {"extname": ".mjs", "module": "esnext", "moduleResolution": "node"}
+  ],
+  "projects": ["tsconfig.json"]
+}

packages/custom-feeds/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../node_modules/gts/tsconfig-google.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "outDir": "build",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "lib": ["esnext", "dom", "dom.iterable"]
+  },
+  "include": ["**/*.ts"]
+}

packages/lib/Tools.ts

@@ -5,6 +5,8 @@ import {bech32, utf8} from "@scure/base"
 
 export type Nil = null | undefined
 
+export const isNil = (x: any) => [null, undefined].includes(x)
+
 // Regular old utils
 
 export const now = () => Math.round(Date.now() / 1000)

packages/lib/package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "@scure/base": "^1.1.6",
+    "events": "^3.3.0",
     "throttle-debounce": "^5.0.0"
   }
 }