More re-work of Tags

src/main.ts

@@ -1,8 +1,12 @@
+export * from "./util/misc"
 export * from "./util/nostr"
 export * from "./util/LRUCache"
 export * from "./util/Deferred"
 export * from "./util/Emitter"
 export * from "./util/Queue"
+export * from "./util/Tag"
+export * from "./util/Tags"
+export * from "./util/Fluent"
 export * from "./connect/Socket"
 export * from "./connect/Connection"
 export * from "./connect/ConnectionMeta"
@@ -13,3 +17,7 @@ export * from "./connect/target/Plex"
 export * from "./connect/target/Relay"
 export * from "./connect/target/Relays"
 export * from "./connect/target/Multi"
+export * from "./target/Plex"
+export * from "./target/Relay"
+export * from "./target/Relays"
+export * from "./target/Multi"

src/nostr/Router.ts

@@ -0,0 +1,218 @@
+
+/*
+  Smart relay selection
+
+  From Mike Dilger:
+
+  1) Other people's write relays — pull events from people you follow,
+     including their contact lists
+  2) Other people's read relays — push events that tag them (replies or just tagging).
+     However, these may be authenticated, use with caution
+  3) Your write relays —- write events you post to your microblog feed for the
+     world to see.  ALSO write your contact list.  ALSO read back your own contact list.
+  4) Your read relays —- read events that tag you.  ALSO both write and read
+     client-private data like client configuration events or anything that the world
+     doesn't need to see.
+  5) Advertise relays — write and read back your own relay list
+*/
+
+export type RouterOptions = {
+  hintLimit: number
+  getUserPubkey: () => string[][]
+  getGroupRelayTags: (address: string) => string[][]
+  getPubkeyRelayTags: (pubkey: string) => string[][]
+  getRelayQuality: (url: string) => number
+}
+
+export class Router {
+  constructor(readonly options: RouterOptions) {}
+
+  FetchUserDMs = () => new RouterScenario(() => {
+    const tags = Tags.from(this.options.getPubkeyRelayTags(this.options.getUserPubkey()))
+
+    return tags.
+  })
+}
+
+
+export class RouterScenario {
+  constructor(readonly getAllHints) {}
+
+  getHints = (limit: number) => this.getAllHints().slice(0, limit)
+}
+
+Router.getHints(
+
+export const selectHints = (hints: Iterable<string>, limit: number = null) => {
+  const {FORCE_RELAYS} = env.get()
+  const seen = new Set()
+  const ok = []
+  const bad = []
+
+  if (!limit) {
+    limit = getSetting("relay_limit")
+  }
+
+  for (const url of FORCE_RELAYS.length > 0 ? FORCE_RELAYS : hints) {
+    if (seen.has(url)) {
+      continue
+    }
+
+    seen.add(url)
+
+    // Skip relays that just shouldn't ever be published
+    if (!isShareableRelay(url)) {
+      continue
+    }
+
+    // Filter out relays that appear to be broken or slow
+    if (relayIsLowQuality(url)) {
+      bad.push(url)
+    } else {
+      ok.push(url)
+    }
+
+    if (ok.length > limit) {
+      break
+    }
+  }
+
+  // If we don't have enough hints, use the broken ones
+  const result = ok.concat(bad).slice(0, limit)
+
+  if (result.length === 0) {
+    warn("No results returned from selectHints")
+  }
+
+  return result
+}
+
+export const selectHintsWithFallback = (hints: Iterable<string> = null, limit = null) =>
+  selectHints(chain(hints || [], getUserRelayUrls(RelayMode.Read), env.get().DEFAULT_RELAYS), limit)
+
+export class HintSelector {
+  constructor(
+    readonly generateHints,
+    readonly hintsLimit = null,
+  ) {}
+
+  limit = hintsLimit => new HintSelector(this.generateHints, hintsLimit)
+
+  getHints = (...args) =>
+    selectHints(this.generateHints(...args), this.hintsLimit || getSetting("relay_limit"))
+}
+
+export const hintSelector = (generateHints: (...args: any[]) => Iterable<string>) => {
+  const selector = new HintSelector(generateHints)
+  const getHints = selector.getHints
+
+  ;(getHints as any).limit = selector.limit
+
+  return getHints as typeof getHints & {limit: typeof selector.limit}
+}
+
+export const getPubkeyHints = hintSelector(function* (pubkey: string, mode: RelayMode) {
+  yield* getPubkeyRelayUrls(pubkey, mode)
+})
+
+export const getPubkeyHint = (pubkey: string): string =>
+  first(getPubkeyHints(1, pubkey, "write")) || ""
+
+export const getUserHints = hintSelector(function* (mode: RelayMode) {
+  yield* getUserRelayUrls(mode)
+})
+
+export const getUserHint = (pubkey: string): string => first(getUserHints(1, "write")) || ""
+
+export const getEventHints = hintSelector(function* (event: Event) {
+  for (const address of Tags.from(event).circles().all()) {
+    yield* getGroupHints(address)
+  }
+
+  yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Write)
+  yield* event.seen_on.filter(isShareableRelay)
+})
+
+export const getEventHint = (event: Event) => first(getEventHints.limit(1).getHints(event)) || ""
+
+// If we're looking for an event's children, the read relays the author has
+// advertised would be the most reliable option, since well-behaved clients
+// will write replies there.
+export const getReplyHints = hintSelector(function* (event) {
+  for (const address of Tags.from(event).circles().all()) {
+    yield* getGroupHints(address)
+  }
+
+  yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
+})
+
+// If we're looking for an event's parent, tags are the most reliable hint,
+// but we can also look at where the author of the note reads from
+export const getParentHints = hintSelector(function* (event) {
+  yield* Tags.from(event).getReplyHints()
+  yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
+})
+
+export const getRootHints = hintSelector(function* (event) {
+  yield* Tags.from(event).getRootHints()
+  yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
+})
+
+// If we're replying or reacting to an event, we want the author to know, as well as
+// anyone else who is tagged in the original event or the reply. Get everyone's read
+// relays. Limit how many per pubkey we publish to though. We also want to advertise
+// our content to our followers, so publish to our write relays as well.
+export const getPublishHints = hintSelector(function* (event: Event) {
+  for (const address of Tags.from(event).circles().all()) {
+    yield* getGroupHints(address)
+  }
+
+  const pubkeys = Tags.from(event).type("p").values().all()
+  const hintGroups = pubkeys.map(pubkey => getPubkeyRelayUrls(pubkey, RelayMode.Read))
+  const authorRelays = getPubkeyRelayUrls(event.pubkey, RelayMode.Write)
+
+  yield* mergeHints([...hintGroups, authorRelays, getUserHints(RelayMode.Write)])
+})
+
+export const getInboxHints = hintSelector(function* (pubkeys: string[]) {
+  yield* mergeHints(pubkeys.map(pk => getPubkeyHints(pk, "read")))
+})
+
+export const getGroupHints = hintSelector(function* (address: string) {
+  yield* getGroupRelayUrls(address)
+  yield* getPubkeyHints(Naddr.fromTagValue(address).pubkey)
+})
+
+export const getGroupPublishHints = (addresses: string[]) => {
+  const urls = mergeHints(addresses.map(getGroupRelayUrls))
+
+  return urls.length === 0 ? getUserHints("write") : urls
+}
+
+export const mergeHints = (groups: string[][], limit: number = null) => {
+  const scores = {} as Record<string, any>
+
+  for (const hints of groups) {
+    hints.forEach((hint, i) => {
+      const score = 1 / (i + 1) / hints.length
+
+      if (!scores[hint]) {
+        scores[hint] = {score: 0, count: 0}
+      }
+
+      scores[hint].score += score
+      scores[hint].count += 1
+    })
+  }
+
+  // Use the log-sum-exp and a weighted sum
+  for (const score of Object.values(scores)) {
+    const weight = Math.log(groups.length / score.count)
+
+    score.score = weight + Math.log1p(Math.exp(score.score - score.count))
+  }
+
+  return sortBy(([hint, {score}]) => score, Object.entries(scores))
+    .map(nth(0))
+    .slice(0, limit || getSetting("relay_limit"))
+}

src/util/Fluent.ts

@@ -0,0 +1,49 @@
+import {last} from './misc'
+
+export class Fluent<T> {
+  constructor(readonly parts: T[]) {}
+
+  static from<T>(parts: Iterable<T>) {
+    return new Fluent<T>(Array.from(parts))
+  }
+
+  clone<K extends Fluent<T>>(this: K, parts: T[]): K {
+    return new (this.constructor as { new (parts: T[]): K })(parts)
+  }
+
+  *[Symbol.iterator]() {
+    for (const x of this.parts) {
+      yield x
+    }
+  }
+
+  first = () => Array.from(this.parts)[0]
+
+  nth = (i: number) => Array.from(this.parts)[i]
+
+  last = () => last(Array.from(this.parts))
+
+  count = () => Array.from(this.parts).length
+
+  exists = () => Array.from(this.parts).length > 0
+
+  every = (f: (t: T) => boolean) => Array.from(this.parts).every(f)
+
+  some = (f: (t: T) => boolean) => Array.from(this.parts).some(f)
+
+  find = (f: (t: T) => boolean) => Array.from(this.parts).find(f)
+
+  uniq = () => this.clone(Array.from(new Set(this.parts)))
+
+  slice = (a: number, b?: number) => this.clone(Array.from(this.parts).slice(a, b))
+
+  take = (n: number) => this.slice(0, n)
+
+  drop = (n: number) => this.slice(n)
+
+  filter = (f: (t: T) => boolean) => this.clone(Array.from(this.parts).filter(f))
+
+  reject = (f: (t: T) => boolean) => this.clone(Array.from(this.parts).filter(t => !f(t)))
+
+  map = <U>(f: (t: T) => U) => new Fluent(Array.from(this.parts).map(f))
+}

src/util/Tag.ts

@@ -1,25 +1,19 @@
-export class Tag {
+import type {OmitStatics} from './misc'
-  constructor(readonly parts: string[]) {}
+import {last} from './misc'
+import {Fluent} from './Fluent'
 
-  key() {
+export class Tag extends (Fluent<string> as OmitStatics<typeof Fluent<string>, 'from'>) {
-    return this.parts[0]
+  static from(parts: Iterable<string>) {
+    return new Tag(Array.from(parts))
   }
 
-  val() {
+  key = () => this.parts[0]
-    return this.parts[1]
-  }
 
-  mark() {
+  value = () => this.parts[1]
-    return this.parts.slice(0, 2).slice(-1)[0]
-  }
 
-  nth(n: number) {
+  mark = () => last(this.parts.slice(2))
-    return this.parts[n]
-  }
 
-  *[Symbol.iterator]() {
+  entry = () => this.parts.slice(0, 2)
-    for (const x of this.parts) {
+
-      yield x
+  append = (s: string) => Tag.from(this.parts.concat(s))
-    }
-  }
 }

src/util/Tags.ts

@@ -1,161 +1,146 @@
-export class Fluent<T> {
+import type {Event} from 'nostr-tools'
-  ItemClass?: Fluent<T>
+import {Tag} from './Tag'
+import {Fluent} from './Fluent'
+import type {OmitStatics} from './misc'
+import {isIterable, uniq} from './misc'
+import {isShareableRelay} from './nostr'
+import {isCommunityAddress, isGroupAddress, isCommunityOrGroupAddress} from './kinds'
 
-  constructor(value: T[]) {
+export class Tags extends (Fluent<Tag> as OmitStatics<typeof Fluent<Tag>, 'from'>) {
-    this.value = value.filter(identity)
+  static from(p: Iterable<Tag | string[]>) {
+    return new Tags(Array.from(p).map(Tag.from))
   }
 
-  static create(value: T[]) {
+  static fromEvent(event: Event) {
-    this.value = value.filter(identity)
+    return Tags.from(event.tags)
   }
 
-  item(item: T) {
+  static fromEvents(events: Event[]) {
-    const {ItemClass} = this.constructor
+    return Tags.from(events.flatMap((e: Event) => e?.tags))
-
-    return ItemClass ? ItemClass.create(item) : item
   }
 
-  valueOf = () => this.value
+  // General purpose filters
 
-  count = () => this.value.length
+  whereKey = (key: string) => this.filter(t => t.key() === key)
 
-  exists = () => this.value.length > 0
+  whereValue = (value: string) => this.filter(t => t.value() === value)
 
-  f = <U>(f: (t: T) => U) => f(this.value)
+  whereMark = (mark: string) => this.filter(t => t.mark() === mark)
 
-  any = (f: (t: T) => boolean) => this.value.any(f)
+  // General purpose methods that return a list of values
 
-  every = (f: (t: T) => boolean) => this.value.every(f)
+  keys = () => new Fluent(this.parts.map(t => t.key()))
 
-  some = (f: (t: T) => boolean) => this.value.some(f)
+  values = () => new Fluent(this.parts.map(t => t.value()))
 
-  first = () => this.item(this.values[0])
+  marks = () => new Fluent(this.parts.map(t => t.mark()))
 
-  nth = (i: number) => this.item(this.values[i])
+  entries = () => new Fluent(this.parts.map(t => t.entry()))
-
-  last = () => this.item(last(this.values))
-
-  find = (f: (t: T) => boolean) => this.item(this.value.find(f))
-
-  filter = (f: (t: T) => boolean) => this.constructor.create(this.value.filter(f))
-
-  reject = (f: (t: T) => boolean) => this.constructor.create(this.value.filter(t => !f(t)))
 }
 
-export class Tag extends Fluent<string[]> {
+export type CoercibleToTags = Event | Iterable<Event> | Tags | Tag | Tag[] | string[] | Iterable<string[]>
-  type = () => this.value[0]
 
-  value = () => this.value[1]
+export const coerceToTags = (x: CoercibleToTags) => {
+  const xs = isIterable(x) ? Array.from(x as Iterable<any>) : [x]
 
-  mark = () => last(this.value)
+  if (xs.length === 0) {
+    return new Tags(xs)
+  }
+
+  if (xs[0] instanceof Event) {
+    return Tags.fromEvents(xs)
+  }
+
+  if (xs[0] instanceof Array) {
+    return Tags.from(xs)
+  }
+
+  if (typeof xs[0] === 'string') {
+    return Tags.from([xs])
+  }
+
+  throw new Error('Received invalid value to coerceToTags: ${x}')
 }
 
-export class Tags extends Fluent<string[][]> {
+export const getRelays = (x: CoercibleToTags) =>
-  ItemClass: Tag
+  uniq(Array.from(coerceToTags(x)).flatMap((t: Tag) => Array.from(t)).filter(isShareableRelay))
 
-  static from (e: Event | Event[]) {
+export const getTopics = (x: CoercibleToTags) =>
-    const events = Array.isArray(e) ? e : [e]
+  Array.from(coerceToTags(x).whereKey("t").values()).map((t: string) => t.replace(/^#/, ""))
 
-    return new Tags(events.flatMap(e => e?.tags))
+export const getPubkeys = (x: CoercibleToTags) =>
-  }
+  Array.from(coerceToTags(x).whereKey("p").values())
 
-  where(conditions: Record<string, (x: any) => boolean>) {
+export const getUrls = (x: CoercibleToTags) =>
-    return this.filter(t => {
+  Array.from(coerceToTags(x).whereKey("r").values())
-      const tag = new Tag(t)
 
-      for ([k, f] of Object.entries(conditions)) {
+export const getAncestorsLegacy = (x: CoercibleToTags) => {
-        if (!f(tag[k]())) {
+  // Legacy only supports e tags. Normalize their length to 3
-          return false
+  const eTags = Tags.from(
-        }
+    coerceToTags(x).whereKey("e").map((t: Tag) => {
-      }
+      while (t.count() < 3) {
-
+        t.append("")
-      return true
-    })
-  }
-
-  whereEq(conditions: Record<string, any>) {
-    return this.filter(t => {
-      const tag = new Tag(t)
-
-      for ([k, v] of Object.entries(conditions)) {
-        v = Array.isArray(v) ? v : [v]
-
-        if (!v.includes(tag[k]())) {
-          return false
-        }
-      }
-
-      return true
-    })
-  }
-
-  value = () => this.value.find(t => t[1])
-
-  values = () => this.value.map(t => t[1])
-
-  relays = () => uniq(flatten(this.value).filter(isShareableRelay))
-
-  topics = () => this.whereEq({type: "t"}).values().map((t: string) => t.replace(/^#/, ""))
-
-  pubkeys = () => this.whereEq({type: "p"}).values()
-
-  urls = () => this.whereEq({type: "r"}).values()
-
-  getDict() {
-    const meta: Record<string, string> = {}
-
-    for (const [k, v] of this.value) {
-      if (!meta[k]) {
-        meta[k] = v
-      }
-    }
-
-    return meta
-  }
-
-  getAncestorsLegacy() {
-    // Legacy only supports e tags. Normalize their length to 3
-    const eTags = this.whereEq({type: "e"}).map(t => {
-      while (t.length < 3) {
-        t.push("")
       }
 
       return t.slice(0, 3)
     })
+  )
 
-    return {
+  return {
-      roots: eTags.count() > 1 ? new Tags([eTags.first()]) : new Tags([]),
+    roots: eTags.slice(0, 1),
-      replies: new Tags([eTags.last()]),
+    replies: eTags.slice(-1),
-      mentions: new Tags(eTags.all().slice(1, -1)),
+    mentions: eTags.slice(1, -1),
-    }
   }
-
-  getAncestors(type = null) {
-    // If we have a mark, we're not using the legacy format
-    if (!this.any(t => t.length === 4 && ["reply", "root", "mention"].includes(last(t)))) {
-      return this.getAncestorsLegacy()
-    }
-
-    const tags = new Tags(this.whereEq({type: type || ["}a", "e"]).all().filter(t => !String(t[1]).startsWith('34550:')))
-
-    return {
-      roots: new Tags(tags.mark('root').take(3).all()),
-      replies: new Tags(tags.mark('reply').take(3).all()),
-      mentions: new Tags(tags.mark('mention').take(3).all()),
-    }
-  }
-
-  roots = (type = null) => this.getAncestors(type).roots
-
-  replies = (type = null) => this.getAncestors(type).replies
-
-  communities = () => this.whereEq({type: "a"}).values().filter(a => a.startsWith('34550:'))
-
-  getReply = (type = null) => this.replies(type).values().first()
-
-  getRoot = (type = null) => this.roots(type).values().first()
-
-  getReplyHints = (type = null) => this.replies(type).relays().all()
-
-  getRootHints = (type = null) => this.roots(type).relays().all()
 }
+
+type GetAncestorsReturn = {
+  roots: Tags
+  replies: Tags
+  mentions: Tags
+}
+
+export const getAncestors = (x: CoercibleToTags, key?: string): GetAncestorsReturn => {
+  const tags = coerceToTags(x)
+
+  // If we have a mark, we're not using the legacy format
+  if (!tags.some((t: Tag) => t.count() === 4 && ["reply", "root", "mention"].includes(t.mark()))) {
+    return getAncestorsLegacy(tags)
+  }
+
+  const eTags = tags.whereKey("e")
+  const aTags = tags.whereKey("a").reject((t: Tag) => isCommunityOrGroupAddress(t.value()))
+  const allTags = coerceToTags([...eTags, ...aTags])
+
+  return {
+    roots: allTags.whereMark('root').take(3),
+    replies: allTags.whereMark('reply').take(3),
+    mentions: allTags.whereMark('mention').take(3),
+  }
+}
+
+export const getRoots = (x: CoercibleToTags, key?: string) =>
+  getAncestors(x, key).roots
+
+export const getReplies = (x: CoercibleToTags, key?: string) =>
+  getAncestors(x, key).replies
+
+export const getGroups = (x: CoercibleToTags) =>
+  coerceToTags(x).whereKey("a").values().filter(isGroupAddress)
+
+export const getCommunities = (x: CoercibleToTags) =>
+  coerceToTags(x).whereKey("a").values().filter(isCommunityAddress)
+
+export const getCommunitiesAndGroups = (x: CoercibleToTags) =>
+  coerceToTags(x).whereKey("a").values().filter(isCommunityOrGroupAddress)
+
+export const getRoot = (x: CoercibleToTags, key?: string) =>
+  getRoots(x, key).values().first()
+
+export const getReply = (x: CoercibleToTags, key?: string) =>
+  getReplies(x, key).values().first()
+
+export const getRootHints = (x: CoercibleToTags, key?: string) =>
+  getRelays(getRoots(x, key))
+
+export const getReplyHints = (x: CoercibleToTags, key?: string) =>
+  getRelays(getReplies(x, key))

src/util/kinds.ts

@@ -0,0 +1,8 @@
+export const GROUP = 35834
+export const COMMUNITY = 34550
+
+export const isGroupAddress = (a: string) => a.startsWith(`${GROUP}:`)
+
+export const isCommunityAddress = (a: string) => a.startsWith(`${COMMUNITY}:`)
+
+export const isCommunityOrGroupAddress = (a: string) => isCommunityAddress(a) || isGroupAddress(a)

src/util/misc.ts

@@ -0,0 +1,17 @@
+export const now = () => Math.round(Date.now() / 1000)
+
+export const last = <T>(xs: T[]) => xs[xs.length - 1]
+
+export const identity = <T>(x: T) => x
+
+export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
+
+export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
+
+// https://github.com/microsoft/TypeScript/issues/4628#issuecomment-1147905253
+export type OmitStatics<T, S extends string> =
+    T extends {new(...args: infer A): infer R} ?
+        {new(...args: A): R}&Omit<T, S> :
+        Omit<T, S>;
+
+export const isIterable = (x: any) => Symbol.iterator in Object(x)

src/util/nostr.ts

@@ -2,19 +2,7 @@ import type {Event} from 'nostr-tools'
 import normalizeUrl from "normalize-url"
 import {verifyEvent, getEventHash, matchFilter as nostrToolsMatchFilter} from 'nostr-tools'
 import {cached} from "./LRUCache"
-
+import {now} from './misc'
-// ===========================================================================
-// General-purpose
-
-export const now = () => Math.round(Date.now() / 1000)
-
-export const last = <T>(xs: T[]) => xs[xs.length - 1]
-
-export const identity = <T>(x: T) => x
-
-export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
-
-export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
 
 // ===========================================================================
 // Relays