Add diff utils, change how adapter is created, scope down subscribe/publish to a single relay for fastest possible completion

2025-03-24 15:30:49 -07:00
parent d7b74d2c7e
commit 88b44776d8
4 changed files with 276 additions and 112 deletions
@@ -1,6 +1,7 @@
 import {EventEmitter} from "events"
-import {on, randomId} from "@welshman/lib"
+import {on, sleep, randomId, groupBy, pushToMapKey, inc, flatten, chunk} from "@welshman/lib"
 import {SignedEvent, Filter} from "@welshman/util"
+import {TypedEmitter} from "./util.js"
 import {
  RelayMessage,
  isRelayNegErr,
@@ -8,37 +9,47 @@ import {
  RelayMessageType,
  ClientMessageType,
 } from "./message.js"
-import {AbstractAdapter, AdapterEventType} from "./adapter.js"
+import {getAdapter, AdapterContext, AbstractAdapter, AdapterEventType} from "./adapter.js"
 import {Negentropy, NegentropyStorageVector} from "./negentropy.js"
-import {TypedEmitter} from "./util.js"
+import {subscribe, SubscriptionEventType} from "./subscribe.js"
+import {publish, PublishEventType} from "./publish.js"

-export enum DiffEventType {
-  Message = "diff:event:message",
-  Error = "diff:event:error",
-  Close = "diff:event:close",
+export enum DifferenceEventType {
+  Message = "difference:event:message",
+  Error = "difference:event:error",
+  Close = "difference:event:close",
 }

-export type DiffEvents = {
-  [DiffEventType.Message]: (payload: {have: string[]; need: string[]}, url: string) => void
-  [DiffEventType.Error]: (error: string, url: string) => void
-  [DiffEventType.Close]: () => void
+export type DifferenceEvents = {
+  [DifferenceEventType.Message]: (payload: {have: string[]; need: string[]}, url: string) => void
+  [DifferenceEventType.Error]: (error: string, url: string) => void
+  [DifferenceEventType.Close]: () => void
 }

-export type DiffOptions = {
+export type DifferenceOptions = {
+  relay: string
  filter: Filter
  events: SignedEvent[]
-  adapter: AbstractAdapter
-  on?: Partial<DiffEvents>
+  context: AdapterContext
+  on?: Partial<DifferenceEvents>
 }

-export class Diff extends (EventEmitter as new () => TypedEmitter<DiffEvents>) {
+export class Difference extends (EventEmitter as new () => TypedEmitter<DifferenceEvents>) {
+  have = new Set<string>()
+  need = new Set<string>()
+
  _id = `NEG-${randomId().slice(0, 8)}`
  _unsubscriber: () => void
+  _adapter: AbstractAdapter
  _closed = false

-  constructor(readonly options: DiffOptions) {
+  constructor(readonly options: DifferenceOptions) {
    super()

+    // Set up our adapter
+    this._adapter = getAdapter(this.options.relay, this.options.context)
+
+    // Set up negentropy
    const storage = new NegentropyStorageVector()
    const neg = new Negentropy(storage, 50_000)

@@ -48,8 +59,9 @@ export class Diff extends (EventEmitter as new () => TypedEmitter<DiffEvents>) {

    storage.seal()

+    // Add listeners
    this._unsubscriber = on(
-      this.options.adapter,
+      this._adapter,
      AdapterEventType.Receive,
      async (message: RelayMessage, url: string) => {
        if (isRelayNegMsg(message)) {
@@ -58,12 +70,18 @@ export class Diff extends (EventEmitter as new () => TypedEmitter<DiffEvents>) {
          if (negid === this._id) {
            const [newMsg, have, need] = await neg.reconcile(msg)

-            this.emit(DiffEventType.Message, {have, need}, url)
+            for (const id of have) {
+              this.have.add(id)
+            }
+
+            for (const id of need) {
+              this.need.add(id)
+            }
+
+            this.emit(DifferenceEventType.Message, {have, need}, url)

            if (newMsg) {
-              this.options.adapter.send([RelayMessageType.NegMsg, this._id, newMsg])
-            } else {
-              this.close()
+              this._adapter.send([RelayMessageType.NegMsg, this._id, newMsg])
            }
          }
        }
@@ -72,7 +90,7 @@ export class Diff extends (EventEmitter as new () => TypedEmitter<DiffEvents>) {
          const [_, negid, msg] = message

          if (negid === this._id) {
-            this.emit(DiffEventType.Error, msg, url)
+            this.emit(DifferenceEventType.Error, msg, url)
          }
        }
      },
@@ -81,22 +99,159 @@ export class Diff extends (EventEmitter as new () => TypedEmitter<DiffEvents>) {
    // Register listeners
    if (this.options.on) {
      for (const [k, listener] of Object.entries(this.options.on)) {
-        this.on(k as keyof DiffEvents, listener)
+        this.on(k as keyof DifferenceEvents, listener)
      }
    }

    neg.initiate().then((msg: string) => {
-      this.options.adapter.send([ClientMessageType.NegOpen, this._id, this.options.filter, msg])
+      this._adapter.send([ClientMessageType.NegOpen, this._id, this.options.filter, msg])
    })
  }

  close() {
    if (this._closed) return

-    this.options.adapter.send([ClientMessageType.NegClose, this._id])
-    this.emit(DiffEventType.Close)
+    this._adapter.send([ClientMessageType.NegClose, this._id])
+    this.emit(DifferenceEventType.Close)
    this.removeAllListeners()
+    this._adapter.cleanup()
    this._unsubscriber()
    this._closed = true
  }
 }
+
+// diff is a shortcut for diffing multiple filters across multiple relays
+
+export type DiffOptions = {
+  relays: string[]
+  filters: Filter[]
+  events: SignedEvent[]
+  context: AdapterContext
+}
+
+export type DiffItem = {
+  relay: string
+  have: Set<string>
+  need: Set<string>
+}
+
+export const diff = async ({relays, filters, ...options}: DiffOptions) => {
+  const diffs = flatten(
+    await Promise.all(
+      relays.flatMap(async relay => {
+        return await Promise.all(
+          filters.map(
+            async filter =>
+              new Promise<DiffItem>((resolve, reject) => {
+                const diff = new Difference({relay, filter, ...options})
+
+                diff.on(DifferenceEventType.Close, () => {
+                  resolve({relay, have: diff.have, need: diff.need})
+                  diff.close()
+                })
+
+                diff.on(DifferenceEventType.Error, (url, message) => {
+                  reject(message)
+                  diff.close()
+                })
+
+                sleep(30_000).then(() => {
+                  reject("timeout")
+                  diff.close()
+                })
+              }),
+          ),
+        )
+      }),
+    ),
+  )
+
+  return Array.from(groupBy(diff => diff.relay, diffs).entries()).map(([relay, diffs]) => {
+    const have = new Set<string>()
+    const need = new Set<string>()
+
+    for (const diff of diffs) {
+      for (const id of diff.have) {
+        have.add(id)
+      }
+
+      for (const id of diff.need) {
+        need.add(id)
+      }
+    }
+
+    return {relay, have: Array.from(have), need: Array.from(need)}
+  })
+}
+
+// Pull diffs multiple arrays and fetches missing events
+
+export type PullOptions = DiffOptions
+
+export const pull = async ({context, ...options}: PullOptions) => {
+  const countById = new Map<string, number>()
+  const idsByRelay = new Map<string, string[]>()
+
+  for (const {relay, need} of await diff({context, ...options})) {
+    for (const id of need) {
+      const count = countById.get(id) || 0
+
+      // Reduce, but don't completely eliminate duplicates, just in case a relay
+      // won't give us what we ask for.
+      if (count < 2) {
+        pushToMapKey(idsByRelay, relay, id)
+        countById.set(id, inc(count))
+      }
+    }
+  }
+
+  const result: SignedEvent[] = []
+
+  await Promise.all(
+    Array.from(idsByRelay.entries()).map(([relay, allIds]) => {
+      return Promise.all(
+        chunk(500, allIds).map(ids => {
+          return new Promise<void>(resolve => {
+            const sub = subscribe({relay, filter: {ids}, context, autoClose: true})
+
+            sub.on(SubscriptionEventType.Close, resolve)
+            sub.on(SubscriptionEventType.Event, event => result.push(event))
+          })
+        }),
+      )
+    }),
+  )
+
+  return result
+}
+
+// Push diffs multiple relays and publishes missing events
+
+export type PushOptions = DiffOptions
+
+export const push = async ({context, events, ...options}: PushOptions) => {
+  const relaysById = new Map<string, string[]>()
+
+  for (const {relay, have} of await diff({context, events, ...options})) {
+    for (const id of have) {
+      pushToMapKey(relaysById, id, relay)
+    }
+  }
+
+  await Promise.all(
+    events.map(async event => {
+      const relays = relaysById.get(event.id)
+
+      if (relays) {
+        await Promise.all(
+          relays.map(
+            relay =>
+              new Promise<void>(resolve => {
+                publish({event, relay, context}).on(PublishEventType.Complete, resolve)
+              }),
+          ),
+        )
+      }
+    }),
+  )
+}