Add negentropy support to executor

packages/app/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/content/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/dvm/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/feeds/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/lib/src/Emitter.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from 'events'
 
 export class Emitter extends EventEmitter {
-  emit(type: string | number, ...args: any[]) {
+  emit(type: string, ...args: any[]) {
     const a = super.emit(type, ...args)
     const b = super.emit('*', type, ...args)
 

packages/lib/src/Worker.ts

@@ -44,7 +44,7 @@ export class Worker<T> {
 
   #enqueueWork = () => {
     if (!this.timeout && this.buffer.length > 0) {
-      this.timeout = setTimeout(this.#doWork, 50)
+      this.timeout = setTimeout(this.#doWork, 50) as unknown as number
     }
   }
 

packages/lib/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/net/package.json

@@ -23,10 +23,12 @@
     "pub": "npm run lint && npm run build && npm publish",
     "build": "gts clean && tsc-multi",
     "lint": "gts lint",
-    "fix": "gts fix"
+    "fix": "gts fix",
+    "test": "mocha"
   },
   "devDependencies": {
     "gts": "^5.0.1",
+    "mocha": "^10.7.3",
     "tsc-multi": "^1.1.0",
     "typescript": "~5.1.6"
   },

packages/net/src/Executor.ts

@@ -3,6 +3,7 @@ import type {Emitter} from '@welshman/lib'
 import type {SignedEvent, Filter} from '@welshman/util'
 import type {Message} from './Socket'
 import type {Connection} from './Connection'
+import {Negentropy, NegentropyStorageVector} from './Negentropy'
 
 export type Target = Emitter & {
   connections: Connection[]
@@ -12,10 +13,13 @@ export type Target = Emitter & {
 
 type EventCallback = (url: string, event: SignedEvent) => void
 type EoseCallback = (url: string) => void
+type CloseCallback = () => void
 type OkCallback = (url: string, id: string, ...extra: any[]) => void
 type ErrorCallback = (url: string, id: string, ...extra: any[]) => void
+type DiffMessageCallback = (url: string, {have, need}: {have: string[], need: string[]}) => void
 type SubscribeOpts = {onEvent?: EventCallback, onEose?: EoseCallback}
 type PublishOpts = {verb?: string, onOk?: OkCallback, onError?: ErrorCallback}
+type DiffOpts = {onError?: ErrorCallback, onMessage?: DiffMessageCallback, onClose?: CloseCallback}
 
 const createSubId = (prefix: string) => [prefix, Math.random().toString().slice(2, 10)].join('-')
 
@@ -50,11 +54,11 @@ export class Executor {
 
     return {
       unsubscribe: () => {
-        if (!closed) {
-          this.target.send("CLOSE", id)
-          this.target.off('EVENT', eventListener)
-          this.target.off('EOSE', eoseListener)
-        }
+        if (closed) return
+
+        this.target.send("CLOSE", id)
+        this.target.off('EVENT', eventListener)
+        this.target.off('EOSE', eoseListener)
 
         closed = true
       },
@@ -86,5 +90,60 @@ export class Executor {
       }
     }
   }
-}
 
+  diff(filter: Filter, events: SignedEvent[], {onMessage, onError, onClose}: DiffOpts = {}) {
+    let closed = false
+
+    const id = createSubId('NEG')
+    const storage = new NegentropyStorageVector()
+    const neg = new Negentropy(storage, 50_000)
+
+    for (const event of events) {
+      storage.insert(event.created_at, event.id)
+    }
+
+    storage.seal()
+
+    const msgListener = async (url: string, negid: string, msg: string) => {
+      if (negid === id) {
+        const [newMsg, have, need] = await neg.reconcile(msg)
+
+        onMessage?.(url, {have, need})
+
+        if (newMsg) {
+          this.target.send('NEG-MSG', id, newMsg)
+        } else {
+          close()
+        }
+      }
+    }
+
+    const errListener = (url: string, negid: string, msg: string) => {
+      if (negid === id) {
+        onError?.(url, msg)
+      }
+    }
+
+    const close = () => {
+      if (closed) return
+
+      this.target.send('NEG-CLOSE', id)
+      this.target.off('NEG-MSG', msgListener)
+      this.target.off('NEG-ERR', errListener)
+
+      closed = true
+      onClose?.()
+    }
+
+    this.target.on('NEG-MSG', msgListener)
+    this.target.on('NEG-ERR', errListener)
+
+    neg.initiate().then((msg: string) => {
+      this.target.send("NEG-OPEN", id, filter, msg)
+    })
+
+    return {
+      unsubscribe: close,
+    }
+  }
+}

packages/net/src/Subscribe.ts

@@ -326,7 +326,9 @@ export const executeSubscriptionBatched = (() => {
 
   return (sub: Subscription) => {
     subs.push(sub)
-    timeouts.push(setTimeout(executeAll, Math.max(16, sub.request.delay!)))
+    timeouts.push(
+      setTimeout(executeAll, Math.max(16, sub.request.delay!)) as unknown as number
+    )
   }
 })()
 

packages/net/src/index.ts

@@ -7,6 +7,7 @@ export * from "./Publish"
 export * from "./Socket"
 export * from "./Subscribe"
 export * from "./Tracker"
+export * from "./target/Echo"
 export * from "./target/Multi"
 export * from "./target/Plex"
 export * from "./target/Relay"

packages/net/src/target/Echo.ts

@@ -0,0 +1,16 @@
+import {Emitter} from '@welshman/lib'
+import type {Message} from '../Socket'
+
+export class Echo extends Emitter {
+  get connections() {
+    return []
+  }
+
+  send(...payload: Message) {
+    this.emit(...payload)
+  }
+
+  cleanup = () => {
+    this.removeAllListeners()
+  }
+}

packages/net/test/Executor.test.cjs

@@ -0,0 +1,37 @@
+const assert = require('assert')
+const {setContext} = require('@welshman/lib')
+const {Executor, Echo, getDefaultNetContext} = require('@welshman/net')
+
+const event = {
+  "content": "👀",
+  "created_at":1727389659,
+  "id": "acaee505278bd8842ab6df906bf39bb143cf9905f36453c9bc13554cf5006e2d",
+  "kind": 1,
+  "pubkey": "6e468422dfb74a5738702a8823b9b28168abab8655faacb6853cd0ee15deee93",
+  "sig": "3aa512e2dbcd704bd287e6a35eaa8c4388606d553d385e482cc94d536eea25585731c36da6658c941c4668a473860a12d75ba588ca50470df09f8827e164e640",
+  "tags": [
+    ["p","460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"],
+    ["e","d423aa132e5dc741ddecbac5e67515b6fd900c2559058397ec7fd860b3d77ea6","wss://nostr.mom","root"]
+  ]
+}
+
+setContext({net: getDefaultNetContext()})
+
+describe('myFunction', () => {
+  const target = new Echo()
+  const executor = new Executor(target)
+
+  it('should return the correct result', done => {
+    const messages = []
+    const neg = executor.diff({kinds: [1]}, [event], {})
+
+    target.on('*', (...message) => messages.push(message))
+
+    setTimeout(() => {
+      neg.unsubscribe()
+      assert.equal(messages[0][0], 'NEG-OPEN')
+      assert.equal(messages[1][0], 'NEG-CLOSE')
+      done()
+    }, 10)
+  })
+})

packages/net/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/signer/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/store/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom", "dom.iterable"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }

packages/util/tsconfig.json

@@ -7,5 +7,5 @@
     "skipLibCheck": true,
     "lib": ["esnext", "dom"]
   },
-  "include": ["**/*.ts"]
+  "include": ["src/**/*.ts"]
 }