// (C) 2023 Doug Hoyte. MIT license /* eslint-disable */ // @ts-nocheck const PROTOCOL_VERSION = 0x61 // Version 1 const ID_SIZE = 32 const FINGERPRINT_SIZE = 16 const Mode = { Skip: 0, Fingerprint: 1, IdList: 2, } class WrappedBuffer { constructor(buffer) { this._raw = new Uint8Array(buffer || 512) this.length = buffer ? buffer.length : 0 } unwrap() { return this._raw.subarray(0, this.length) } get capacity() { return this._raw.byteLength } extend(buf) { if (buf._raw) buf = buf.unwrap() if (typeof buf.length !== "number") throw Error("bad length") const targetSize = buf.length + this.length if (this.capacity < targetSize) { const oldRaw = this._raw const newCapacity = Math.max(this.capacity * 2, targetSize) this._raw = new Uint8Array(newCapacity) this._raw.set(oldRaw) } this._raw.set(buf, this.length) this.length += buf.length } shift() { const first = this._raw[0] this._raw = this._raw.subarray(1) this.length-- return first } shiftN(n = 1) { const firstSubarray = this._raw.subarray(0, n) this._raw = this._raw.subarray(n) this.length -= n return firstSubarray } } function decodeVarInt(buf) { let res = 0 while (1) { if (buf.length === 0) throw Error("parse ends prematurely") const byte = buf.shift() res = (res << 7) | (byte & 127) if ((byte & 128) === 0) break } return res } function encodeVarInt(n) { if (n === 0) return new WrappedBuffer([0]) const o = [] while (n !== 0) { o.push(n & 127) n >>>= 7 } o.reverse() for (let i = 0; i < o.length - 1; i++) o[i] |= 128 return new WrappedBuffer(o) } function getByte(buf) { return getBytes(buf, 1)[0] } function getBytes(buf, n) { if (buf.length < n) throw Error("parse ends prematurely") return buf.shiftN(n) } class Accumulator { constructor() { this.setToZero() if (typeof window === "undefined") { // node.js const crypto = require("crypto") this.sha256 = async slice => new Uint8Array(crypto.createHash("sha256").update(slice).digest()) } else { // browser this.sha256 = async slice => new Uint8Array(await crypto.subtle.digest("SHA-256", slice)) } } setToZero() { this.buf = new Uint8Array(ID_SIZE) } add(otherBuf) { let currCarry = 0, nextCarry = 0 const p = new DataView(this.buf.buffer) const po = new DataView(otherBuf.buffer) for (let i = 0; i < 8; i++) { const offset = i * 4 const orig = p.getUint32(offset, true) const otherV = po.getUint32(offset, true) let next = orig next += currCarry next += otherV if (next > 0xffffffff) nextCarry = 1 p.setUint32(offset, next & 0xffffffff, true) currCarry = nextCarry nextCarry = 0 } } negate() { const p = new DataView(this.buf.buffer) for (let i = 0; i < 8; i++) { const offset = i * 4 p.setUint32(offset, ~p.getUint32(offset, true)) } const one = new Uint8Array(ID_SIZE) one[0] = 1 this.add(one) } async getFingerprint(n) { const input = new WrappedBuffer() input.extend(this.buf) input.extend(encodeVarInt(n)) const hash = await this.sha256(input.unwrap()) return hash.subarray(0, FINGERPRINT_SIZE) } } class NegentropyStorageVector { constructor() { this.items = [] this.sealed = false } insert(timestamp, id) { if (this.sealed) throw Error("already sealed") id = loadInputBuffer(id) if (id.byteLength !== ID_SIZE) throw Error("bad id size for added item") this.items.push({timestamp, id}) } seal() { if (this.sealed) throw Error("already sealed") this.sealed = true this.items.sort(itemCompare) for (let i = 1; i < this.items.length; i++) { if (itemCompare(this.items[i - 1], this.items[i]) === 0) throw Error("duplicate item inserted") } } unseal() { this.sealed = false } size() { this._checkSealed() return this.items.length } getItem(i) { this._checkSealed() if (i >= this.items.length) throw Error("out of range") return this.items[i] } iterate(begin, end, cb) { this._checkSealed() this._checkBounds(begin, end) for (let i = begin; i < end; ++i) { if (!cb(this.items[i], i)) break } } findLowerBound(begin, end, bound) { this._checkSealed() this._checkBounds(begin, end) return this._binarySearch(this.items, begin, end, a => itemCompare(a, bound) < 0) } async fingerprint(begin, end) { const out = new Accumulator() out.setToZero() this.iterate(begin, end, (item, i) => { out.add(item.id) return true }) return await out.getFingerprint(end - begin) } _checkSealed() { if (!this.sealed) throw Error("not sealed") } _checkBounds(begin, end) { if (begin > end || end > this.items.length) throw Error("bad range") } _binarySearch(arr, first, last, cmp) { let count = last - first while (count > 0) { let it = first const step = Math.floor(count / 2) it += step if (cmp(arr[it])) { first = ++it count -= step + 1 } else { count = step } } return first } } class Negentropy { constructor(storage, frameSizeLimit = 0) { if (frameSizeLimit !== 0 && frameSizeLimit < 4096) throw Error("frameSizeLimit too small") this.storage = storage this.frameSizeLimit = frameSizeLimit this.lastTimestampIn = 0 this.lastTimestampOut = 0 } _bound(timestamp, id) { return {timestamp, id: id ? id : new Uint8Array(0)} } async initiate() { if (this.isInitiator) throw Error("already initiated") this.isInitiator = true const output = new WrappedBuffer() output.extend([PROTOCOL_VERSION]) await this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output) return this._renderOutput(output) } setInitiator() { this.isInitiator = true } async reconcile(query) { const haveIds = [], needIds = [] query = new WrappedBuffer(loadInputBuffer(query)) this.lastTimestampIn = this.lastTimestampOut = 0 // reset for each message const fullOutput = new WrappedBuffer() fullOutput.extend([PROTOCOL_VERSION]) const protocolVersion = getByte(query) if (protocolVersion < 0x60 || protocolVersion > 0x6f) throw Error("invalid negentropy protocol version byte") if (protocolVersion !== PROTOCOL_VERSION) { if (this.isInitiator) throw Error( "unsupported negentropy protocol version requested: " + (protocolVersion - 0x60), ) else return [this._renderOutput(fullOutput), haveIds, needIds] } const storageSize = this.storage.size() let prevBound = this._bound(0) let prevIndex = 0 let skip = false while (query.length !== 0) { let o = new WrappedBuffer() const doSkip = () => { if (skip) { skip = false o.extend(this.encodeBound(prevBound)) o.extend(encodeVarInt(Mode.Skip)) } } const currBound = this.decodeBound(query) const mode = decodeVarInt(query) const lower = prevIndex let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound) if (mode === Mode.Skip) { skip = true } else if (mode === Mode.Fingerprint) { const theirFingerprint = getBytes(query, FINGERPRINT_SIZE) const ourFingerprint = await this.storage.fingerprint(lower, upper) if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) { doSkip() await this.splitRange(lower, upper, currBound, o) } else { skip = true } } else if (mode === Mode.IdList) { const numIds = decodeVarInt(query) const theirElems = {} // stringified Uint8Array -> original Uint8Array (or hex) for (let i = 0; i < numIds; i++) { const e = getBytes(query, ID_SIZE) if (this.isInitiator) theirElems[e] = e } if (this.isInitiator) { skip = true this.storage.iterate(lower, upper, item => { const k = item.id if (!theirElems[k]) { // ID exists on our side, but not their side if (this.isInitiator) haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k)) } else { // ID exists on both sides delete theirElems[k] } return true }) for (const v of Object.values(theirElems)) { // ID exists on their side, but not our side needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v)) } } else { doSkip() const responseIds = new WrappedBuffer() let numResponseIds = 0 let endBound = currBound this.storage.iterate(lower, upper, (item, index) => { if (this.exceededFrameSizeLimit(fullOutput.length + responseIds.length)) { endBound = item upper = index // shrink upper so that remaining range gets correct fingerprint return false } responseIds.extend(item.id) numResponseIds++ return true }) o.extend(this.encodeBound(endBound)) o.extend(encodeVarInt(Mode.IdList)) o.extend(encodeVarInt(numResponseIds)) o.extend(responseIds) fullOutput.extend(o) o = new WrappedBuffer() } } else { throw Error("unexpected mode") } if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) { // frameSizeLimit exceeded: Stop range processing and return a fingerprint for the remaining range const remainingFingerprint = await this.storage.fingerprint(upper, storageSize) fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE))) fullOutput.extend(encodeVarInt(Mode.Fingerprint)) fullOutput.extend(remainingFingerprint) break } else { fullOutput.extend(o) } prevIndex = upper prevBound = currBound } return [ fullOutput.length === 1 && this.isInitiator ? null : this._renderOutput(fullOutput), haveIds, needIds, ] } async splitRange(lower, upper, upperBound, o) { const numElems = upper - lower const buckets = 16 if (numElems < buckets * 2) { o.extend(this.encodeBound(upperBound)) o.extend(encodeVarInt(Mode.IdList)) o.extend(encodeVarInt(numElems)) this.storage.iterate(lower, upper, item => { o.extend(item.id) return true }) } else { const itemsPerBucket = Math.floor(numElems / buckets) const bucketsWithExtra = numElems % buckets let curr = lower for (let i = 0; i < buckets; i++) { const bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0) const ourFingerprint = await this.storage.fingerprint(curr, curr + bucketSize) curr += bucketSize let nextBound if (curr === upper) { nextBound = upperBound } else { let prevItem, currItem this.storage.iterate(curr - 1, curr + 1, (item, index) => { if (index === curr - 1) prevItem = item else currItem = item return true }) nextBound = this.getMinimalBound(prevItem, currItem) } o.extend(this.encodeBound(nextBound)) o.extend(encodeVarInt(Mode.Fingerprint)) o.extend(ourFingerprint) } } } _renderOutput(o) { o = o.unwrap() if (!this.wantUint8ArrayOutput) o = uint8ArrayToHex(o) return o } exceededFrameSizeLimit(n) { return this.frameSizeLimit && n > this.frameSizeLimit - 200 } // Decoding decodeTimestampIn(encoded) { let timestamp = decodeVarInt(encoded) timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1 if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) { this.lastTimestampIn = Number.MAX_VALUE return Number.MAX_VALUE } timestamp += this.lastTimestampIn this.lastTimestampIn = timestamp return timestamp } decodeBound(encoded) { const timestamp = this.decodeTimestampIn(encoded) const len = decodeVarInt(encoded) if (len > ID_SIZE) throw Error("bound key too long") const id = getBytes(encoded, len) return {timestamp, id} } // Encoding encodeTimestampOut(timestamp) { if (timestamp === Number.MAX_VALUE) { this.lastTimestampOut = Number.MAX_VALUE return encodeVarInt(0) } const temp = timestamp timestamp -= this.lastTimestampOut this.lastTimestampOut = temp return encodeVarInt(timestamp + 1) } encodeBound(key) { const output = new WrappedBuffer() output.extend(this.encodeTimestampOut(key.timestamp)) output.extend(encodeVarInt(key.id.length)) output.extend(key.id) return output } getMinimalBound(prev, curr) { if (curr.timestamp !== prev.timestamp) { return this._bound(curr.timestamp) } else { let sharedPrefixBytes = 0 const currKey = curr.id const prevKey = prev.id for (let i = 0; i < ID_SIZE; i++) { if (currKey[i] !== prevKey[i]) break sharedPrefixBytes++ } return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1)) } } } function loadInputBuffer(inp) { if (typeof inp === "string") inp = hexToUint8Array(inp) else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp) // node Buffer? return inp } function hexToUint8Array(h) { if (h.startsWith("0x")) h = h.substr(2) if (h.length % 2 === 1) throw Error("odd length of hex string") const arr = new Uint8Array(h.length / 2) for (let i = 0; i < arr.length; i++) arr[i] = parseInt(h.substr(i * 2, 2), 16) return arr } const uint8ArrayToHexLookupTable = new Array(256) { const hexAlphabet = [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", ] for (let i = 0; i < 256; i++) { uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xf] + hexAlphabet[i & 0xf] } } function uint8ArrayToHex(arr) { let out = "" for (let i = 0, edx = arr.length; i < edx; i++) { out += uint8ArrayToHexLookupTable[arr[i]] } return out } function compareUint8Array(a, b) { for (let i = 0; i < a.byteLength; i++) { if (a[i] < b[i]) return -1 if (a[i] > b[i]) return 1 } if (a.byteLength > b.byteLength) return 1 if (a.byteLength < b.byteLength) return -1 return 0 } function itemCompare(a, b) { if (a.timestamp === b.timestamp) { return compareUint8Array(a.id, b.id) } return a.timestamp - b.timestamp } export {Negentropy, NegentropyStorageVector}