Jon Staab
617 lines
15 KiB
TypeScript
617 lines
15 KiB
TypeScript
// (C) 2023 Doug Hoyte. MIT license
|
|
/* eslint-disable */
|
|
// @ts-nocheck
|
|
|
|
const PROTOCOL_VERSION = 0x61 // Version 1
|
|
const ID_SIZE = 32
|
|
const FINGERPRINT_SIZE = 16
|
|
|
|
const Mode = {
|
|
Skip: 0,
|
|
Fingerprint: 1,
|
|
IdList: 2,
|
|
}
|
|
|
|
class WrappedBuffer {
|
|
constructor(buffer) {
|
|
this._raw = new Uint8Array(buffer || 512)
|
|
this.length = buffer ? buffer.length : 0
|
|
}
|
|
|
|
unwrap() {
|
|
return this._raw.subarray(0, this.length)
|
|
}
|
|
|
|
get capacity() {
|
|
return this._raw.byteLength
|
|
}
|
|
|
|
extend(buf) {
|
|
if (buf._raw) buf = buf.unwrap()
|
|
if (typeof buf.length !== "number") throw Error("bad length")
|
|
const targetSize = buf.length + this.length
|
|
if (this.capacity < targetSize) {
|
|
const oldRaw = this._raw
|
|
const newCapacity = Math.max(this.capacity * 2, targetSize)
|
|
this._raw = new Uint8Array(newCapacity)
|
|
this._raw.set(oldRaw)
|
|
}
|
|
|
|
this._raw.set(buf, this.length)
|
|
this.length += buf.length
|
|
}
|
|
|
|
shift() {
|
|
const first = this._raw[0]
|
|
this._raw = this._raw.subarray(1)
|
|
this.length--
|
|
return first
|
|
}
|
|
|
|
shiftN(n = 1) {
|
|
const firstSubarray = this._raw.subarray(0, n)
|
|
this._raw = this._raw.subarray(n)
|
|
this.length -= n
|
|
return firstSubarray
|
|
}
|
|
}
|
|
|
|
function decodeVarInt(buf) {
|
|
let res = 0
|
|
|
|
while (1) {
|
|
if (buf.length === 0) throw Error("parse ends prematurely")
|
|
const byte = buf.shift()
|
|
res = (res << 7) | (byte & 127)
|
|
if ((byte & 128) === 0) break
|
|
}
|
|
|
|
return res
|
|
}
|
|
|
|
function encodeVarInt(n) {
|
|
if (n === 0) return new WrappedBuffer([0])
|
|
|
|
const o = []
|
|
|
|
while (n !== 0) {
|
|
o.push(n & 127)
|
|
n >>>= 7
|
|
}
|
|
|
|
o.reverse()
|
|
|
|
for (let i = 0; i < o.length - 1; i++) o[i] |= 128
|
|
|
|
return new WrappedBuffer(o)
|
|
}
|
|
|
|
function getByte(buf) {
|
|
return getBytes(buf, 1)[0]
|
|
}
|
|
|
|
function getBytes(buf, n) {
|
|
if (buf.length < n) throw Error("parse ends prematurely")
|
|
return buf.shiftN(n)
|
|
}
|
|
|
|
class Accumulator {
|
|
constructor() {
|
|
this.setToZero()
|
|
|
|
if (typeof window === "undefined") {
|
|
// node.js
|
|
this.sha256 = async slice =>
|
|
new Uint8Array(crypto.createHash("sha256").update(slice).digest())
|
|
} else {
|
|
// browser
|
|
this.sha256 = async slice => new Uint8Array(await crypto.subtle.digest("SHA-256", slice))
|
|
}
|
|
}
|
|
|
|
setToZero() {
|
|
this.buf = new Uint8Array(ID_SIZE)
|
|
}
|
|
|
|
add(otherBuf) {
|
|
let currCarry = 0,
|
|
nextCarry = 0
|
|
const p = new DataView(this.buf.buffer)
|
|
const po = new DataView(otherBuf.buffer)
|
|
|
|
for (let i = 0; i < 8; i++) {
|
|
const offset = i * 4
|
|
const orig = p.getUint32(offset, true)
|
|
const otherV = po.getUint32(offset, true)
|
|
|
|
let next = orig
|
|
|
|
next += currCarry
|
|
next += otherV
|
|
if (next > 0xffffffff) nextCarry = 1
|
|
|
|
p.setUint32(offset, next & 0xffffffff, true)
|
|
currCarry = nextCarry
|
|
nextCarry = 0
|
|
}
|
|
}
|
|
|
|
negate() {
|
|
const p = new DataView(this.buf.buffer)
|
|
|
|
for (let i = 0; i < 8; i++) {
|
|
const offset = i * 4
|
|
p.setUint32(offset, ~p.getUint32(offset, true))
|
|
}
|
|
|
|
const one = new Uint8Array(ID_SIZE)
|
|
one[0] = 1
|
|
this.add(one)
|
|
}
|
|
|
|
async getFingerprint(n) {
|
|
const input = new WrappedBuffer()
|
|
input.extend(this.buf)
|
|
input.extend(encodeVarInt(n))
|
|
|
|
const hash = await this.sha256(input.unwrap())
|
|
|
|
return hash.subarray(0, FINGERPRINT_SIZE)
|
|
}
|
|
}
|
|
|
|
class NegentropyStorageVector {
|
|
constructor() {
|
|
this.items = []
|
|
this.sealed = false
|
|
}
|
|
|
|
insert(timestamp, id) {
|
|
if (this.sealed) throw Error("already sealed")
|
|
id = loadInputBuffer(id)
|
|
if (id.byteLength !== ID_SIZE) throw Error("bad id size for added item")
|
|
this.items.push({timestamp, id})
|
|
}
|
|
|
|
seal() {
|
|
if (this.sealed) throw Error("already sealed")
|
|
this.sealed = true
|
|
|
|
this.items.sort(itemCompare)
|
|
|
|
for (let i = 1; i < this.items.length; i++) {
|
|
if (itemCompare(this.items[i - 1], this.items[i]) === 0)
|
|
throw Error("duplicate item inserted")
|
|
}
|
|
}
|
|
|
|
unseal() {
|
|
this.sealed = false
|
|
}
|
|
|
|
size() {
|
|
this._checkSealed()
|
|
return this.items.length
|
|
}
|
|
|
|
getItem(i) {
|
|
this._checkSealed()
|
|
if (i >= this.items.length) throw Error("out of range")
|
|
return this.items[i]
|
|
}
|
|
|
|
iterate(begin, end, cb) {
|
|
this._checkSealed()
|
|
this._checkBounds(begin, end)
|
|
|
|
for (let i = begin; i < end; ++i) {
|
|
if (!cb(this.items[i], i)) break
|
|
}
|
|
}
|
|
|
|
findLowerBound(begin, end, bound) {
|
|
this._checkSealed()
|
|
this._checkBounds(begin, end)
|
|
|
|
return this._binarySearch(this.items, begin, end, a => itemCompare(a, bound) < 0)
|
|
}
|
|
|
|
async fingerprint(begin, end) {
|
|
const out = new Accumulator()
|
|
out.setToZero()
|
|
|
|
this.iterate(begin, end, (item, i) => {
|
|
out.add(item.id)
|
|
return true
|
|
})
|
|
|
|
return await out.getFingerprint(end - begin)
|
|
}
|
|
|
|
_checkSealed() {
|
|
if (!this.sealed) throw Error("not sealed")
|
|
}
|
|
|
|
_checkBounds(begin, end) {
|
|
if (begin > end || end > this.items.length) throw Error("bad range")
|
|
}
|
|
|
|
_binarySearch(arr, first, last, cmp) {
|
|
let count = last - first
|
|
|
|
while (count > 0) {
|
|
let it = first
|
|
const step = Math.floor(count / 2)
|
|
it += step
|
|
|
|
if (cmp(arr[it])) {
|
|
first = ++it
|
|
count -= step + 1
|
|
} else {
|
|
count = step
|
|
}
|
|
}
|
|
|
|
return first
|
|
}
|
|
}
|
|
|
|
class Negentropy {
|
|
constructor(storage, frameSizeLimit = 0) {
|
|
if (frameSizeLimit !== 0 && frameSizeLimit < 4096) throw Error("frameSizeLimit too small")
|
|
|
|
this.storage = storage
|
|
this.frameSizeLimit = frameSizeLimit
|
|
|
|
this.lastTimestampIn = 0
|
|
this.lastTimestampOut = 0
|
|
}
|
|
|
|
_bound(timestamp, id) {
|
|
return {timestamp, id: id ? id : new Uint8Array(0)}
|
|
}
|
|
|
|
async initiate() {
|
|
if (this.isInitiator) throw Error("already initiated")
|
|
this.isInitiator = true
|
|
|
|
const output = new WrappedBuffer()
|
|
output.extend([PROTOCOL_VERSION])
|
|
|
|
await this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output)
|
|
|
|
return this._renderOutput(output)
|
|
}
|
|
|
|
setInitiator() {
|
|
this.isInitiator = true
|
|
}
|
|
|
|
async reconcile(query) {
|
|
const haveIds = [],
|
|
needIds = []
|
|
query = new WrappedBuffer(loadInputBuffer(query))
|
|
|
|
this.lastTimestampIn = this.lastTimestampOut = 0 // reset for each message
|
|
|
|
const fullOutput = new WrappedBuffer()
|
|
fullOutput.extend([PROTOCOL_VERSION])
|
|
|
|
const protocolVersion = getByte(query)
|
|
if (protocolVersion < 0x60 || protocolVersion > 0x6f)
|
|
throw Error("invalid negentropy protocol version byte")
|
|
if (protocolVersion !== PROTOCOL_VERSION) {
|
|
if (this.isInitiator)
|
|
throw Error(
|
|
"unsupported negentropy protocol version requested: " + (protocolVersion - 0x60),
|
|
)
|
|
else return [this._renderOutput(fullOutput), haveIds, needIds]
|
|
}
|
|
|
|
const storageSize = this.storage.size()
|
|
let prevBound = this._bound(0)
|
|
let prevIndex = 0
|
|
let skip = false
|
|
|
|
while (query.length !== 0) {
|
|
let o = new WrappedBuffer()
|
|
|
|
const doSkip = () => {
|
|
if (skip) {
|
|
skip = false
|
|
o.extend(this.encodeBound(prevBound))
|
|
o.extend(encodeVarInt(Mode.Skip))
|
|
}
|
|
}
|
|
|
|
const currBound = this.decodeBound(query)
|
|
const mode = decodeVarInt(query)
|
|
|
|
const lower = prevIndex
|
|
let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound)
|
|
|
|
if (mode === Mode.Skip) {
|
|
skip = true
|
|
} else if (mode === Mode.Fingerprint) {
|
|
const theirFingerprint = getBytes(query, FINGERPRINT_SIZE)
|
|
const ourFingerprint = await this.storage.fingerprint(lower, upper)
|
|
|
|
if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) {
|
|
doSkip()
|
|
await this.splitRange(lower, upper, currBound, o)
|
|
} else {
|
|
skip = true
|
|
}
|
|
} else if (mode === Mode.IdList) {
|
|
const numIds = decodeVarInt(query)
|
|
|
|
const theirElems = {} // stringified Uint8Array -> original Uint8Array (or hex)
|
|
for (let i = 0; i < numIds; i++) {
|
|
const e = getBytes(query, ID_SIZE)
|
|
if (this.isInitiator) theirElems[e] = e
|
|
}
|
|
|
|
if (this.isInitiator) {
|
|
skip = true
|
|
|
|
this.storage.iterate(lower, upper, item => {
|
|
const k = item.id
|
|
|
|
if (!theirElems[k]) {
|
|
// ID exists on our side, but not their side
|
|
if (this.isInitiator) haveIds.push(this.wantUint8ArrayOutput ? k : uint8ArrayToHex(k))
|
|
} else {
|
|
// ID exists on both sides
|
|
delete theirElems[k]
|
|
}
|
|
|
|
return true
|
|
})
|
|
|
|
for (const v of Object.values(theirElems)) {
|
|
// ID exists on their side, but not our side
|
|
needIds.push(this.wantUint8ArrayOutput ? v : uint8ArrayToHex(v))
|
|
}
|
|
} else {
|
|
doSkip()
|
|
|
|
const responseIds = new WrappedBuffer()
|
|
let numResponseIds = 0
|
|
let endBound = currBound
|
|
|
|
this.storage.iterate(lower, upper, (item, index) => {
|
|
if (this.exceededFrameSizeLimit(fullOutput.length + responseIds.length)) {
|
|
endBound = item
|
|
upper = index // shrink upper so that remaining range gets correct fingerprint
|
|
return false
|
|
}
|
|
|
|
responseIds.extend(item.id)
|
|
numResponseIds++
|
|
return true
|
|
})
|
|
|
|
o.extend(this.encodeBound(endBound))
|
|
o.extend(encodeVarInt(Mode.IdList))
|
|
o.extend(encodeVarInt(numResponseIds))
|
|
o.extend(responseIds)
|
|
|
|
fullOutput.extend(o)
|
|
o = new WrappedBuffer()
|
|
}
|
|
} else {
|
|
throw Error("unexpected mode")
|
|
}
|
|
|
|
if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) {
|
|
// frameSizeLimit exceeded: Stop range processing and return a fingerprint for the remaining range
|
|
const remainingFingerprint = await this.storage.fingerprint(upper, storageSize)
|
|
|
|
fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE)))
|
|
fullOutput.extend(encodeVarInt(Mode.Fingerprint))
|
|
fullOutput.extend(remainingFingerprint)
|
|
break
|
|
} else {
|
|
fullOutput.extend(o)
|
|
}
|
|
|
|
prevIndex = upper
|
|
prevBound = currBound
|
|
}
|
|
|
|
return [
|
|
fullOutput.length === 1 && this.isInitiator ? null : this._renderOutput(fullOutput),
|
|
haveIds,
|
|
needIds,
|
|
]
|
|
}
|
|
|
|
async splitRange(lower, upper, upperBound, o) {
|
|
const numElems = upper - lower
|
|
const buckets = 16
|
|
|
|
if (numElems < buckets * 2) {
|
|
o.extend(this.encodeBound(upperBound))
|
|
o.extend(encodeVarInt(Mode.IdList))
|
|
|
|
o.extend(encodeVarInt(numElems))
|
|
this.storage.iterate(lower, upper, item => {
|
|
o.extend(item.id)
|
|
return true
|
|
})
|
|
} else {
|
|
const itemsPerBucket = Math.floor(numElems / buckets)
|
|
const bucketsWithExtra = numElems % buckets
|
|
let curr = lower
|
|
|
|
for (let i = 0; i < buckets; i++) {
|
|
const bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0)
|
|
const ourFingerprint = await this.storage.fingerprint(curr, curr + bucketSize)
|
|
curr += bucketSize
|
|
|
|
let nextBound
|
|
|
|
if (curr === upper) {
|
|
nextBound = upperBound
|
|
} else {
|
|
let prevItem, currItem
|
|
|
|
this.storage.iterate(curr - 1, curr + 1, (item, index) => {
|
|
if (index === curr - 1) prevItem = item
|
|
else currItem = item
|
|
return true
|
|
})
|
|
|
|
nextBound = this.getMinimalBound(prevItem, currItem)
|
|
}
|
|
|
|
o.extend(this.encodeBound(nextBound))
|
|
o.extend(encodeVarInt(Mode.Fingerprint))
|
|
o.extend(ourFingerprint)
|
|
}
|
|
}
|
|
}
|
|
|
|
_renderOutput(o) {
|
|
o = o.unwrap()
|
|
if (!this.wantUint8ArrayOutput) o = uint8ArrayToHex(o)
|
|
return o
|
|
}
|
|
|
|
exceededFrameSizeLimit(n) {
|
|
return this.frameSizeLimit && n > this.frameSizeLimit - 200
|
|
}
|
|
|
|
// Decoding
|
|
|
|
decodeTimestampIn(encoded) {
|
|
let timestamp = decodeVarInt(encoded)
|
|
timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1
|
|
if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) {
|
|
this.lastTimestampIn = Number.MAX_VALUE
|
|
return Number.MAX_VALUE
|
|
}
|
|
timestamp += this.lastTimestampIn
|
|
this.lastTimestampIn = timestamp
|
|
return timestamp
|
|
}
|
|
|
|
decodeBound(encoded) {
|
|
const timestamp = this.decodeTimestampIn(encoded)
|
|
const len = decodeVarInt(encoded)
|
|
if (len > ID_SIZE) throw Error("bound key too long")
|
|
const id = getBytes(encoded, len)
|
|
return {timestamp, id}
|
|
}
|
|
|
|
// Encoding
|
|
|
|
encodeTimestampOut(timestamp) {
|
|
if (timestamp === Number.MAX_VALUE) {
|
|
this.lastTimestampOut = Number.MAX_VALUE
|
|
return encodeVarInt(0)
|
|
}
|
|
|
|
const temp = timestamp
|
|
timestamp -= this.lastTimestampOut
|
|
this.lastTimestampOut = temp
|
|
return encodeVarInt(timestamp + 1)
|
|
}
|
|
|
|
encodeBound(key) {
|
|
const output = new WrappedBuffer()
|
|
|
|
output.extend(this.encodeTimestampOut(key.timestamp))
|
|
output.extend(encodeVarInt(key.id.length))
|
|
output.extend(key.id)
|
|
|
|
return output
|
|
}
|
|
|
|
getMinimalBound(prev, curr) {
|
|
if (curr.timestamp !== prev.timestamp) {
|
|
return this._bound(curr.timestamp)
|
|
} else {
|
|
let sharedPrefixBytes = 0
|
|
const currKey = curr.id
|
|
const prevKey = prev.id
|
|
|
|
for (let i = 0; i < ID_SIZE; i++) {
|
|
if (currKey[i] !== prevKey[i]) break
|
|
sharedPrefixBytes++
|
|
}
|
|
|
|
return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1))
|
|
}
|
|
}
|
|
}
|
|
|
|
function loadInputBuffer(inp) {
|
|
if (typeof inp === "string") inp = hexToUint8Array(inp)
|
|
else if (__proto__ !== Uint8Array.prototype) inp = new Uint8Array(inp) // node Buffer?
|
|
return inp
|
|
}
|
|
|
|
function hexToUint8Array(h) {
|
|
if (h.startsWith("0x")) h = h.substr(2)
|
|
if (h.length % 2 === 1) throw Error("odd length of hex string")
|
|
const arr = new Uint8Array(h.length / 2)
|
|
for (let i = 0; i < arr.length; i++) arr[i] = parseInt(h.substr(i * 2, 2), 16)
|
|
return arr
|
|
}
|
|
|
|
const uint8ArrayToHexLookupTable = new Array(256)
|
|
{
|
|
const hexAlphabet = [
|
|
"0",
|
|
"1",
|
|
"2",
|
|
"3",
|
|
"4",
|
|
"5",
|
|
"6",
|
|
"7",
|
|
"8",
|
|
"9",
|
|
"a",
|
|
"b",
|
|
"c",
|
|
"d",
|
|
"e",
|
|
"f",
|
|
]
|
|
for (let i = 0; i < 256; i++) {
|
|
uint8ArrayToHexLookupTable[i] = hexAlphabet[(i >>> 4) & 0xf] + hexAlphabet[i & 0xf]
|
|
}
|
|
}
|
|
|
|
function uint8ArrayToHex(arr) {
|
|
let out = ""
|
|
for (let i = 0, edx = arr.length; i < edx; i++) {
|
|
out += uint8ArrayToHexLookupTable[arr[i]]
|
|
}
|
|
return out
|
|
}
|
|
|
|
function compareUint8Array(a, b) {
|
|
for (let i = 0; i < a.byteLength; i++) {
|
|
if (a[i] < b[i]) return -1
|
|
if (a[i] > b[i]) return 1
|
|
}
|
|
|
|
if (a.byteLength > b.byteLength) return 1
|
|
if (a.byteLength < b.byteLength) return -1
|
|
|
|
return 0
|
|
}
|
|
|
|
function itemCompare(a, b) {
|
|
if (a.timestamp === b.timestamp) {
|
|
return compareUint8Array(a.id, b.id)
|
|
}
|
|
|
|
return a.timestamp - b.timestamp
|
|
}
|
|
|
|
export {Negentropy, NegentropyStorageVector}
|