Remove relay package, move everything into net

This commit is contained in:
Jon Staab
2025-10-20 13:09:53 -07:00
parent 88650fb166
commit 0be540c0d2
42 changed files with 128 additions and 528 deletions
+59 -10
View File
@@ -1,8 +1,16 @@
import EventEmitter from "events"
import {call, mergeRight, on} from "@welshman/lib"
import {isRelayUrl} from "@welshman/util"
import {LocalRelay, LOCAL_RELAY_URL} from "@welshman/relay"
import {RelayMessage, ClientMessage} from "./message.js"
import {call, sleep, mergeRight, on} from "@welshman/lib"
import {isRelayUrl, matchFilters, Filter} from "@welshman/util"
import {LOCAL_RELAY_URL, Repository} from "./repository"
import {
RelayMessage,
RelayMessageType,
ClientMessage,
ClientMessageType,
ClientEvent,
ClientReq,
ClientClose,
} from "./message.js"
import {Socket, SocketEvent} from "./socket.js"
import {Unsubscriber} from "./util.js"
import {netContext, NetContext} from "./context.js"
@@ -53,12 +61,20 @@ export class SocketAdapter extends AbstractAdapter {
}
export class LocalAdapter extends AbstractAdapter {
constructor(readonly relay: LocalRelay) {
subs = new Map<string, Filter[]>()
constructor(readonly repository: Repository) {
super()
this._unsubscribers.push(
on(relay, "*", (...message: RelayMessage) => {
this.emit(AdapterEvent.Receive, message, LOCAL_RELAY_URL)
on(repository, "update", ({added}) => {
for (const [subId, filters] of this.subs.entries()) {
for (const event of added) {
if (matchFilters(filters, event)) {
this.#receive([RelayMessageType.Event, subId, event])
}
}
}
}),
)
}
@@ -72,9 +88,42 @@ export class LocalAdapter extends AbstractAdapter {
}
send(message: ClientMessage) {
const [type, ...rest] = message
switch (message[0]) {
case ClientMessageType.Event:
return this.#handleEVENT(message as ClientEvent)
case ClientMessageType.Close:
return this.#handleCLOSE(message as ClientClose)
case ClientMessageType.Req:
return this.#handleREQ(message as ClientReq)
}
}
this.relay.send(type, ...rest)
#receive(message: RelayMessage) {
this.emit(AdapterEvent.Receive, message, LOCAL_RELAY_URL)
}
#handleEVENT([_, event]: ClientEvent) {
this.repository.publish(event)
// Callers generally expect async relays
sleep(1).then(() => this.#receive([RelayMessageType.Ok, event.id, true, ""]))
}
#handleCLOSE([_, subId]: ClientClose) {
this.subs.delete(subId)
}
#handleREQ([_, subId, ...filters]: ClientReq) {
this.subs.set(subId, filters)
// Callers generally expect async relays
sleep(1).then(() => {
for (const event of this.repository.query(filters)) {
this.#receive([RelayMessageType.Event, subId, event])
}
this.#receive([RelayMessageType.Eose, subId])
})
}
}
@@ -113,7 +162,7 @@ export const getAdapter = (url: string, adapterContext: AdapterContext = {}) =>
}
if (url === LOCAL_RELAY_URL) {
return new LocalAdapter(new LocalRelay(context.repository))
return new LocalAdapter(context.repository)
}
if (isRelayUrl(url)) {
+1 -1
View File
@@ -1,6 +1,6 @@
import {Repository} from "@welshman/relay"
import {verifyEvent, TrustedEvent} from "@welshman/util"
import {AbstractAdapter} from "./adapter.js"
import {Repository} from "./repository.js"
import {Pool} from "./pool.js"
export type NetContext = {
+3
View File
@@ -8,4 +8,7 @@ export * from "./policy.js"
export * from "./pool.js"
export * from "./publish.js"
export * from "./socket.js"
export * from "./repository.js"
export * from "./request.js"
export * from "./tracker.js"
export * from "./wrapManager.js"
+387
View File
@@ -0,0 +1,387 @@
import {DAY, Emitter, flatten, pluck, sortBy, inc, uniq, omit, now, range} from "@welshman/lib"
import {
DELETE,
EPOCH,
matchFilter,
isReplaceable,
getAddress,
Filter,
TrustedEvent,
} from "@welshman/util"
export const LOCAL_RELAY_URL = "local://welshman.relay/"
const getDay = (ts: number) => Math.floor(ts / DAY)
export let repositorySingleton: Repository
export type RepositoryUpdate = {
added: TrustedEvent[]
removed: Set<string>
}
export class Repository extends Emitter {
eventsById = new Map<string, TrustedEvent>()
eventsByAddress = new Map<string, TrustedEvent>()
eventsByTag = new Map<string, TrustedEvent[]>()
eventsByDay = new Map<number, TrustedEvent[]>()
eventsByAuthor = new Map<string, TrustedEvent[]>()
eventsByKind = new Map<number, TrustedEvent[]>()
deletes = new Map<string, number>()
expired = new Map<string, number>()
static get() {
if (!repositorySingleton) {
repositorySingleton = new Repository()
}
return repositorySingleton
}
constructor() {
super()
this.setMaxListeners(1000)
}
// Dump/load/clear
dump = () => {
return Array.from(this.eventsById.values())
}
load = (events: TrustedEvent[]) => {
const stale = new Set(this.eventsById.keys())
this.eventsById.clear()
this.eventsByAddress.clear()
this.eventsByTag.clear()
this.eventsByDay.clear()
this.eventsByAuthor.clear()
this.eventsByKind.clear()
this.deletes.clear()
this.expired.clear()
const added = []
for (const event of events) {
if (this.publish(event, {shouldNotify: false})) {
// Don't send duplicate events to subscribers
if (!stale.has(event.id)) {
added.push(event)
}
}
}
const removed = new Set<string>()
// Anything we had before clearing the repository has been removed
for (const id of stale) {
if (!this.eventsById.has(id)) {
removed.add(id)
}
}
// Anything removed via delete or replace has been removed
for (const id of this.deletes.keys()) {
removed.add(id)
}
// Anything expired has been removed
for (const id of this.expired.keys()) {
removed.add(id)
}
this.emit("update", {added, removed})
}
// API
getEvent = (idOrAddress: string) => {
return idOrAddress.includes(":")
? this.eventsByAddress.get(idOrAddress)
: this.eventsById.get(idOrAddress)
}
hasEvent = (event: TrustedEvent) => {
const duplicate = this.eventsById.get(event.id) || this.eventsByAddress.get(getAddress(event))
return duplicate && duplicate.created_at >= event.created_at
}
removeEvent = (idOrAddress: string) => {
const event = this.getEvent(idOrAddress)
if (event) {
this.eventsById.delete(event.id)
this.eventsByAddress.delete(getAddress(event))
for (const [k, v] of event.tags) {
if (k.length === 1) {
this._updateIndex(this.eventsByTag, `${k}:${v}`, undefined, event)
}
}
this._updateIndex(this.eventsByDay, getDay(event.created_at), undefined, event)
this._updateIndex(this.eventsByAuthor, event.pubkey, undefined, event)
this._updateIndex(this.eventsByKind, event.kind, undefined, event)
this.emit("update", {added: [], removed: [event.id]})
}
}
query = (
filters: Filter[],
{includeDeleted = false, includeExpired = false, shouldSort = true} = {},
) => {
const result: TrustedEvent[][] = []
for (const originalFilter of filters) {
if (originalFilter.limit !== undefined && !shouldSort) {
throw new Error("Unable to skip sorting if limit is defined")
}
// Attempt to fulfill the query using one of our indexes. Fall back to all events.
const applied = this._applyAnyFilter(originalFilter)
const filter = applied?.filter || originalFilter
const events = applied ? this._getEvents(applied!.ids) : this.dump()
const sorted = this._sortEvents(shouldSort && Boolean(filter.limit), events)
const chunk: TrustedEvent[] = []
for (const event of sorted) {
if (filter.limit && chunk.length >= filter.limit) {
break
}
if (!includeDeleted && this.isDeleted(event)) {
continue
}
if (!includeExpired && this.isExpired(event)) {
continue
}
if (matchFilter(filter, event)) {
chunk.push(event)
}
}
result.push(chunk)
}
// Only re-sort if we had multiple filters, or if our single filter wasn't sorted
const shouldSortAll = shouldSort && (filters.length > 1 || !filters[0]?.limit)
return this._sortEvents(shouldSortAll, uniq(flatten(result)))
}
publish = (event: TrustedEvent, {shouldNotify = true} = {}): boolean => {
if (!event?.id) {
console.warn("Attempted to publish invalid event to repository", event)
return false
}
// If we've already seen this event we're done
if (this.eventsById.get(event.id)) {
return false
}
const removed = new Set<string>()
const address = getAddress(event)
const duplicate = this.eventsByAddress.get(address)
if (duplicate) {
// If our event is younger than the duplicate, we're done
if (event.created_at < duplicate.created_at) {
return false
}
// If our event is newer than what it's replacing, delete the old version
this.deletes.set(duplicate.id, event.created_at)
// Notify listeners that it's been removed
removed.add(duplicate.id)
}
// Add our new event by id
this.eventsById.set(event.id, event)
// Add our new event by address
if (isReplaceable(event)) {
this.eventsByAddress.set(address, event)
}
// Update our timestamp and author indexes
this._updateIndex(this.eventsByDay, getDay(event.created_at), event, duplicate)
this._updateIndex(this.eventsByAuthor, event.pubkey, event, duplicate)
this._updateIndex(this.eventsByKind, event.kind, event, duplicate)
// Update our tag indexes
for (const tag of event.tags) {
if (tag[0]?.length === 1) {
this._updateIndex(this.eventsByTag, tag.slice(0, 2).join(":"), event, duplicate)
// If this is a delete event, the tag value is an id or address. Track when it was
// deleted so that replaceables can be restored.
if (event.kind === DELETE && ["a", "e"].includes(tag[0]) && tag[1]) {
this.deletes.set(tag[1], Math.max(event.created_at, this.deletes.get(tag[1]) || 0))
const deletedEvent = this.getEvent(tag[1])
if (deletedEvent && this.isDeleted(deletedEvent)) {
removed.add(deletedEvent.id)
}
}
}
// Keep track of whether this event is expired
if (tag[0] === "expiration") {
const expiration = parseInt(tag[1] || "")
if (!isNaN(expiration)) {
this.expired.set(event.id, expiration)
}
}
}
if (shouldNotify) {
this.emit("update", {added: [event], removed})
}
return true
}
isDeletedByAddress = (event: TrustedEvent) =>
(this.deletes.get(getAddress(event)) || 0) > event.created_at
isDeletedById = (event: TrustedEvent) => (this.deletes.get(event.id) || 0) > event.created_at
isDeleted = (event: TrustedEvent) => this.isDeletedByAddress(event) || this.isDeletedById(event)
isExpired = (event: TrustedEvent) => {
const ts = this.expired.get(event.id)
return Boolean(ts && ts < now())
}
// Utilities
_sortEvents = (shouldSort: boolean, events: TrustedEvent[]) =>
shouldSort ? sortBy(e => -e.created_at, events) : events
_updateIndex = <K>(
m: Map<K, TrustedEvent[]>,
k: K,
add?: TrustedEvent,
remove?: TrustedEvent,
) => {
let a = m.get(k) || []
if (remove) {
a = a.filter((x: TrustedEvent) => x !== remove)
}
if (add) {
a.push(add)
}
m.set(k, a)
}
_getEvents = (ids: Iterable<string>) => {
const events: TrustedEvent[] = []
for (const id of ids) {
const event = this.eventsById.get(id)
if (event) {
events.push(event)
}
}
return events
}
_applyIdsFilter = (filter: Filter) => {
if (!filter.ids) return undefined
return {
filter: omit(["ids"], filter),
ids: new Set(filter.ids),
}
}
_applyAuthorsFilter = (filter: Filter) => {
if (!filter.authors) return undefined
const events = filter.authors.flatMap(pubkey => this.eventsByAuthor.get(pubkey) || [])
return {
filter: omit(["authors"], filter),
ids: new Set(pluck<string>("id", events)),
}
}
_applyTagsFilter = (filter: Filter) => {
for (const [k, values] of Object.entries(filter)) {
if (!k.startsWith("#") || k.length !== 2) {
continue
}
const ids = new Set<string>()
for (const v of values as string[]) {
for (const event of this.eventsByTag.get(`${k[1]}:${v}`) || []) {
ids.add(event.id)
}
}
return {filter: omit([k], filter), ids}
}
return undefined
}
_applyKindsFilter = (filter: Filter) => {
if (!filter.kinds) return undefined
const events = filter.kinds.flatMap(kind => this.eventsByKind.get(kind) || [])
return {
filter: omit(["kinds"], filter),
ids: new Set(pluck<string>("id", events)),
}
}
_applyDaysFilter = (filter: Filter) => {
if (!filter.since && !filter.until) return undefined
const sinceDay = getDay(filter.since || EPOCH)
const untilDay = getDay(filter.until || now())
const days = Array.from(range(sinceDay, inc(untilDay)))
const events = days.flatMap((day: number) => this.eventsByDay.get(day) || [])
const ids = new Set(pluck<string>("id", events))
return {filter, ids}
}
_applyAnyFilter = (filter: Filter) => {
const matchers = [
this._applyIdsFilter,
this._applyAuthorsFilter,
this._applyTagsFilter,
this._applyKindsFilter,
this._applyDaysFilter,
]
for (const matcher of matchers) {
const result = matcher(filter)
if (result) {
return result
}
}
return undefined
}
}
+1 -1
View File
@@ -18,11 +18,11 @@ import {
deduplicateEvents,
getFilterResultCardinality,
} from "@welshman/util"
import {Tracker} from "@welshman/relay"
import {RelayMessage, ClientMessageType, isRelayEvent, isRelayEose} from "./message.js"
import {getAdapter, AdapterContext, AdapterEvent} from "./adapter.js"
import {SocketEvent, SocketStatus} from "./socket.js"
import {netContext} from "./context.js"
import {Tracker} from "./tracker.js"
export type BaseRequestOptions = {
signal?: AbortSignal
+85
View File
@@ -0,0 +1,85 @@
import {Emitter, addToMapKey} from "@welshman/lib"
export class Tracker extends Emitter {
relaysById = new Map<string, Set<string>>()
idsByRelay = new Map<string, Set<string>>()
constructor() {
super()
this.setMaxListeners(100)
}
getIds = (relay: string) => this.idsByRelay.get(relay) || new Set<string>()
getRelays = (eventId: string) => this.relaysById.get(eventId) || new Set<string>()
hasRelay = (eventId: string, relay: string) => this.relaysById.get(eventId)?.has(relay)
addRelay = (eventId: string, relay: string) => {
let relays = this.relaysById.get(eventId)
let ids = this.idsByRelay.get(relay)
if (relays?.has(relay) && ids?.has(eventId)) return
if (!relays) {
relays = new Set()
}
if (!ids) {
ids = new Set()
}
relays.add(relay)
ids.add(eventId)
this.relaysById.set(eventId, relays)
this.idsByRelay.set(relay, ids)
this.emit("add", eventId, relay)
}
removeRelay = (eventId: string, relay: string) => {
const didDeleteRelay = this.relaysById.get(eventId)?.delete(relay)
const didDeleteId = this.idsByRelay.get(relay)?.delete(eventId)
if (!didDeleteRelay && !didDeleteId) return
this.emit("remove", eventId, relay)
}
track = (eventId: string, relay: string) => {
const seen = this.relaysById.has(eventId)
this.addRelay(eventId, relay)
return seen
}
copy = (eventId1: string, eventId2: string) => {
for (const relay of this.getRelays(eventId1)) {
this.addRelay(eventId2, relay)
}
}
load = (relaysById: Tracker["relaysById"]) => {
this.relaysById.clear()
this.idsByRelay.clear()
for (const [id, relays] of relaysById.entries()) {
for (const relay of relays) {
addToMapKey(this.relaysById, id, relay)
addToMapKey(this.idsByRelay, relay, id)
}
}
this.emit("load")
}
clear = () => {
this.relaysById.clear()
this.idsByRelay.clear()
this.emit("clear")
}
}
+128
View File
@@ -0,0 +1,128 @@
import {Emitter, remove, omit} from "@welshman/lib"
import {HashedEvent, SignedEvent} from "@welshman/util"
import {Tracker} from "./tracker.js"
import {Repository} from "./repository.js"
export type WrapItem = Omit<HashedEvent, "content"> & {
rumorId: string
recipient: string
}
export type WrapReference = string[]
export type WrapManagerOptions = {
repository: Repository
tracker: Tracker
}
export class WrapManager extends Emitter {
_wrapIndex = new Map<string, WrapItem>()
_rumorIndex = new Map<string, WrapReference>()
constructor(readonly options: WrapManagerOptions) {
super()
}
// Reading/exporting
dump = () => Array.from(this._wrapIndex.values())
getWraps = (rumorId: string) =>
this._rumorIndex.get(rumorId).map(wrapId => this._wrapIndex.get(wrapId)!)
getRumor = (wrapId: string) => {
const wrapItem = this._wrapIndex.get(wrapId)
if (wrapItem) {
return this.options.repository.getEvent(wrapItem.rumorId)
}
}
// Adding/importing
load = (wrapItems: WrapItem[]) => {
this._wrapIndex.clear()
this._rumorIndex.clear()
this._recipientIndex.clear()
for (const wrapItem of wrapItems) {
this._add(wrapItem)
}
this.emit("load")
}
add = ({recipient, rumor, wrap}: {recipient: string; rumor: HashedEvent; wrap: SignedEvent}) => {
const wrapItem = {
...omit(["content"], wrap),
rumorId: rumor.id,
recipient,
}
this._add(wrapItem)
// Save to our repository
this.options.repository.publish(rumor)
// Mark the rumor as having come from the wrap's urls
this.options.tracker.copy(wrap.id, rumor.id)
this.emit("add", wrapItem)
}
// Removing
remove = (id: string) => {
const wrapItem = this._wrapIndex.get(id)
if (wrapItem) {
this._remove(wrapItem)
this.options.repository.removeEvent(wrapItem.rumorId)
this.emit("remove", wrapItem)
}
}
removeByRumorId = (rumorId: string) => {
for (const id of this._rumorIndex.get(rumorId) || []) {
this.remove(id)
}
}
// Utils
_add = (wrapItem: WrapItem) => {
this._wrapIndex.set(wrapItem.id, wrapItem)
this._addReference(this._rumorIndex, wrapItem.rumorId, wrapItem.id)
this._addReference(this._recipientIndex, wrapItem.recipient, wrapItem.id)
}
_addReference = (index: Map<string, WrapReference>, key: string, wrapId: string) => {
const reference = index.get(key)
if (reference) {
reference.push(wrapId)
} else {
index.set(key, [wrapId])
}
}
_remove = (wrapItem: WrapItem) => {
this._wrapIndex.delete(wrapItem.id)
this._removeReference(this._rumorIndex, wrapItem.rumorId, wrapItem.id)
this._removeReference(this._recipientIndex, wrapItem.recipient, wrapItem.id)
}
_removeReference = (index: Map<string, WrapReference>, key: string, wrapId: string) => {
const reference = index.get(key)
if (reference) {
const wraps = remove(wrapId, reference)
if (wraps.length > 0) {
index.set(key, wraps)
} else {
index.delete(key)
}
}
}
}