Fix various repository bugs
This commit is contained in:
Jon Staab
@@ -45,7 +45,7 @@ export const createEvent = (kind: number, {content = "", tags = [], created_at =
|
||||
({kind, content, tags, created_at})
|
||||
|
||||
export const isEventTemplate = (e: EventTemplate): e is EventTemplate =>
|
||||
Boolean(e.kind && e.tags && e.content && e.created_at)
|
||||
Boolean(typeof e.kind === "number" && e.tags && typeof e.content === "string" && e.created_at)
|
||||
|
||||
export const isOwnedEvent = (e: OwnedEvent): e is OwnedEvent =>
|
||||
Boolean(isEventTemplate(e) && e.pubkey)
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
export const fromNostrURI = (s: string) => s.replace(/^[\w+]+:\/?\/?/, "")
|
||||
export const fromNostrURI = (s: string) => s.replace(/^nostr:\/?\/?/, "")
|
||||
|
||||
export const toNostrURI = (s: string) => `nostr:${s}`
|
||||
|
||||
+49
-16
@@ -1,42 +1,75 @@
|
||||
import {Emitter} from '@welshman/lib'
|
||||
import {Emitter, normalizeUrl, stripProtocol} from '@welshman/lib'
|
||||
import {matchFilters} from './Filters'
|
||||
import type {Repository} from './Repository'
|
||||
import type {Filter} from './Filters'
|
||||
import type {TrustedEvent} from './Events'
|
||||
|
||||
export class Relay<E extends TrustedEvent> extends Emitter {
|
||||
export const LOCAL_RELAY_URL = "local://welshman.relay"
|
||||
|
||||
export const BOGUS_RELAY_URL = "bogus://welshman.relay"
|
||||
|
||||
export const isShareableRelayUrl = (url: string) =>
|
||||
Boolean(
|
||||
typeof url === 'string' &&
|
||||
// Is it actually a websocket url and has a dot
|
||||
url.match(/^wss:\/\/.+\..+/) &&
|
||||
// Sometimes bugs cause multiple relays to get concatenated
|
||||
url.match(/:\/\//g)?.length === 1 &&
|
||||
// It shouldn't have any whitespace, url-encoded or otherwise
|
||||
!url.match(/\s|%/) &&
|
||||
// Don't match stuff with a port number
|
||||
!url.slice(6).match(/:\d+/) &&
|
||||
// Don't match raw ip addresses
|
||||
!url.slice(6).match(/\d+\.\d+\.\d+\.\d+/) &&
|
||||
// Skip nostr.wine's virtual relays
|
||||
!url.slice(6).match(/\/npub/)
|
||||
)
|
||||
|
||||
type NormalizeRelayUrlOpts = {
|
||||
allowInsecure?: boolean
|
||||
}
|
||||
|
||||
export const normalizeRelayUrl = (url: string, {allowInsecure = false}: NormalizeRelayUrlOpts = {}) => {
|
||||
const prefix = allowInsecure ? url.match(/^wss?:\/\//)?.[0] || "wss://" : "wss://"
|
||||
|
||||
// Use our library to normalize
|
||||
url = normalizeUrl(url, {stripHash: true, stripAuthentication: false})
|
||||
|
||||
// Strip the protocol since only wss works, lowercase
|
||||
url = stripProtocol(url).toLowerCase()
|
||||
|
||||
// Urls without pathnames are supposed to have a trailing slash
|
||||
if (!url.includes("/")) {
|
||||
url += "/"
|
||||
}
|
||||
|
||||
return prefix + url
|
||||
}
|
||||
|
||||
export class Relay extends Emitter {
|
||||
subs = new Map<string, Filter[]>()
|
||||
|
||||
constructor(readonly repository: Repository<E>) {
|
||||
constructor(readonly repository: Repository) {
|
||||
super()
|
||||
}
|
||||
|
||||
send(type: string, ...message: any[]) {
|
||||
switch(type) {
|
||||
case 'EVENT': return this.handleEVENT(message as [string])
|
||||
case 'EVENT': return this.handleEVENT(message as [TrustedEvent])
|
||||
case 'CLOSE': return this.handleCLOSE(message as [string])
|
||||
case 'REQ': return this.handleREQ(message as [string, ...Filter[]])
|
||||
}
|
||||
}
|
||||
|
||||
handleEVENT([json]: [string]) {
|
||||
let event: E
|
||||
try {
|
||||
event = JSON.parse(json)
|
||||
} catch (e) {
|
||||
return
|
||||
}
|
||||
|
||||
handleEVENT([event]: [TrustedEvent]) {
|
||||
this.repository.publish(event)
|
||||
|
||||
this.emit('OK', event.id, true, "")
|
||||
|
||||
if (!this.repository.isDeleted(event)) {
|
||||
const json = JSON.stringify(event)
|
||||
|
||||
for (const [subId, filters] of this.subs.entries()) {
|
||||
if (matchFilters(filters, event)) {
|
||||
this.emit('EVENT', subId, json)
|
||||
this.emit('EVENT', subId, event)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,7 +83,7 @@ export class Relay<E extends TrustedEvent> extends Emitter {
|
||||
this.subs.set(subId, filters)
|
||||
|
||||
for (const event of this.repository.query(filters)) {
|
||||
this.emit('EVENT', subId, JSON.stringify(event))
|
||||
this.emit('EVENT', subId, event)
|
||||
}
|
||||
|
||||
this.emit('EOSE', subId)
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import {normalizeUrl, stripProtocol} from '@welshman/lib'
|
||||
|
||||
export const LOCAL_RELAY_URL = "local://welshman.relay"
|
||||
|
||||
export const BOGUS_RELAY_URL = "bogus://welshman.relay"
|
||||
|
||||
export const isShareableRelayUrl = (url: string) =>
|
||||
Boolean(
|
||||
typeof url === 'string' &&
|
||||
// Is it actually a websocket url and has a dot
|
||||
url.match(/^wss:\/\/.+\..+/) &&
|
||||
// Sometimes bugs cause multiple relays to get concatenated
|
||||
url.match(/:\/\//g)?.length === 1 &&
|
||||
// It shouldn't have any whitespace, url-encoded or otherwise
|
||||
!url.match(/\s|%/) &&
|
||||
// Don't match stuff with a port number
|
||||
!url.slice(6).match(/:\d+/) &&
|
||||
// Don't match raw ip addresses
|
||||
!url.slice(6).match(/\d+\.\d+\.\d+\.\d+/) &&
|
||||
// Skip nostr.wine's virtual relays
|
||||
!url.slice(6).match(/\/npub/)
|
||||
)
|
||||
|
||||
type NormalizeRelayUrlOpts = {
|
||||
allowInsecure?: boolean
|
||||
}
|
||||
|
||||
export const normalizeRelayUrl = (url: string, {allowInsecure = false}: NormalizeRelayUrlOpts = {}) => {
|
||||
const prefix = allowInsecure ? url.match(/^wss?:\/\//)?.[0] || "wss://" : "wss://"
|
||||
|
||||
// Use our library to normalize
|
||||
url = normalizeUrl(url, {stripHash: true, stripAuthentication: false})
|
||||
|
||||
// Strip the protocol since only wss works, lowercase
|
||||
url = stripProtocol(url).toLowerCase()
|
||||
|
||||
// Urls without pathnames are supposed to have a trailing slash
|
||||
if (!url.includes("/")) {
|
||||
url += "/"
|
||||
}
|
||||
|
||||
return prefix + url
|
||||
}
|
||||
+122
-133
@@ -1,10 +1,9 @@
|
||||
import {throttle} from 'throttle-debounce'
|
||||
import type {IReadable, Subscriber, Invalidator} from '@welshman/lib'
|
||||
import {Derived, Emitter, writable, first, always, chunk, sleep, uniq, omit, now, range, identity} from '@welshman/lib'
|
||||
import {Derived, Emitter, sortBy, customStore, inc, first, always, chunk, sleep, uniq, omit, now, range, identity} from '@welshman/lib'
|
||||
import {DELETE} from './Kinds'
|
||||
import {matchFilter, getIdFilters, matchFilters} from './Filters'
|
||||
import {encodeAddress, addressFromEvent} from './Address'
|
||||
import {isReplaceable} from './Events'
|
||||
import {isReplaceable, isTrustedEvent, getAddress} from './Events'
|
||||
import type {Filter} from './Filters'
|
||||
import type {TrustedEvent} from './Events'
|
||||
|
||||
@@ -16,14 +15,14 @@ export type RepositoryOptions = {
|
||||
throttle?: number
|
||||
}
|
||||
|
||||
export class Repository<E extends TrustedEvent> extends Emitter implements IReadable<Repository<E>> {
|
||||
eventsById = new Map<string, E>()
|
||||
eventsByAddress = new Map<string, E>()
|
||||
eventsByTag = new Map<string, E[]>()
|
||||
eventsByDay = new Map<number, E[]>()
|
||||
eventsByAuthor = new Map<string, E[]>()
|
||||
export class Repository extends Emitter implements IReadable<TrustedEvent[]> {
|
||||
eventsById = new Map<string, TrustedEvent>()
|
||||
eventsByAddress = new Map<string, TrustedEvent>()
|
||||
eventsByTag = new Map<string, TrustedEvent[]>()
|
||||
eventsByDay = new Map<number, TrustedEvent[]>()
|
||||
eventsByAuthor = new Map<string, TrustedEvent[]>()
|
||||
deletes = new Map<string, number>()
|
||||
subs: Subscriber<typeof this>[] = []
|
||||
subs: Subscriber<TrustedEvent[]>[] = []
|
||||
|
||||
constructor(private options: RepositoryOptions) {
|
||||
super()
|
||||
@@ -36,76 +35,13 @@ export class Repository<E extends TrustedEvent> extends Emitter implements IRead
|
||||
// Methods for implementing store interface
|
||||
|
||||
get() {
|
||||
return this
|
||||
}
|
||||
|
||||
subscribe(f: Subscriber<Repository<E>>, invalidate?: Invalidator<Repository<E>>) {
|
||||
this.subs.push(f)
|
||||
|
||||
return () => {
|
||||
this.subs = this.subs.filter(sub => sub !== f)
|
||||
}
|
||||
}
|
||||
|
||||
derived<U>(f: (v: Repository<E>) => U): Derived<U> {
|
||||
return new Derived<U>(this, f)
|
||||
}
|
||||
|
||||
throttle(t: number): Derived<Repository<E>> {
|
||||
return new Derived<Repository<E>>(this, identity, t)
|
||||
}
|
||||
|
||||
filter(getFilters: () => Filter[]) {
|
||||
const store = writable<E[]>([])
|
||||
|
||||
const onNotify = (event?: E) => {
|
||||
const filters = getFilters()
|
||||
|
||||
if (!event || matchFilters(filters, event)) {
|
||||
store.set(Array.from(this.query(filters)))
|
||||
}
|
||||
}
|
||||
|
||||
const subscribe = store.subscribe.bind(store)
|
||||
|
||||
store.subscribe = (f: Subscriber<E[]>) => {
|
||||
if (store.subs.length === 0) {
|
||||
this.on('notify', onNotify)
|
||||
onNotify()
|
||||
}
|
||||
|
||||
const unsubscribe = subscribe(f)
|
||||
|
||||
return () => {
|
||||
unsubscribe()
|
||||
|
||||
if (store.subs.length === 0) {
|
||||
this.off('notify', onNotify)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return store
|
||||
}
|
||||
|
||||
notify(event?: E) {
|
||||
for (const sub of this.subs) {
|
||||
sub(this)
|
||||
}
|
||||
|
||||
this.emit('notify', event)
|
||||
}
|
||||
|
||||
// Load/dump
|
||||
|
||||
dump() {
|
||||
return Array.from(this.eventsById.values())
|
||||
}
|
||||
|
||||
async load(events: E[], chunkSize = 1000) {
|
||||
async set(events: TrustedEvent[], chunkSize = 1000) {
|
||||
for (const eventsChunk of chunk(chunkSize, events)) {
|
||||
for (const event of eventsChunk) {
|
||||
this._addEvent(event)
|
||||
this.publish(event, {notify: false})
|
||||
}
|
||||
|
||||
if (eventsChunk.length === chunkSize) {
|
||||
@@ -116,6 +52,52 @@ export class Repository<E extends TrustedEvent> extends Emitter implements IRead
|
||||
this.notify()
|
||||
}
|
||||
|
||||
|
||||
subscribe(f: Subscriber<TrustedEvent[]>, invalidate?: Invalidator<TrustedEvent[]>) {
|
||||
this.subs.push(f)
|
||||
|
||||
return () => {
|
||||
this.subs = this.subs.filter(sub => sub !== f)
|
||||
}
|
||||
}
|
||||
|
||||
derived<U>(f: (v: TrustedEvent[]) => U): Derived<U> {
|
||||
return new Derived<U>(this, f)
|
||||
}
|
||||
|
||||
throttle(t: number): Derived<TrustedEvent[]> {
|
||||
return new Derived<TrustedEvent[]>(this, identity, t)
|
||||
}
|
||||
|
||||
filter(getFilters: () => Filter[]) {
|
||||
const getValue = () => Array.from(this.query(getFilters()))
|
||||
|
||||
return customStore<TrustedEvent[]>({
|
||||
get: getValue,
|
||||
start: setValue => {
|
||||
const onNotify = (event?: TrustedEvent) => {
|
||||
if (!event || matchFilters(getFilters(), event)) {
|
||||
setValue(getValue())
|
||||
}
|
||||
}
|
||||
|
||||
this.on('notify', onNotify)
|
||||
|
||||
return () => this.off('notify', onNotify)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
notify(event?: TrustedEvent) {
|
||||
const events = this.get()
|
||||
|
||||
for (const sub of this.subs) {
|
||||
sub(events)
|
||||
}
|
||||
|
||||
this.emit('notify', event)
|
||||
}
|
||||
|
||||
// API
|
||||
|
||||
getEvent(idOrAddress: string) {
|
||||
@@ -130,21 +112,20 @@ export class Repository<E extends TrustedEvent> extends Emitter implements IRead
|
||||
|
||||
*query(filters: Filter[]) {
|
||||
for (let filter of filters) {
|
||||
let events: Iterable<E> = this.eventsById.values()
|
||||
let events: TrustedEvent[] = Array.from(this.eventsById.values())
|
||||
|
||||
if (filter.ids) {
|
||||
events = filter.ids!.map(id => this.eventsById.get(id)).filter(identity) as E[]
|
||||
events = filter.ids!.map(id => this.eventsById.get(id)).filter(identity) as TrustedEvent[]
|
||||
filter = omit(['ids'], filter)
|
||||
} else if (filter.authors) {
|
||||
events = uniq(filter.authors!.flatMap(pubkey => this.eventsByAuthor.get(pubkey) || []))
|
||||
filter = omit(['authors'], filter)
|
||||
} else if (filter.since || filter.until) {
|
||||
const sinceDay = getDay(filter.since || 0)
|
||||
const untilDay = getDay(filter.since || now())
|
||||
const untilDay = getDay(filter.until || now())
|
||||
|
||||
filter = omit(['since', 'until'], filter)
|
||||
events = uniq(
|
||||
Array.from(range(sinceDay, untilDay))
|
||||
Array.from(range(sinceDay, inc(untilDay)))
|
||||
.flatMap((day: number) => this.eventsByDay.get(day) || [])
|
||||
)
|
||||
} else {
|
||||
@@ -162,79 +143,67 @@ export class Repository<E extends TrustedEvent> extends Emitter implements IRead
|
||||
}
|
||||
}
|
||||
|
||||
for (const event of events) {
|
||||
let i = 0
|
||||
|
||||
for (const event of sortBy((e: TrustedEvent) => -e.created_at, events)) {
|
||||
if (filter.limit && i > filter.limit) {
|
||||
break
|
||||
}
|
||||
|
||||
if (!this.isDeleted(event) && matchFilter(filter, event)) {
|
||||
yield event
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
publish(event: E) {
|
||||
const duplicateById = this.eventsById.get(event.id)
|
||||
|
||||
if (duplicateById) {
|
||||
return false
|
||||
publish(event: TrustedEvent, {notify = false} = {}) {
|
||||
if (!isTrustedEvent(event)) {
|
||||
throw new Error("Invalid event published to Repository", event)
|
||||
}
|
||||
|
||||
const hasAddress = isReplaceable(event)
|
||||
const address = encodeAddress(addressFromEvent(event))
|
||||
const duplicateByAddress = hasAddress ? this.eventsByAddress.get(address) : undefined
|
||||
const address = getAddress(event)
|
||||
const duplicate = (
|
||||
this.eventsById.get(event.id) ||
|
||||
this.eventsByAddress.get(address)
|
||||
)
|
||||
|
||||
if (duplicateByAddress && duplicateByAddress.created_at >= event.created_at) {
|
||||
return false
|
||||
}
|
||||
// If our duplicate is newer than the event we're adding, we're done
|
||||
if (!duplicate || duplicate.created_at < event.created_at) {
|
||||
this.eventsById.set(event.id, event)
|
||||
|
||||
this._addEvent(event, duplicateByAddress)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
isDeleted(event: E) {
|
||||
const idDeletedAt = this.deletes.get(event.id) || 0
|
||||
|
||||
if (idDeletedAt > event.created_at) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (isReplaceable(event)) {
|
||||
const address = encodeAddress(addressFromEvent(event))
|
||||
const addressDeletedAt = this.deletes.get(address) || 0
|
||||
|
||||
if (addressDeletedAt > event.created_at) {
|
||||
return true
|
||||
if (isReplaceable(event)) {
|
||||
this.eventsByAddress.set(address, event)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
if (duplicate) {
|
||||
this.eventsById.delete(duplicate.id)
|
||||
|
||||
// Implementation
|
||||
if (isReplaceable(duplicate)) {
|
||||
this.eventsByAddress.delete(address)
|
||||
}
|
||||
}
|
||||
|
||||
_addEvent(event: E, duplicate?: E) {
|
||||
this.eventsById.set(event.id, event)
|
||||
this._updateIndex(this.eventsByDay, getDay(event.created_at), event, duplicate)
|
||||
this._updateIndex(this.eventsByAuthor, event.pubkey, event, duplicate)
|
||||
|
||||
if (isReplaceable(event)) {
|
||||
this.eventsByAddress.set(encodeAddress(addressFromEvent(event)), event)
|
||||
}
|
||||
// Store our event by tags
|
||||
for (const tag of event.tags) {
|
||||
if (tag[0].length === 1) {
|
||||
this._updateIndex(this.eventsByTag, tag.slice(0, 2).join(':'), event, duplicate)
|
||||
|
||||
this._updateIndex(this.eventsByDay, getDay(event.created_at), event, duplicate)
|
||||
this._updateIndex(this.eventsByAuthor, event.pubkey, event, duplicate)
|
||||
if (event.kind === DELETE) {
|
||||
const id = tag[1]
|
||||
const ts = Math.max(event.created_at, this.deletes.get(tag[1]) || 0)
|
||||
|
||||
for (const tag of event.tags) {
|
||||
if (tag[0].length === 1) {
|
||||
this._updateIndex(this.eventsByTag, tag.slice(0, 2).join(':'), event, duplicate)
|
||||
|
||||
if (event.kind === DELETE) {
|
||||
const id = tag[1]
|
||||
const ts = Math.max(event.created_at, this.deletes.get(tag[1]) || 0)
|
||||
|
||||
this.deletes.set(id, ts)
|
||||
this.deletes.set(id, ts)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.isDeleted(event)) {
|
||||
if (notify && !this.isDeleted(event)) {
|
||||
// Deletes are tricky, re-evaluate all subscriptions if that's what we're dealing with
|
||||
if (event.kind === DELETE) {
|
||||
this.notify()
|
||||
@@ -244,11 +213,31 @@ export class Repository<E extends TrustedEvent> extends Emitter implements IRead
|
||||
}
|
||||
}
|
||||
|
||||
_updateIndex<K>(m: Map<K, E[]>, k: K, e: E, duplicate?: E) {
|
||||
isDeleted(event: TrustedEvent) {
|
||||
const idDeletedAt = this.deletes.get(event.id) || 0
|
||||
|
||||
if (idDeletedAt > event.created_at) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (isReplaceable(event)) {
|
||||
const addressDeletedAt = this.deletes.get(getAddress(event)) || 0
|
||||
|
||||
if (addressDeletedAt > event.created_at) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Utilities
|
||||
|
||||
_updateIndex<K>(m: Map<K, TrustedEvent[]>, k: K, e: TrustedEvent, duplicate?: TrustedEvent) {
|
||||
let a = m.get(k) || []
|
||||
|
||||
if (duplicate) {
|
||||
a = a.filter((x: E) => x !== duplicate)
|
||||
a = a.filter((x: TrustedEvent) => x !== duplicate)
|
||||
}
|
||||
|
||||
a.push(e)
|
||||
|
||||
@@ -2,7 +2,7 @@ import {first, splitAt, identity, sortBy, uniq, shuffle, pushToMapKey} from '@we
|
||||
import {Tags, Tag} from '@welshman/util'
|
||||
import type {TrustedEvent} from './Events'
|
||||
import {getAddress, isReplaceable} from './Events'
|
||||
import {isShareableRelayUrl} from './Relays'
|
||||
import {isShareableRelayUrl} from './Relay'
|
||||
import {addressFromEvent, decodeAddress, isCommunityAddress, isGroupAddress} from './Address'
|
||||
|
||||
export enum RelayMode {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {EventTemplate} from 'nostr-tools'
|
||||
import type {OmitStatics} from '@welshman/lib'
|
||||
import {Fluent, ensurePlural} from '@welshman/lib'
|
||||
import {isShareableRelayUrl, normalizeRelayUrl} from './Relays'
|
||||
import {isShareableRelayUrl, normalizeRelayUrl} from './Relay'
|
||||
import type {Address} from './Address'
|
||||
import {encodeAddress, decodeAddress} from './Address'
|
||||
import {GROUP, COMMUNITY} from './Kinds'
|
||||
|
||||
@@ -4,7 +4,6 @@ export * from './Filters'
|
||||
export * from './Kinds'
|
||||
export * from './Links'
|
||||
export * from './Relay'
|
||||
export * from './Relays'
|
||||
export * from './Repository'
|
||||
export * from './Router'
|
||||
export * from './Tags'
|
||||
|
||||
Reference in New Issue
Block a user