Add in-memory relay

packages/feeds/core.ts

@@ -1,10 +1,10 @@
 import type {Filter} from '@coracle.social/util'
 
 export enum FeedType {
-  Difference = "\\",
-  Intersection = "∩",
-  SymmetricDifference = "Δ",
-  Union = "∪",
+  Difference = "difference",
+  Intersection = "intersection",
+  SymmetricDifference = "symdiff",
+  Union = "union",
   Filter = "filter",
   Relay = "relay",
   List = "list",
@@ -15,7 +15,6 @@ export enum FeedType {
 export enum Scope {
   Followers = "followers",
   Follows = "follows",
-  Global = "global",
   Network = "network",
   Self = "self",
 }
@@ -60,6 +59,15 @@ export const listFeed = (...addresses: string[]) => [FeedType.List, ...addresses
 export const lolFeed = (...addresses: string[]) => [FeedType.LOL, ...addresses] as Feed
 export const dvmFeed = (...requests: DVMItem[]) => [FeedType.DVM, ...requests] as Feed
 
+export const hasSubFeeds = ([type]: Feed) =>
+  [
+    FeedType.Union,
+    FeedType.Intersection,
+    FeedType.Difference,
+    FeedType.SymmetricDifference,
+    FeedType.Relay,
+  ].includes(type)
+
 export const getSubFeeds = ([type, ...feed]: Feed): Feed[] => {
   switch (type) {
     case FeedType.Relay:

packages/lib/Tools.ts

@@ -33,6 +33,34 @@ export const drop = <T>(n: number, xs: T[]) => xs.slice(n)
 
 export const take = <T>(n: number, xs: T[]) => xs.slice(0, n)
 
+export const omit = <T extends Record<string, any>>(ks: string[], x: T) => {
+  const r: T = {...x}
+
+  for (const k of ks) {
+    delete r[k]
+  }
+
+  return r
+}
+
+export const pick = <T extends Record<string, any>>(ks: string[], x: T) => {
+  const r: T = {...x}
+
+  for (const k of Object.keys(x)) {
+    if (!ks.includes(k)) {
+      delete r[k]
+    }
+  }
+
+  return r
+}
+
+export function* range(a: number, b: number, step = 1) {
+  for (let i = a; i < b; i += step) {
+    yield i
+  }
+}
+
 export const between = (low: number, high: number, n: number) => n > low && n < high
 
 export const randomId = (): string => Math.random().toString().slice(2)
@@ -169,14 +197,14 @@ export const batch = <T>(t: number, f: (xs: T[]) => void) => {
   }
 }
 
-export const addToMapKey = <T>(m: Map<string, Set<T>>, k: string, v: T) => {
+export const addToMapKey = <K, T>(m: Map<K, Set<T>>, k: K, v: T) => {
   const a = m.get(k) || new Set<T>()
 
   a.add(v)
   m.set(k, a)
 }
 
-export const pushToMapKey = <T>(m: Map<string, T[]>, k: string, v: T) => {
+export const pushToMapKey = <K, T>(m: Map<K, T[]>, k: K, v: T) => {
   const a = m.get(k) || []
 
   a.push(v)

packages/network/ConnectionMeta.ts

@@ -1,6 +1,6 @@
 import type {Event, Filter} from 'nostr-tools'
+import type {Message} from '@coracle.social/util'
 import type {Connection} from './Connection'
-import type {Message} from './Socket'
 
 export type PublishMeta = {
   sent: number

packages/network/Executor.ts

@@ -1,7 +1,7 @@
 import type {Event, Filter} from 'nostr-tools'
 import type {Emitter} from '@coracle.social/lib'
+import type {Message} from '@coracle.social/util'
 import type {Connection} from './Connection'
-import type {Message} from './Socket'
 import {NetworkContext} from './Context'
 
 export type Target = Emitter & {

packages/network/Socket.ts

@@ -1,7 +1,6 @@
 import WebSocket from "isomorphic-ws"
 import {Deferred, defer} from '@coracle.social/lib'
-
-export type Message = [string, ...any[]]
+import type {Message} from '@coracle.social/util'
 
 export type PlexMessage = [{relays: string[]}, Message]
 

packages/network/target/Multi.ts

@@ -1,6 +1,6 @@
 import {Emitter} from '@coracle.social/lib'
+import type {Message} from '@coracle.social/util'
 import type {Target} from '../Executor'
-import type {Message} from '../Socket'
 
 export class Multi extends Emitter {
   constructor(readonly targets: Target[]) {

packages/network/target/Plex.ts

@@ -1,5 +1,6 @@
 import {Emitter} from '@coracle.social/lib'
-import type {PlexMessage, Message} from '../Socket'
+import type {Message} from '@coracle.social/util'
+import type {PlexMessage} from '../Socket'
 import type {Connection} from '../Connection'
 
 export class Plex extends Emitter {

packages/network/target/Relay.ts

@@ -1,5 +1,5 @@
 import {Emitter} from '@coracle.social/lib'
-import type {Message} from '../Socket'
+import type {Message} from '@coracle.social/util'
 import type {Connection} from '../Connection'
 
 export class Relay extends Emitter {

packages/network/target/Relays.ts

@@ -1,5 +1,5 @@
 import {Emitter} from '@coracle.social/lib'
-import type {Message} from '../Socket'
+import type {Message} from '@coracle.social/util'
 import type {Connection} from '../Connection'
 
 export class Relays extends Emitter {

packages/util/Filters.ts

@@ -18,8 +18,8 @@ export type Filter = {
   [key: `#${string}`]: string[]
 }
 
-export const matchFilter = (filter: Filter, event: Event) => {
-  if (!nostrToolsMatchFilter(filter, event)) {
+export const matchFilter = <E extends Rumor>(filter: Filter, event: E) => {
+  if (!nostrToolsMatchFilter(filter, event as unknown as Event)) {
     return false
   }
 
@@ -39,7 +39,7 @@ export const matchFilter = (filter: Filter, event: Event) => {
   return true
 }
 
-export const matchFilters = (filters: Filter[], event: Event) => {
+export const matchFilters = <E extends Rumor>(filters: Filter[], event: E) => {
   for (const filter of filters) {
     if (matchFilter(filter, event)) {
       return true

packages/util/Kinds.ts

@@ -1,12 +1,12 @@
+import {kinds} from 'nostr-tools'
 import {between} from '@coracle.social/lib'
 
-export const isEphemeralKind = (kind: number) => between(19999, 29999, kind)
-
-export const isPlainReplaceableKind = (kind: number) => between(9999, 20000, kind)
-
-export const isParameterizedReplaceableKind = (kind: number) => between(29999, 40000, kind)
-
-export const isReplaceableKind = (kind: number) => isPlainReplaceableKind(kind) || isParameterizedReplaceableKind(kind)
+export const isRegularKind = kinds.isRegularKind
+export const isEphemeralKind = kinds.isEphemeralKind
+export const isPlainReplaceableKind = kinds.isReplaceableKind
+export const isParameterizedReplaceableKind = kinds.isParameterizedReplaceableKind
+export const isReplaceableKind = (kind: number) =>
+  isPlainReplaceableKind(kind) || isParameterizedReplaceableKind(kind)
 
 export const PROFILE = 0
 export const NOTE = 1

packages/util/Relay.ts

@@ -0,0 +1,182 @@
+import {Emitter, uniq, omit, now, range, identity, pushToMapKey} from '@coracle.social/lib'
+import {matchFilters, matchFilter} from './Filters'
+import {encodeAddress, addressFromEvent} from './Address'
+import {isReplaceable} from './Events'
+import type {Filter} from './Filters'
+import type {Rumor} from './Events'
+
+export const DAY = 86400
+
+const getDay = (ts: number) => Math.floor(ts / DAY)
+
+export type Message = [string, ...any[]]
+
+export class Relay<E extends Rumor> extends Emitter {
+  eventsById = new Map<string, E>()
+  eventsByAddress = new Map<string, E>()
+  eventsByTag = new Map<string, E[]>()
+  eventsByDay = new Map<number, E[]>()
+  eventsByAuthor = new Map<string, E[]>()
+  subs = new Map<string, Filter[]>()
+  deletes = new Map<string, number>()
+
+  dump() {
+    return Array.from(this.eventsById.values())
+  }
+
+  load(events: E[]) {
+    for (const event of events) {
+      this._addEvent(event)
+    }
+  }
+
+  send(type: string, ...message: any[]) {
+    switch(type) {
+      case 'EVENT': return this._onEVENT(message as [string])
+      case 'CLOSE': return this._onCLOSE(message as [string])
+      case 'REQ': return this._onREQ(message as [string, ...Filter[]])
+    }
+  }
+
+  _onEVENT([json]: [string]) {
+    let event: E
+    try {
+      event = JSON.parse(json)
+    } catch (e) {
+      return
+    }
+
+    const duplicateById = this.eventsById.get(event.id)
+
+    if (duplicateById) {
+      return
+    }
+
+    const hasAddress = isReplaceable(event)
+    const address = encodeAddress(addressFromEvent(event))
+    const duplicateByAddress = hasAddress ? this.eventsByAddress.get(address) : undefined
+
+    if (duplicateByAddress && duplicateByAddress.created_at >= event.created_at) {
+      return
+    }
+
+    this._addEvent(event, duplicateByAddress)
+
+    this.emit('OK', event.id, true, "")
+
+    if (!this._isDeleted(event)) {
+      for (const [subId, filters] of this.subs.entries()) {
+        if (matchFilters(filters, event)) {
+          this.emit('EVENT', subId, json)
+        }
+      }
+    }
+  }
+
+  _onCLOSE([subId]: [string]) {
+    this.subs.delete(subId)
+  }
+
+  _onREQ([subId, ...filters]: [string, ...Filter[]]) {
+    this.subs.set(subId, filters)
+
+    const result = new Set()
+
+    for (let filter of filters) {
+      let events: Iterable<E> = this.eventsById.values()
+
+      if (filter.ids) {
+        filter = omit(['ids'], filter)
+        events = filter.ids!.map(id => this.eventsById.get(id)).filter(identity) as E[]
+      } else if (filter.authors) {
+        filter = omit(['authors'], filter)
+        events = uniq(filter.authors!.flatMap(pubkey => this.eventsByAuthor.get(pubkey) || []))
+      } else if (filter.since || filter.until) {
+        const sinceDay = getDay(filter.since || 0)
+        const untilDay = getDay(filter.since || now())
+
+        filter = omit(['since', 'until'], filter)
+        events = uniq(
+          Array.from(range(sinceDay, untilDay))
+            .flatMap((day: number) => this.eventsByDay.get(day) || [])
+        )
+      } else {
+        for (const [k, values] of Object.entries(filter)) {
+          if (!k.startsWith('#') || k.length !== 2) {
+            continue
+          }
+
+          filter = omit([k], filter)
+          events = uniq(
+            (values as string[]).flatMap(v => this.eventsByTag.get(`${k[1]}:${v}`) || [])
+          )
+
+          break
+        }
+      }
+
+      for (const event of events) {
+        if (!this._isDeleted(event) && matchFilter(filter, event)) {
+          result.add(event)
+        }
+      }
+    }
+
+    for (const event of result) {
+      this.emit('EVENT', subId, JSON.stringify(event))
+    }
+
+    this.emit('EOSE', subId)
+  }
+
+  _isDeleted(event: E) {
+    const idDeletedAt = this.deletes.get(event.id) || 0
+
+    if (idDeletedAt > event.created_at) {
+      return true
+    }
+
+    if (isReplaceable(event)) {
+      const address = encodeAddress(addressFromEvent(event))
+      const addressDeletedAt = this.deletes.get(address) || 0
+
+      if (addressDeletedAt > event.created_at) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  _addEvent(event: E, duplicate?: E) {
+    this.eventsById.set(event.id, event)
+
+    if (isReplaceable(event)) {
+      this.eventsByAddress.set(encodeAddress(addressFromEvent(event)), event)
+    }
+
+    this._updateIndex(this.eventsByDay, getDay(event.created_at), event, duplicate)
+    this._updateIndex(this.eventsByAuthor, event.pubkey, event, duplicate)
+
+    for (const tag of event.tags) {
+      if (tag[0].length === 1) {
+        this._updateIndex(this.eventsByTag, tag.slice(0, 2).join(':'), event, duplicate)
+
+        if (event.kind === 5) {
+          this.deletes.set(tag[1], Math.max(event.created_at, this.deletes.get(tag[1]) || 0))
+        }
+      }
+    }
+  }
+
+  _updateIndex<K>(m: Map<K, E[]>, k: K, e: E, duplicate?: E) {
+    let a = m.get(k) || []
+
+    if (duplicate) {
+      a = a.filter((x: E) => x !== duplicate)
+    }
+
+    a.push(e)
+    m.set(k, a)
+  }
+}

packages/util/Relays.ts

@@ -27,8 +27,8 @@ export const normalizeRelayUrl = (url: string, {allowInsecure = false}: Normaliz
   // Use our library to normalize
   url = normalizeUrl(url, {stripHash: true, stripAuthentication: false})
 
-  // Strip the protocol since only wss works
-  url = stripProtocol(url)
+  // Strip the protocol since only wss works, lowercase
+  url = stripProtocol(url).toLowerCase()
 
   // Urls without pathnames are supposed to have a trailing slash
   if (!url.includes("/")) {

packages/util/index.ts

@@ -3,6 +3,7 @@ export * from './Events'
 export * from './Filters'
 export * from './Kinds'
 export * from './Links'
+export * from './Relay'
 export * from './Relays'
 export * from './Router'
 export * from './Tags'