More re-work of Tags

This commit is contained in:
Jon Staab
2024-01-24 09:30:17 -08:00
parent 4fb072ec0a
commit aa963af77e
8 changed files with 427 additions and 160 deletions
+8
View File
@@ -1,8 +1,12 @@
export * from "./util/misc"
export * from "./util/nostr"
export * from "./util/LRUCache"
export * from "./util/Deferred"
export * from "./util/Emitter"
export * from "./util/Queue"
export * from "./util/Tag"
export * from "./util/Tags"
export * from "./util/Fluent"
export * from "./connect/Socket"
export * from "./connect/Connection"
export * from "./connect/ConnectionMeta"
@@ -13,3 +17,7 @@ export * from "./connect/target/Plex"
export * from "./connect/target/Relay"
export * from "./connect/target/Relays"
export * from "./connect/target/Multi"
export * from "./target/Plex"
export * from "./target/Relay"
export * from "./target/Relays"
export * from "./target/Multi"
+218
View File
@@ -0,0 +1,218 @@
/*
Smart relay selection
From Mike Dilger:
1) Other people's write relays — pull events from people you follow,
including their contact lists
2) Other people's read relays — push events that tag them (replies or just tagging).
However, these may be authenticated, use with caution
3) Your write relays —- write events you post to your microblog feed for the
world to see. ALSO write your contact list. ALSO read back your own contact list.
4) Your read relays —- read events that tag you. ALSO both write and read
client-private data like client configuration events or anything that the world
doesn't need to see.
5) Advertise relays — write and read back your own relay list
*/
export type RouterOptions = {
hintLimit: number
getUserPubkey: () => string[][]
getGroupRelayTags: (address: string) => string[][]
getPubkeyRelayTags: (pubkey: string) => string[][]
getRelayQuality: (url: string) => number
}
export class Router {
constructor(readonly options: RouterOptions) {}
FetchUserDMs = () => new RouterScenario(() => {
const tags = Tags.from(this.options.getPubkeyRelayTags(this.options.getUserPubkey()))
return tags.
})
}
export class RouterScenario {
constructor(readonly getAllHints) {}
getHints = (limit: number) => this.getAllHints().slice(0, limit)
}
Router.getHints(
export const selectHints = (hints: Iterable<string>, limit: number = null) => {
const {FORCE_RELAYS} = env.get()
const seen = new Set()
const ok = []
const bad = []
if (!limit) {
limit = getSetting("relay_limit")
}
for (const url of FORCE_RELAYS.length > 0 ? FORCE_RELAYS : hints) {
if (seen.has(url)) {
continue
}
seen.add(url)
// Skip relays that just shouldn't ever be published
if (!isShareableRelay(url)) {
continue
}
// Filter out relays that appear to be broken or slow
if (relayIsLowQuality(url)) {
bad.push(url)
} else {
ok.push(url)
}
if (ok.length > limit) {
break
}
}
// If we don't have enough hints, use the broken ones
const result = ok.concat(bad).slice(0, limit)
if (result.length === 0) {
warn("No results returned from selectHints")
}
return result
}
export const selectHintsWithFallback = (hints: Iterable<string> = null, limit = null) =>
selectHints(chain(hints || [], getUserRelayUrls(RelayMode.Read), env.get().DEFAULT_RELAYS), limit)
export class HintSelector {
constructor(
readonly generateHints,
readonly hintsLimit = null,
) {}
limit = hintsLimit => new HintSelector(this.generateHints, hintsLimit)
getHints = (...args) =>
selectHints(this.generateHints(...args), this.hintsLimit || getSetting("relay_limit"))
}
export const hintSelector = (generateHints: (...args: any[]) => Iterable<string>) => {
const selector = new HintSelector(generateHints)
const getHints = selector.getHints
;(getHints as any).limit = selector.limit
return getHints as typeof getHints & {limit: typeof selector.limit}
}
export const getPubkeyHints = hintSelector(function* (pubkey: string, mode: RelayMode) {
yield* getPubkeyRelayUrls(pubkey, mode)
})
export const getPubkeyHint = (pubkey: string): string =>
first(getPubkeyHints(1, pubkey, "write")) || ""
export const getUserHints = hintSelector(function* (mode: RelayMode) {
yield* getUserRelayUrls(mode)
})
export const getUserHint = (pubkey: string): string => first(getUserHints(1, "write")) || ""
export const getEventHints = hintSelector(function* (event: Event) {
for (const address of Tags.from(event).circles().all()) {
yield* getGroupHints(address)
}
yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Write)
yield* event.seen_on.filter(isShareableRelay)
})
export const getEventHint = (event: Event) => first(getEventHints.limit(1).getHints(event)) || ""
// If we're looking for an event's children, the read relays the author has
// advertised would be the most reliable option, since well-behaved clients
// will write replies there.
export const getReplyHints = hintSelector(function* (event) {
for (const address of Tags.from(event).circles().all()) {
yield* getGroupHints(address)
}
yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
})
// If we're looking for an event's parent, tags are the most reliable hint,
// but we can also look at where the author of the note reads from
export const getParentHints = hintSelector(function* (event) {
yield* Tags.from(event).getReplyHints()
yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
})
export const getRootHints = hintSelector(function* (event) {
yield* Tags.from(event).getRootHints()
yield* getPubkeyRelayUrls(event.pubkey, RelayMode.Read)
})
// If we're replying or reacting to an event, we want the author to know, as well as
// anyone else who is tagged in the original event or the reply. Get everyone's read
// relays. Limit how many per pubkey we publish to though. We also want to advertise
// our content to our followers, so publish to our write relays as well.
export const getPublishHints = hintSelector(function* (event: Event) {
for (const address of Tags.from(event).circles().all()) {
yield* getGroupHints(address)
}
const pubkeys = Tags.from(event).type("p").values().all()
const hintGroups = pubkeys.map(pubkey => getPubkeyRelayUrls(pubkey, RelayMode.Read))
const authorRelays = getPubkeyRelayUrls(event.pubkey, RelayMode.Write)
yield* mergeHints([...hintGroups, authorRelays, getUserHints(RelayMode.Write)])
})
export const getInboxHints = hintSelector(function* (pubkeys: string[]) {
yield* mergeHints(pubkeys.map(pk => getPubkeyHints(pk, "read")))
})
export const getGroupHints = hintSelector(function* (address: string) {
yield* getGroupRelayUrls(address)
yield* getPubkeyHints(Naddr.fromTagValue(address).pubkey)
})
export const getGroupPublishHints = (addresses: string[]) => {
const urls = mergeHints(addresses.map(getGroupRelayUrls))
return urls.length === 0 ? getUserHints("write") : urls
}
export const mergeHints = (groups: string[][], limit: number = null) => {
const scores = {} as Record<string, any>
for (const hints of groups) {
hints.forEach((hint, i) => {
const score = 1 / (i + 1) / hints.length
if (!scores[hint]) {
scores[hint] = {score: 0, count: 0}
}
scores[hint].score += score
scores[hint].count += 1
})
}
// Use the log-sum-exp and a weighted sum
for (const score of Object.values(scores)) {
const weight = Math.log(groups.length / score.count)
score.score = weight + Math.log1p(Math.exp(score.score - score.count))
}
return sortBy(([hint, {score}]) => score, Object.entries(scores))
.map(nth(0))
.slice(0, limit || getSetting("relay_limit"))
}
+49
View File
@@ -0,0 +1,49 @@
import {last} from './misc'
export class Fluent<T> {
constructor(readonly parts: T[]) {}
static from<T>(parts: Iterable<T>) {
return new Fluent<T>(Array.from(parts))
}
clone<K extends Fluent<T>>(this: K, parts: T[]): K {
return new (this.constructor as { new (parts: T[]): K })(parts)
}
*[Symbol.iterator]() {
for (const x of this.parts) {
yield x
}
}
first = () => Array.from(this.parts)[0]
nth = (i: number) => Array.from(this.parts)[i]
last = () => last(Array.from(this.parts))
count = () => Array.from(this.parts).length
exists = () => Array.from(this.parts).length > 0
every = (f: (t: T) => boolean) => Array.from(this.parts).every(f)
some = (f: (t: T) => boolean) => Array.from(this.parts).some(f)
find = (f: (t: T) => boolean) => Array.from(this.parts).find(f)
uniq = () => this.clone(Array.from(new Set(this.parts)))
slice = (a: number, b?: number) => this.clone(Array.from(this.parts).slice(a, b))
take = (n: number) => this.slice(0, n)
drop = (n: number) => this.slice(n)
filter = (f: (t: T) => boolean) => this.clone(Array.from(this.parts).filter(f))
reject = (f: (t: T) => boolean) => this.clone(Array.from(this.parts).filter(t => !f(t)))
map = <U>(f: (t: T) => U) => new Fluent(Array.from(this.parts).map(f))
}
+12 -18
View File
@@ -1,25 +1,19 @@
export class Tag {
constructor(readonly parts: string[]) {}
import type {OmitStatics} from './misc'
import {last} from './misc'
import {Fluent} from './Fluent'
key() {
return this.parts[0]
export class Tag extends (Fluent<string> as OmitStatics<typeof Fluent<string>, 'from'>) {
static from(parts: Iterable<string>) {
return new Tag(Array.from(parts))
}
val() {
return this.parts[1]
}
key = () => this.parts[0]
mark() {
return this.parts.slice(0, 2).slice(-1)[0]
}
value = () => this.parts[1]
nth(n: number) {
return this.parts[n]
}
mark = () => last(this.parts.slice(2))
*[Symbol.iterator]() {
for (const x of this.parts) {
yield x
}
}
entry = () => this.parts.slice(0, 2)
append = (s: string) => Tag.from(this.parts.concat(s))
}
+114 -129
View File
@@ -1,161 +1,146 @@
export class Fluent<T> {
ItemClass?: Fluent<T>
import type {Event} from 'nostr-tools'
import {Tag} from './Tag'
import {Fluent} from './Fluent'
import type {OmitStatics} from './misc'
import {isIterable, uniq} from './misc'
import {isShareableRelay} from './nostr'
import {isCommunityAddress, isGroupAddress, isCommunityOrGroupAddress} from './kinds'
constructor(value: T[]) {
this.value = value.filter(identity)
export class Tags extends (Fluent<Tag> as OmitStatics<typeof Fluent<Tag>, 'from'>) {
static from(p: Iterable<Tag | string[]>) {
return new Tags(Array.from(p).map(Tag.from))
}
static create(value: T[]) {
this.value = value.filter(identity)
static fromEvent(event: Event) {
return Tags.from(event.tags)
}
item(item: T) {
const {ItemClass} = this.constructor
return ItemClass ? ItemClass.create(item) : item
static fromEvents(events: Event[]) {
return Tags.from(events.flatMap((e: Event) => e?.tags))
}
valueOf = () => this.value
// General purpose filters
count = () => this.value.length
whereKey = (key: string) => this.filter(t => t.key() === key)
exists = () => this.value.length > 0
whereValue = (value: string) => this.filter(t => t.value() === value)
f = <U>(f: (t: T) => U) => f(this.value)
whereMark = (mark: string) => this.filter(t => t.mark() === mark)
any = (f: (t: T) => boolean) => this.value.any(f)
// General purpose methods that return a list of values
every = (f: (t: T) => boolean) => this.value.every(f)
keys = () => new Fluent(this.parts.map(t => t.key()))
some = (f: (t: T) => boolean) => this.value.some(f)
values = () => new Fluent(this.parts.map(t => t.value()))
first = () => this.item(this.values[0])
marks = () => new Fluent(this.parts.map(t => t.mark()))
nth = (i: number) => this.item(this.values[i])
last = () => this.item(last(this.values))
find = (f: (t: T) => boolean) => this.item(this.value.find(f))
filter = (f: (t: T) => boolean) => this.constructor.create(this.value.filter(f))
reject = (f: (t: T) => boolean) => this.constructor.create(this.value.filter(t => !f(t)))
entries = () => new Fluent(this.parts.map(t => t.entry()))
}
export class Tag extends Fluent<string[]> {
type = () => this.value[0]
export type CoercibleToTags = Event | Iterable<Event> | Tags | Tag | Tag[] | string[] | Iterable<string[]>
value = () => this.value[1]
export const coerceToTags = (x: CoercibleToTags) => {
const xs = isIterable(x) ? Array.from(x as Iterable<any>) : [x]
mark = () => last(this.value)
if (xs.length === 0) {
return new Tags(xs)
}
if (xs[0] instanceof Event) {
return Tags.fromEvents(xs)
}
if (xs[0] instanceof Array) {
return Tags.from(xs)
}
if (typeof xs[0] === 'string') {
return Tags.from([xs])
}
throw new Error('Received invalid value to coerceToTags: ${x}')
}
export class Tags extends Fluent<string[][]> {
ItemClass: Tag
export const getRelays = (x: CoercibleToTags) =>
uniq(Array.from(coerceToTags(x)).flatMap((t: Tag) => Array.from(t)).filter(isShareableRelay))
static from (e: Event | Event[]) {
const events = Array.isArray(e) ? e : [e]
export const getTopics = (x: CoercibleToTags) =>
Array.from(coerceToTags(x).whereKey("t").values()).map((t: string) => t.replace(/^#/, ""))
return new Tags(events.flatMap(e => e?.tags))
}
export const getPubkeys = (x: CoercibleToTags) =>
Array.from(coerceToTags(x).whereKey("p").values())
where(conditions: Record<string, (x: any) => boolean>) {
return this.filter(t => {
const tag = new Tag(t)
export const getUrls = (x: CoercibleToTags) =>
Array.from(coerceToTags(x).whereKey("r").values())
for ([k, f] of Object.entries(conditions)) {
if (!f(tag[k]())) {
return false
}
}
return true
})
}
whereEq(conditions: Record<string, any>) {
return this.filter(t => {
const tag = new Tag(t)
for ([k, v] of Object.entries(conditions)) {
v = Array.isArray(v) ? v : [v]
if (!v.includes(tag[k]())) {
return false
}
}
return true
})
}
value = () => this.value.find(t => t[1])
values = () => this.value.map(t => t[1])
relays = () => uniq(flatten(this.value).filter(isShareableRelay))
topics = () => this.whereEq({type: "t"}).values().map((t: string) => t.replace(/^#/, ""))
pubkeys = () => this.whereEq({type: "p"}).values()
urls = () => this.whereEq({type: "r"}).values()
getDict() {
const meta: Record<string, string> = {}
for (const [k, v] of this.value) {
if (!meta[k]) {
meta[k] = v
}
}
return meta
}
getAncestorsLegacy() {
// Legacy only supports e tags. Normalize their length to 3
const eTags = this.whereEq({type: "e"}).map(t => {
while (t.length < 3) {
t.push("")
export const getAncestorsLegacy = (x: CoercibleToTags) => {
// Legacy only supports e tags. Normalize their length to 3
const eTags = Tags.from(
coerceToTags(x).whereKey("e").map((t: Tag) => {
while (t.count() < 3) {
t.append("")
}
return t.slice(0, 3)
})
)
return {
roots: eTags.count() > 1 ? new Tags([eTags.first()]) : new Tags([]),
replies: new Tags([eTags.last()]),
mentions: new Tags(eTags.all().slice(1, -1)),
}
return {
roots: eTags.slice(0, 1),
replies: eTags.slice(-1),
mentions: eTags.slice(1, -1),
}
getAncestors(type = null) {
// If we have a mark, we're not using the legacy format
if (!this.any(t => t.length === 4 && ["reply", "root", "mention"].includes(last(t)))) {
return this.getAncestorsLegacy()
}
const tags = new Tags(this.whereEq({type: type || ["}a", "e"]).all().filter(t => !String(t[1]).startsWith('34550:')))
return {
roots: new Tags(tags.mark('root').take(3).all()),
replies: new Tags(tags.mark('reply').take(3).all()),
mentions: new Tags(tags.mark('mention').take(3).all()),
}
}
roots = (type = null) => this.getAncestors(type).roots
replies = (type = null) => this.getAncestors(type).replies
communities = () => this.whereEq({type: "a"}).values().filter(a => a.startsWith('34550:'))
getReply = (type = null) => this.replies(type).values().first()
getRoot = (type = null) => this.roots(type).values().first()
getReplyHints = (type = null) => this.replies(type).relays().all()
getRootHints = (type = null) => this.roots(type).relays().all()
}
type GetAncestorsReturn = {
roots: Tags
replies: Tags
mentions: Tags
}
export const getAncestors = (x: CoercibleToTags, key?: string): GetAncestorsReturn => {
const tags = coerceToTags(x)
// If we have a mark, we're not using the legacy format
if (!tags.some((t: Tag) => t.count() === 4 && ["reply", "root", "mention"].includes(t.mark()))) {
return getAncestorsLegacy(tags)
}
const eTags = tags.whereKey("e")
const aTags = tags.whereKey("a").reject((t: Tag) => isCommunityOrGroupAddress(t.value()))
const allTags = coerceToTags([...eTags, ...aTags])
return {
roots: allTags.whereMark('root').take(3),
replies: allTags.whereMark('reply').take(3),
mentions: allTags.whereMark('mention').take(3),
}
}
export const getRoots = (x: CoercibleToTags, key?: string) =>
getAncestors(x, key).roots
export const getReplies = (x: CoercibleToTags, key?: string) =>
getAncestors(x, key).replies
export const getGroups = (x: CoercibleToTags) =>
coerceToTags(x).whereKey("a").values().filter(isGroupAddress)
export const getCommunities = (x: CoercibleToTags) =>
coerceToTags(x).whereKey("a").values().filter(isCommunityAddress)
export const getCommunitiesAndGroups = (x: CoercibleToTags) =>
coerceToTags(x).whereKey("a").values().filter(isCommunityOrGroupAddress)
export const getRoot = (x: CoercibleToTags, key?: string) =>
getRoots(x, key).values().first()
export const getReply = (x: CoercibleToTags, key?: string) =>
getReplies(x, key).values().first()
export const getRootHints = (x: CoercibleToTags, key?: string) =>
getRelays(getRoots(x, key))
export const getReplyHints = (x: CoercibleToTags, key?: string) =>
getRelays(getReplies(x, key))
+8
View File
@@ -0,0 +1,8 @@
export const GROUP = 35834
export const COMMUNITY = 34550
export const isGroupAddress = (a: string) => a.startsWith(`${GROUP}:`)
export const isCommunityAddress = (a: string) => a.startsWith(`${COMMUNITY}:`)
export const isCommunityOrGroupAddress = (a: string) => isCommunityAddress(a) || isGroupAddress(a)
+17
View File
@@ -0,0 +1,17 @@
export const now = () => Math.round(Date.now() / 1000)
export const last = <T>(xs: T[]) => xs[xs.length - 1]
export const identity = <T>(x: T) => x
export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
// https://github.com/microsoft/TypeScript/issues/4628#issuecomment-1147905253
export type OmitStatics<T, S extends string> =
T extends {new(...args: infer A): infer R} ?
{new(...args: A): R}&Omit<T, S> :
Omit<T, S>;
export const isIterable = (x: any) => Symbol.iterator in Object(x)
+1 -13
View File
@@ -2,19 +2,7 @@ import type {Event} from 'nostr-tools'
import normalizeUrl from "normalize-url"
import {verifyEvent, getEventHash, matchFilter as nostrToolsMatchFilter} from 'nostr-tools'
import {cached} from "./LRUCache"
// ===========================================================================
// General-purpose
export const now = () => Math.round(Date.now() / 1000)
export const last = <T>(xs: T[]) => xs[xs.length - 1]
export const identity = <T>(x: T) => x
export const flatten = <T>(xs: T[]) => xs.flatMap(identity)
export const uniq = <T>(xs: T[]) => Array.from(new Set(xs))
import {now} from './misc'
// ===========================================================================
// Relays