diff --git a/package-lock.json b/package-lock.json index 36a84ed4..6fc72b00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,7 @@ "@welshman/app": "~0.0.36", "@welshman/content": "~0.0.15", "@welshman/dvm": "~0.0.13", - "@welshman/editor": "~0.0.3", + "@welshman/editor": "~0.0.4", "@welshman/feeds": "~0.0.29", "@welshman/lib": "~0.0.37", "@welshman/net": "~0.0.45", @@ -4749,9 +4749,9 @@ } }, "node_modules/@welshman/editor": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/@welshman/editor/-/editor-0.0.3.tgz", - "integrity": "sha512-H0P4yktjoivyWATaAcaWM0LQ124Yrj9h9J27MD+kTingcF1aWegTnQ7lPlozIJEhuit6P8G7ASzENW+BKBwKpA==", + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@welshman/editor/-/editor-0.0.4.tgz", + "integrity": "sha512-tcMwLuBaBtT2JgON5f+Fd4Cg9oM7QMnXW9voGP+RqH1gJt0W6rjjQCtpqEcgdVtHhmaSL1P+tM4ORmOinoCv+A==", "peerDependencies": { "@tiptap/core": "^2.9.1", "@tiptap/extension-code": "^2.9.1", @@ -4766,8 +4766,8 @@ "@tiptap/extension-text": "^2.9.1", "@tiptap/pm": "^2.9.1", "@tiptap/suggestion": "^2.9.1", - "@welshman/lib": "^0.0.36", - "@welshman/util": "^0.0.53", + "@welshman/lib": "~0.0.36", + "@welshman/util": "~0.0.53", "nostr-editor": "github:cesardeazevedo/nostr-editor#a211491c", "nostr-tools": "^2.8.1", "svelte": "^4.0.0", diff --git a/package.json b/package.json index 157d1cf8..4cc568c4 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "@welshman/app": "~0.0.36", "@welshman/content": "~0.0.15", "@welshman/dvm": "~0.0.13", - "@welshman/editor": "~0.0.3", + "@welshman/editor": "~0.0.4", "@welshman/feeds": "~0.0.29", "@welshman/lib": "~0.0.37", "@welshman/net": "~0.0.45", diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 60089ef4..9c3d6a22 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -15,7 +15,6 @@ HOUR, WEEK, Worker, - throttle, } from "@welshman/lib" import type {TrustedEvent} from "@welshman/util" import { @@ -29,7 +28,6 @@ getPubkeyTagValues, getListTags, } from "@welshman/util" - import {custom} from "@welshman/store" import { relays, handles, @@ -91,114 +89,6 @@ ...notifications, }) - const getScoreEvent = () => { - const ALWAYS_KEEP = Infinity - const NEVER_KEEP = 0 - - const reactionKinds = [REACTION, ZAP_RESPONSE] - const metaKinds = [PROFILE, FOLLOWS, RELAYS, INBOX_RELAYS] - const $sessionKeys = new Set(Object.keys(app.sessions.get())) - const $userFollows = new Set(getPubkeyTagValues(getListTags(get(app.userFollows)))) - const $maxWot = get(app.maxWot) - - return (e: TrustedEvent) => { - const isFollowing = $userFollows.has(e.pubkey) - - // No need to keep a record of everyone who follows the current user - if (e.kind === FOLLOWS && !isFollowing) return NEVER_KEEP - - // Always keep stuff by or tagging a signed in user - if ($sessionKeys.has(e.pubkey)) return ALWAYS_KEEP - if (e.tags.some(t => $sessionKeys.has(t[1]))) return ALWAYS_KEEP - - // Get rid of irrelevant messages, reactions, and likes - if (e.wrap || e.kind === 4 || e.kind === WRAP) return NEVER_KEEP - if (reactionKinds.includes(e.kind)) return NEVER_KEEP - - // If the user follows this person, use max wot score - let score = isFollowing ? $maxWot : app.getUserWotScore(e.pubkey) - - // Inflate the score for profiles/relays/follows to avoid redundant fetches - // Demote non-metadata type events, and introduce recency bias - score *= metaKinds.includes(e.kind) ? 2 : e.created_at / now() - - return score - } - } - - const migrateFreshness = (data: {key: string; value: number}[]) => { - const cutoff = ago(HOUR) - - return data.filter(({value}) => value > cutoff) - } - - const migratePlaintext = (data: {key: string; value: number}[]) => data.slice(0, 10_000) - - const migrateEvents = (events: TrustedEvent[]) => { - if (events.length < 50_000) { - return events - } - - const scoreEvent = getScoreEvent() - - return take( - 30_000, - sortBy(e => -scoreEvent(e), events), - ) - } - - const migrate = (data: any[], options: any) => (options.migrate ? options.migrate(data) : data) - - // TODO: remove this - const fromRepositoryAndTracker = (repository: any, tracker: any, options: any = {}) => ({ - options, - keyPath: "id", - store: custom( - setter => { - let onUpdate = () => { - const events = migrate(repository.dump(), options) - - setter( - events.map((event: any) => { - const relays = Array.from(tracker.getRelays(event.id)) - - return {id: event.id, event, relays} - }), - ) - } - - if (options.throttle) { - onUpdate = throttle(options.throttle, onUpdate) - } - - onUpdate() - tracker.on("update", onUpdate) - repository.on("update", onUpdate) - - return () => { - tracker.off("update", onUpdate) - } - }, - { - set: (items: {event: TrustedEvent; relays: string[]}[]) => { - const events: TrustedEvent[] = [] - const relaysById = new Map>() - - for (const {event, relays} of items) { - if (!event) { - continue - } - events.push(event) - relaysById.set(event.id, new Set(relays)) - } - - repository.load(events) - tracker.load(relaysById) - }, - }, - ), - }) - if (!db) { setupTracking() setupAnalytics() @@ -208,15 +98,60 @@ handles: storageAdapters.fromCollectionStore("nip05", handles, {throttle: 3000}), freshness: storageAdapters.fromObjectStore(freshness, { throttle: 3000, - migrate: migrateFreshness, + migrate: (data: {key: string; value: number}[]) => { + const cutoff = ago(HOUR) + + return data.filter(({value}) => value > cutoff) + }, }), plaintext: storageAdapters.fromObjectStore(plaintext, { throttle: 3000, - migrate: migratePlaintext, + migrate: (data: {key: string; value: number}[]) => data.slice(0, 10_000), }), - events: fromRepositoryAndTracker(repository, tracker, { + events: storageAdapters.fromRepositoryAndTracker(repository, tracker, { throttle: 3000, - migrate: migrateEvents, + migrate: (events: TrustedEvent[]) => { + if (events.length < 50_000) { + return events + } + + const NEVER_KEEP = 0 + const ALWAYS_KEEP = Infinity + const reactionKinds = [REACTION, ZAP_RESPONSE] + const metaKinds = [PROFILE, FOLLOWS, RELAYS, INBOX_RELAYS] + const $sessionKeys = new Set(Object.keys(app.sessions.get())) + const $userFollows = new Set(getPubkeyTagValues(getListTags(get(app.userFollows)))) + const $maxWot = get(app.maxWot) + + const scoreEvent = (e: TrustedEvent) => { + const isFollowing = $userFollows.has(e.pubkey) + + // No need to keep a record of everyone who follows the current user + if (e.kind === FOLLOWS && !isFollowing) return NEVER_KEEP + + // Always keep stuff by or tagging a signed in user + if ($sessionKeys.has(e.pubkey)) return ALWAYS_KEEP + if (e.tags.some(t => $sessionKeys.has(t[1]))) return ALWAYS_KEEP + + // Get rid of irrelevant messages, reactions, and likes + if (e.wrap || e.kind === 4 || e.kind === WRAP) return NEVER_KEEP + if (reactionKinds.includes(e.kind)) return NEVER_KEEP + + // If the user follows this person, use max wot score + let score = isFollowing ? $maxWot : app.getUserWotScore(e.pubkey) + + // Inflate the score for profiles/relays/follows to avoid redundant fetches + // Demote non-metadata type events, and introduce recency bias + score *= metaKinds.includes(e.kind) ? 2 : e.created_at / now() + + return score + } + + return take( + 30_000, + sortBy(e => -scoreEvent(e), events), + ) + }, }), }).then(() => sleep(300)) @@ -251,7 +186,6 @@ let unsubSpaces: any userMembership.subscribe($membership => { - console.log("subscribe") unsubSpaces?.() unsubSpaces = listenForNotifications() })