diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 0975b9d6..a0d1493a 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -6,7 +6,9 @@ import {page} from "$app/stores" import {goto} from "$app/navigation" import {browser} from "$app/environment" - import {sleep} from "@welshman/lib" + import {sleep, take, sortBy, ago, now, HOUR} from "@welshman/lib" + import type {TrustedEvent} from "@welshman/util" + import {PROFILE, REACTION, ZAP_RESPONSE, FOLLOWS, RELAYS, INBOX_RELAYS, WRAP, getPubkeyTagValues, getListTags} from "@welshman/util" import {throttled} from "@welshman/store" import { relays, @@ -71,14 +73,68 @@ onMount(async () => { Object.assign(window, {get, ...app, ...state}) + const getScoreEvent = () => { + const ALWAYS_KEEP = Infinity + const NEVER_KEEP = 0 + + const reactionKinds = [REACTION, ZAP_RESPONSE] + const metaKinds = [PROFILE, FOLLOWS, RELAYS, INBOX_RELAYS] + const $sessionKeys = new Set(Object.keys(app.sessions.get())) + const $userFollows = new Set(getPubkeyTagValues(getListTags(get(app.userFollows)))) + const $maxWot = get(app.maxWot) + + return (e: TrustedEvent) => { + const isFollowing = $userFollows.has(e.pubkey) + + // No need to keep a record of everyone who follows the current user + if (e.kind === FOLLOWS && !isFollowing) return NEVER_KEEP + + // Always keep stuff by or tagging a signed in user + if ($sessionKeys.has(e.pubkey)) return ALWAYS_KEEP + if (e.tags.some(t => $sessionKeys.has(t[1]))) return ALWAYS_KEEP + + // Get rid of irrelevant messages, reactions, and likes + if (e.wrap || e.kind === 4 || e.kind === WRAP) return NEVER_KEEP + if (reactionKinds.includes(e.kind)) return NEVER_KEEP + + // If the user follows this person, use max wot score + let score = isFollowing ? $maxWot : app.getUserWotScore(e.pubkey) + + // Inflate the score for profiles/relays/follows to avoid redundant fetches + // Demote non-metadata type events, and introduce recency bias + score *= metaKinds.includes(e.kind) ? 2 : (e.created_at / now()) + + return score + } + } + + const migrateFreshness = (data: {key: string; value: number}[]) => { + const cutoff = ago(HOUR) + + return data.filter(({value}) => value < cutoff) + } + + const migratePlaintext = (data: {key: string; value: number}[]) => + data.slice(0, 10_000) + + const migrateEvents = (events: TrustedEvent[]) => { + if (events.length < 50_000) { + return events + } + + const scoreEvent = getScoreEvent() + + return take(30_000, sortBy(e => -scoreEvent(e), events)) + } + if (!db) { ready = initStorage("flotilla", 4, { - events: storageAdapters.fromRepository(repository, {throttle: 300}), + events: storageAdapters.fromRepository(repository, {throttle: 300, migrate: migrateEvents}), relays: {keyPath: "url", store: throttled(1000, relays)}, handles: {keyPath: "nip05", store: throttled(1000, handles)}, publishStatus: storageAdapters.fromObjectStore(publishStatusData), - freshness: storageAdapters.fromObjectStore(freshness, {throttle: 1000}), - plaintext: storageAdapters.fromObjectStore(plaintext, {throttle: 1000}), + freshness: storageAdapters.fromObjectStore(freshness, {throttle: 1000, migrate: migrateFreshness}), + plaintext: storageAdapters.fromObjectStore(plaintext, {throttle: 1000, migrate: migratePlaintext}), tracker: storageAdapters.fromTracker(tracker, {throttle: 1000}), }).then(() => sleep(300))