Add migrate functions to storage

src/routes/+layout.svelte

@@ -6,7 +6,9 @@
   import {page} from "$app/stores"
   import {goto} from "$app/navigation"
   import {browser} from "$app/environment"
-  import {sleep} from "@welshman/lib"
+  import {sleep, take, sortBy, ago, now, HOUR} from "@welshman/lib"
+  import type {TrustedEvent} from "@welshman/util"
+  import {PROFILE, REACTION, ZAP_RESPONSE, FOLLOWS, RELAYS, INBOX_RELAYS, WRAP, getPubkeyTagValues, getListTags} from "@welshman/util"
   import {throttled} from "@welshman/store"
   import {
     relays,
@@ -71,14 +73,68 @@
   onMount(async () => {
     Object.assign(window, {get, ...app, ...state})
 
+    const getScoreEvent = () => {
+      const ALWAYS_KEEP = Infinity
+      const NEVER_KEEP = 0
+
+      const reactionKinds = [REACTION, ZAP_RESPONSE]
+      const metaKinds = [PROFILE, FOLLOWS, RELAYS, INBOX_RELAYS]
+      const $sessionKeys = new Set(Object.keys(app.sessions.get()))
+      const $userFollows = new Set(getPubkeyTagValues(getListTags(get(app.userFollows))))
+      const $maxWot = get(app.maxWot)
+
+      return (e: TrustedEvent) => {
+        const isFollowing = $userFollows.has(e.pubkey)
+
+        // No need to keep a record of everyone who follows the current user
+        if (e.kind === FOLLOWS && !isFollowing) return NEVER_KEEP
+
+        // Always keep stuff by or tagging a signed in user
+        if ($sessionKeys.has(e.pubkey)) return ALWAYS_KEEP
+        if (e.tags.some(t => $sessionKeys.has(t[1]))) return ALWAYS_KEEP
+
+        // Get rid of irrelevant messages, reactions, and likes
+        if (e.wrap || e.kind === 4 || e.kind === WRAP) return NEVER_KEEP
+        if (reactionKinds.includes(e.kind)) return NEVER_KEEP
+
+        // If the user follows this person, use max wot score
+        let score = isFollowing ? $maxWot : app.getUserWotScore(e.pubkey)
+
+        // Inflate the score for profiles/relays/follows to avoid redundant fetches
+        // Demote non-metadata type events, and introduce recency bias
+        score *= metaKinds.includes(e.kind) ? 2 : (e.created_at / now())
+
+        return score
+      }
+    }
+
+    const migrateFreshness = (data: {key: string; value: number}[]) => {
+      const cutoff = ago(HOUR)
+
+      return data.filter(({value}) => value < cutoff)
+    }
+
+    const migratePlaintext = (data: {key: string; value: number}[]) =>
+      data.slice(0, 10_000)
+
+    const migrateEvents = (events: TrustedEvent[]) => {
+      if (events.length < 50_000) {
+        return events
+      }
+
+      const scoreEvent = getScoreEvent()
+
+      return take(30_000, sortBy(e => -scoreEvent(e), events))
+    }
+
     if (!db) {
       ready = initStorage("flotilla", 4, {
-        events: storageAdapters.fromRepository(repository, {throttle: 300}),
+        events: storageAdapters.fromRepository(repository, {throttle: 300, migrate: migrateEvents}),
         relays: {keyPath: "url", store: throttled(1000, relays)},
         handles: {keyPath: "nip05", store: throttled(1000, handles)},
         publishStatus: storageAdapters.fromObjectStore(publishStatusData),
-        freshness: storageAdapters.fromObjectStore(freshness, {throttle: 1000}),
-        plaintext: storageAdapters.fromObjectStore(plaintext, {throttle: 1000}),
+        freshness: storageAdapters.fromObjectStore(freshness, {throttle: 1000, migrate: migrateFreshness}),
+        plaintext: storageAdapters.fromObjectStore(plaintext, {throttle: 1000, migrate: migratePlaintext}),
         tracker: storageAdapters.fromTracker(tracker, {throttle: 1000}),
       }).then(() => sleep(300))