From 75bee027e1c07034e9ff20edb08a5ae120d6a2d0 Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Tue, 21 Oct 2025 10:29:29 -0700 Subject: [PATCH] Remove shards entirely, fix setup in layout --- src/app/util/storage.ts | 65 ++++++++---------- src/lib/storage.ts | 64 ++++++----------- src/routes/+layout.svelte | 141 ++++++++++++++++++-------------------- 3 files changed, 115 insertions(+), 155 deletions(-) diff --git a/src/app/util/storage.ts b/src/app/util/storage.ts index 3f77317d..2c34f7ca 100644 --- a/src/app/util/storage.ts +++ b/src/app/util/storage.ts @@ -1,4 +1,4 @@ -import {prop, first, call, on, groupBy, throttle, fromPairs, batch, sortBy, concat} from "@welshman/lib" +import {prop, call, on, throttle, fromPairs, batch} from "@welshman/lib" import {throttled, freshness} from "@welshman/store" import { PROFILE, @@ -96,42 +96,23 @@ const syncEvents = async () => { repository, "update", batch(3000, async (updates: RepositoryUpdate[]) => { - let added: TrustedEvent[] = [] - const removed = new Set() + const add: TrustedEvent[] = [] + const remove = new Set() for (const update of updates) { for (const event of update.added) { if (rankEvent(event) > 0) { - added.push(event) - removed.delete(event.id) + add.push(event) + remove.delete(event.id) } } for (const id of update.removed) { - removed.add(id) + remove.add(id) } } - if (removed.size > 0) { - added = added.filter(e => !removed.has(e.id)) - - const removedByShard = groupBy(id => collection.getShardId(id), removed) - const addedByShard = groupBy(collection.getShardIdFromItem, added) - const shards = new Set([...removedByShard.keys(), ...addedByShard.keys()]) - - for (const shard of shards) { - const removedInShard = removedByShard.get(shard) - const addedInShard = addedByShard.get(shard) || [] - const current = await collection.getShard(shard) - const filtered = current.filter(e => !removedInShard?.includes(e.id)) - const sorted = sortBy(e => -rankEvent(e), concat(filtered, addedInShard)) - const pruned = sorted.slice(0, 1000) - - await collection.setShard(shard, pruned) - } - } else if (added.length > 0) { - await collection.add(added) - } + await collection.update({add, remove}) }), ) } @@ -139,7 +120,10 @@ const syncEvents = async () => { type TrackerItem = [string, string[]] const syncTracker = async () => { - const collection = new Collection({table: "tracker", getId: first}) + const collection = new Collection({ + table: "tracker", + getId: (item: TrackerItem) => item[0], + }) const relaysById = new Map>() @@ -199,7 +183,10 @@ const syncZappers = async () => { type FreshnessItem = [string, number] const syncFreshness = async () => { - const collection = new Collection({table: "freshness", getId: first}) + const collection = new Collection({ + table: "freshness", + getId: (item: FreshnessItem) => item[0], + }) freshness.set(fromPairs(await collection.get())) @@ -211,7 +198,10 @@ const syncFreshness = async () => { type PlaintextItem = [string, string] const syncPlaintext = async () => { - const collection = new Collection({table: "plaintext", getId: first}) + const collection = new Collection({ + table: "plaintext", + getId: (item: PlaintextItem) => item[0], + }) plaintext.set(fromPairs(await collection.get())) @@ -239,16 +229,15 @@ const syncWrapManager = async () => { } export const syncDataStores = async () => { - const t = Date.now() const unsubscribers = await Promise.all([ - syncEvents().then(f => console.log("syncEvents", Date.now() - t) || f), - syncTracker().then(f => console.log("syncTracker", Date.now() - t) || f), - syncRelays().then(f => console.log("syncRelays", Date.now() - t) || f), - syncHandles().then(f => console.log("syncHandles", Date.now() - t) || f), - syncZappers().then(f => console.log("syncZappers", Date.now() - t) || f), - syncFreshness().then(f => console.log("syncFreshness", Date.now() - t) || f), - syncPlaintext().then(f => console.log("syncPlaintext", Date.now() - t) || f), - syncWrapManager().then(f => console.log("syncWrapManager", Date.now() - t) || f), + syncEvents(), + syncTracker(), + syncRelays(), + syncHandles(), + syncZappers(), + syncFreshness(), + syncPlaintext(), + syncWrapManager(), ]) return () => unsubscribers.forEach(call) diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 0ab50e51..d0cad00b 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -1,4 +1,4 @@ -import {hash, range, reject, flatten, identity, groupBy} from "@welshman/lib" +import {reject, identity} from "@welshman/lib" import {type StorageProvider} from "@welshman/store" import {Preferences} from "@capacitor/preferences" import {Encoding, Filesystem, Directory} from "@capacitor/filesystem" @@ -37,8 +37,6 @@ export type CollectionOptions = { } export class Collection { - #shardCount = 1000 - constructor(readonly options: CollectionOptions) {} static clearAll = async (): Promise => { @@ -57,18 +55,12 @@ export class Collection { ) } - getShardIds = () => Array.from(range(0, this.#shardCount)) + #path = () => `collection_${this.options.table}.json` - getShardId = (id: string) => String(hash(id) % this.#shardCount) - - getShardIdFromItem = (item: T) => this.getShardId(this.options.getId(item)) - - #path = (shard: string) => `collection_${this.options.table}_${shard}.json` - - getShard = async (shard: string): Promise => { + get = async (): Promise => { try { const file = await Filesystem.readFile({ - path: this.#path(shard), + path: this.#path(), directory: Directory.Data, encoding: Encoding.UTF8, }) @@ -81,48 +73,36 @@ export class Collection { } } - get = async (): Promise => flatten(await Promise.all(this.getShardIds().map(id => this.getShard(id)))) - - setShard = async (shard: string, items: T[]) => + set = (items: T[]) => Filesystem.writeFile({ - path: this.#path(shard), + path: this.#path(), directory: Directory.Data, encoding: Encoding.UTF8, data: items.map(v => JSON.stringify(v)).join("\n"), }) - set = (items: T[]) => - Promise.all( - Array.from(groupBy(this.getShardIdFromItem, items)).map(([shard, chunk]) => - this.setShard(shard, chunk), - ), - ) - - addToShard = (shard: string, items: T[]) => + add = (items: T[]) => Filesystem.appendFile({ - path: this.#path(shard), + path: this.#path(), directory: Directory.Data, encoding: Encoding.UTF8, data: "\n" + items.map(v => JSON.stringify(v)).join("\n"), }) - add = (items: T[]) => - Promise.all( - Array.from(groupBy(this.getShardIdFromItem, items)).map(([shard, chunk]) => - this.addToShard(shard, chunk), - ), - ) + remove = async (ids: Set) => + this.set(reject(item => ids.has(this.options.getId(item)), await this.get())) - removeFromShard = async (shard: string, ids: Set) => - this.setShard( - shard, - reject(item => ids.has(this.options.getId(item)), await this.getShard(shard)), - ) + update = async ({add, remove}: {add?: T[]; remove?: Set}) => { + if (remove && remove.size > 0) { + const items = reject(item => remove.has(this.options.getId(item)), await this.get()) - remove = (ids: Iterable) => - Promise.all( - Array.from(groupBy(this.getShardId, ids)).map(([shard, chunk]) => - this.removeFromShard(shard, new Set(chunk)), - ), - ) + if (add) { + items.push(...add) + } + + await this.set(items) + } else if (add && add.length > 0) { + await this.add(add) + } + } } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 02e51edf..4f596fb1 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -3,6 +3,7 @@ import "@capacitor-community/safe-area" import {throttle} from "throttle-debounce" import * as nip19 from "nostr-tools/nip19" + import type {Unsubscriber} from "svelte/store" import {get} from "svelte/store" import {App, type URLOpenListenerEvent} from "@capacitor/app" import {dev} from "$app/environment" @@ -47,6 +48,8 @@ const {children} = $props() + const policies = [authPolicy, trustPolicy, mostlyRestrictedPolicy] + // Add stuff to window for convenience Object.assign(window, { get, @@ -91,63 +94,9 @@ } }) - // Listen to navigation changes - const unsubscribeHistory = setupHistory() + const unsubscribe = call(async () => { + const unsubscribers: Unsubscriber[] = [] - // Report usage on navigation change - const unsubscribeAnalytics = setupAnalytics() - - // Bug tracking - const unsubscribeTracking = setupTracking() - - // Load user data, listen for messages, etc - const unsubscribeApplicationData = syncApplicationData() - - // Whenever we see a new pubkey, load their outbox event - const unsubscribeRepository = on(repository, "update", ({added}) => { - for (const event of added) { - loadRelaySelections(event.pubkey) - } - }) - - // Subscribe to badge count for changes - const unsubscribeBadgeCount = notifications.badgeCount.subscribe( - notifications.handleBadgeCountChanges, - ) - - // Listen for signer errors, report to user via toast - const unsubscribeSignerLog = signerLog.subscribe( - throttle(10_000, $log => { - const recent = $log.slice(-10) - const success = recent.filter(spec({status: SignerLogEntryStatus.Success})) - const failure = recent.filter(spec({status: SignerLogEntryStatus.Failure})) - - if (!$toast && failure.length > 5 && success.length === 0) { - pushToast({ - theme: "error", - timeout: 60_000, - message: "Your signer appears to be unresponsive.", - action: { - message: "Details", - onclick: () => goto("/settings/profile"), - }, - }) - } - }), - ) - - // Sync theme - const unsubscribeTheme = theme.subscribe($theme => { - document.body.setAttribute("data-theme", $theme) - }) - - // Sync font size - const unsubscribeSettings = userSettingsValues.subscribe($userSettingsValues => { - // @ts-ignore - document.documentElement.style["font-size"] = `${$userSettingsValues.font_size}rem` - }) - - const unsubscribeStorage = call(async () => { // Sync stuff to localstorage await Promise.all([ sync({ @@ -167,29 +116,71 @@ }), ]) - // Sync stuff to indexeddb - return await storage.syncDataStores() + // Wait until data storage is initialized before syncing other stuff + unsubscribers.push(await storage.syncDataStores()) + + // Add our extra policies now that we're set up + defaultSocketPolicies.push(...policies) + + // Remove policies when we're done + unsubscribers.push(() => defaultSocketPolicies.splice(-policies.length)) + + // History, navigation, bug tracking, application data + unsubscribers.push(setupHistory(), setupAnalytics(), setupTracking(), syncApplicationData()) + + // Whenever we see a new pubkey, load their outbox event + unsubscribers.push( + on(repository, "update", ({added}) => { + for (const event of added) { + loadRelaySelections(event.pubkey) + } + }), + ) + + // Subscribe to badge count for changes + unsubscribers.push(notifications.badgeCount.subscribe(notifications.handleBadgeCountChanges)) + + // Listen for signer errors, report to user via toast + unsubscribers.push( + signerLog.subscribe( + throttle(10_000, $log => { + const recent = $log.slice(-10) + const success = recent.filter(spec({status: SignerLogEntryStatus.Success})) + const failure = recent.filter(spec({status: SignerLogEntryStatus.Failure})) + + if (!$toast && failure.length > 5 && success.length === 0) { + pushToast({ + theme: "error", + timeout: 60_000, + message: "Your signer appears to be unresponsive.", + action: { + message: "Details", + onclick: () => goto("/settings/profile"), + }, + }) + } + }), + ), + ) + + // Sync theme and font size + unsubscribers.push( + theme.subscribe($theme => { + document.body.setAttribute("data-theme", $theme) + }), + userSettingsValues.subscribe($userSettingsValues => { + // @ts-ignore + document.documentElement.style["font-size"] = `${$userSettingsValues.font_size}rem` + }), + ) + + return () => unsubscribers.forEach(call) }) - // Default socket policies - const additionalPolicies = [authPolicy, trustPolicy, mostlyRestrictedPolicy] - - defaultSocketPolicies.push(...additionalPolicies) - // Cleanup on hot reload import.meta.hot?.dispose(() => { App.removeAllListeners() - unsubscribeHistory() - unsubscribeAnalytics() - unsubscribeTracking() - unsubscribeApplicationData() - unsubscribeRepository() - unsubscribeBadgeCount() - unsubscribeSignerLog() - unsubscribeTheme() - unsubscribeSettings() - unsubscribeStorage.then(call) - defaultSocketPolicies.splice(-additionalPolicies.length) + unsubscribe.then(call) }) @@ -199,7 +190,7 @@ {/if} -{#await unsubscribeStorage} +{#await unsubscribe} {:then}