diff --git a/src/app/components/ChannelItem.svelte b/src/app/components/ChannelItem.svelte index 191ee178..1eefdc60 100644 --- a/src/app/components/ChannelItem.svelte +++ b/src/app/components/ChannelItem.svelte @@ -59,7 +59,7 @@ const profile = deriveProfile(event.pubkey, [url]) const profileDisplay = deriveProfileDisplay(event.pubkey, [url]) const thunk = mergeThunks($thunks.filter(t => t.event.id === event.id)) - const [_, colorValue] = colors[parseInt(hash(event.pubkey)) % colors.length] + const [_, colorValue] = colors[hash(event.pubkey) % colors.length] const comments = deriveEventsForUrl(url, [{kinds: [COMMENT], "#e": [event.id]}]) const reply = () => replyTo!(event) diff --git a/src/app/components/ChatMessage.svelte b/src/app/components/ChatMessage.svelte index 1592837b..48998fa8 100644 --- a/src/app/components/ChatMessage.svelte +++ b/src/app/components/ChatMessage.svelte @@ -40,7 +40,7 @@ const profile = deriveProfile(event.pubkey) const profileDisplay = deriveProfileDisplay(event.pubkey) const thunk = mergeThunks($thunks.filter(t => t.event.id === event.id)) - const [_, colorValue] = colors[parseInt(hash(event.pubkey)) % colors.length] + const [_, colorValue] = colors[hash(event.pubkey) % colors.length] const reply = () => replyTo(event) diff --git a/src/app/core/state.ts b/src/app/core/state.ts index f7af640d..cb490bcb 100644 --- a/src/app/core/state.ts +++ b/src/app/core/state.ts @@ -10,6 +10,7 @@ import { uniqBy, sortBy, sort, + prop, uniq, nth, pushToMapKey, @@ -524,7 +525,7 @@ export const chats = derived( c => -c.last_activity, Array.from(messagesByChatId.entries()).map(([id, events]): Chat => { const pubkeys = remove($pubkey!, splitChatId(id)) - const messages = sortBy(e => -e.created_at, events) + const messages = sortBy(e => -e.created_at, uniqBy(prop("id"), events)) const last_activity = messages[0].created_at const search_text = pubkeys.length === 0 diff --git a/src/app/util/storage.ts b/src/app/util/storage.ts index 7f081858..3f77317d 100644 --- a/src/app/util/storage.ts +++ b/src/app/util/storage.ts @@ -1,16 +1,4 @@ -import { - always, - call, - on, - hash, - last, - groupBy, - throttle, - fromPairs, - batch, - sortBy, - concat, -} from "@welshman/lib" +import {prop, first, call, on, groupBy, throttle, fromPairs, batch, sortBy, concat} from "@welshman/lib" import {throttled, freshness} from "@welshman/store" import { PROFILE, @@ -50,11 +38,7 @@ import { import {Collection} from "@lib/storage" const syncEvents = async () => { - const collection = new Collection({ - table: "events", - shards: Array.from("0123456789abcdef"), - getShard: (event: TrustedEvent) => last(event.id), - }) + const collection = new Collection({table: "events", getId: prop("id")}) const initialEvents = await collection.get() @@ -131,8 +115,8 @@ const syncEvents = async () => { if (removed.size > 0) { added = added.filter(e => !removed.has(e.id)) - const removedByShard = groupBy(id => last(id), removed) - const addedByShard = groupBy(e => last(e.id), added) + const removedByShard = groupBy(id => collection.getShardId(id), removed) + const addedByShard = groupBy(collection.getShardIdFromItem, added) const shards = new Set([...removedByShard.keys(), ...addedByShard.keys()]) for (const shard of shards) { @@ -141,7 +125,7 @@ const syncEvents = async () => { const current = await collection.getShard(shard) const filtered = current.filter(e => !removedInShard?.includes(e.id)) const sorted = sortBy(e => -rankEvent(e), concat(filtered, addedInShard)) - const pruned = sorted.slice(0, 10_000) + const pruned = sorted.slice(0, 1000) await collection.setShard(shard, pruned) } @@ -155,11 +139,7 @@ const syncEvents = async () => { type TrackerItem = [string, string[]] const syncTracker = async () => { - const collection = new Collection({ - table: "tracker", - shards: Array.from("0123456789abcdef"), - getShard: (item: TrackerItem) => last(item[0]), - }) + const collection = new Collection({table: "tracker", getId: first}) const relaysById = new Map>() @@ -193,11 +173,7 @@ const syncTracker = async () => { } const syncRelays = async () => { - const collection = new Collection({ - table: "relays", - shards: Array.from("0123456789"), - getShard: (item: Relay) => last(hash(item.url)), - }) + const collection = new Collection({table: "relays", getId: prop("url")}) relays.set(await collection.get()) @@ -205,11 +181,7 @@ const syncRelays = async () => { } const syncHandles = async () => { - const collection = new Collection({ - table: "handles", - shards: Array.from("0123456789"), - getShard: (item: Handle) => last(hash(item.nip05)), - }) + const collection = new Collection({table: "handles", getId: prop("nip05")}) handles.set(await collection.get()) @@ -217,11 +189,7 @@ const syncHandles = async () => { } const syncZappers = async () => { - const collection = new Collection({ - table: "zappers", - shards: Array.from("0123456789"), - getShard: (item: Zapper) => last(hash(item.lnurl)), - }) + const collection = new Collection({table: "zappers", getId: prop("lnurl")}) zappers.set(await collection.get()) @@ -231,11 +199,7 @@ const syncZappers = async () => { type FreshnessItem = [string, number] const syncFreshness = async () => { - const collection = new Collection({ - table: "freshness", - shards: ["0"], - getShard: always("0"), - }) + const collection = new Collection({table: "freshness", getId: first}) freshness.set(fromPairs(await collection.get())) @@ -247,11 +211,7 @@ const syncFreshness = async () => { type PlaintextItem = [string, string] const syncPlaintext = async () => { - const collection = new Collection({ - table: "plaintext", - shards: ["0"], - getShard: always("0"), - }) + const collection = new Collection({table: "plaintext", getId: first}) plaintext.set(fromPairs(await collection.get())) @@ -261,11 +221,7 @@ const syncPlaintext = async () => { } const syncWrapManager = async () => { - const collection = new Collection({ - table: "wraps", - shards: Array.from("0123456789abcdef"), - getShard: (item: WrapItem) => last(hash(item.id)), - }) + const collection = new Collection({table: "wraps", getId: prop("id")}) wrapManager.load(await collection.get()) @@ -283,15 +239,16 @@ const syncWrapManager = async () => { } export const syncDataStores = async () => { + const t = Date.now() const unsubscribers = await Promise.all([ - syncEvents(), - syncTracker(), - syncRelays(), - syncHandles(), - syncZappers(), - syncFreshness(), - syncPlaintext(), - syncWrapManager(), + syncEvents().then(f => console.log("syncEvents", Date.now() - t) || f), + syncTracker().then(f => console.log("syncTracker", Date.now() - t) || f), + syncRelays().then(f => console.log("syncRelays", Date.now() - t) || f), + syncHandles().then(f => console.log("syncHandles", Date.now() - t) || f), + syncZappers().then(f => console.log("syncZappers", Date.now() - t) || f), + syncFreshness().then(f => console.log("syncFreshness", Date.now() - t) || f), + syncPlaintext().then(f => console.log("syncPlaintext", Date.now() - t) || f), + syncWrapManager().then(f => console.log("syncWrapManager", Date.now() - t) || f), ]) return () => unsubscribers.forEach(call) diff --git a/src/lib/storage.ts b/src/lib/storage.ts index c5e7455a..0ab50e51 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -1,4 +1,4 @@ -import {flatten, identity, groupBy} from "@welshman/lib" +import {hash, range, reject, flatten, identity, groupBy} from "@welshman/lib" import {type StorageProvider} from "@welshman/store" import {Preferences} from "@capacitor/preferences" import {Encoding, Filesystem, Directory} from "@capacitor/filesystem" @@ -33,12 +33,11 @@ export const preferencesStorageProvider = new PreferencesStorageProvider() export type CollectionOptions = { table: string - shards: string[] - getShard: (item: T) => string + getId: (item: T) => string } export class Collection { - #promises = new Map>() + #shardCount = 1000 constructor(readonly options: CollectionOptions) {} @@ -58,67 +57,72 @@ export class Collection { ) } - #then = (shard: string, f: () => Promise) => { - const oldPromise = this.#promises.get(shard) || Promise.resolve() - const newPromise = oldPromise.then(f) + getShardIds = () => Array.from(range(0, this.#shardCount)) - this.#promises.set(shard, newPromise) + getShardId = (id: string) => String(hash(id) % this.#shardCount) - return newPromise - } + getShardIdFromItem = (item: T) => this.getShardId(this.options.getId(item)) #path = (shard: string) => `collection_${this.options.table}_${shard}.json` - getShard = (shard: string): Promise => - this.#then(shard, async () => { - try { - const file = await Filesystem.readFile({ - path: this.#path(shard), - directory: Directory.Data, - encoding: Encoding.UTF8, - }) - - // Speed things up by parsing only once - return JSON.parse("[" + file.data.toString().split("\n").filter(identity).join(",") + "]") - } catch (err) { - // file doesn't exist, or isn't valid json - return [] - } - }) - - get = async (): Promise => flatten(await Promise.all(this.options.shards.map(this.getShard))) - - setShard = (shard: string, items: T[]) => - this.#then(shard, async () => { - await Filesystem.writeFile({ + getShard = async (shard: string): Promise => { + try { + const file = await Filesystem.readFile({ path: this.#path(shard), directory: Directory.Data, encoding: Encoding.UTF8, - data: items.map(v => JSON.stringify(v)).join("\n"), }) + + // Speed things up by parsing only once + return JSON.parse("[" + file.data.toString().split("\n").filter(identity).join(",") + "]") + } catch (err) { + // file doesn't exist, or isn't valid json + return [] + } + } + + get = async (): Promise => flatten(await Promise.all(this.getShardIds().map(id => this.getShard(id)))) + + setShard = async (shard: string, items: T[]) => + Filesystem.writeFile({ + path: this.#path(shard), + directory: Directory.Data, + encoding: Encoding.UTF8, + data: items.map(v => JSON.stringify(v)).join("\n"), }) set = (items: T[]) => Promise.all( - Array.from(groupBy(this.options.getShard, items)).map(([shard, chunk]) => + Array.from(groupBy(this.getShardIdFromItem, items)).map(([shard, chunk]) => this.setShard(shard, chunk), ), ) addToShard = (shard: string, items: T[]) => - this.#then(shard, async () => { - await Filesystem.appendFile({ - path: this.#path(shard), - directory: Directory.Data, - encoding: Encoding.UTF8, - data: "\n" + items.map(v => JSON.stringify(v)).join("\n"), - }) + Filesystem.appendFile({ + path: this.#path(shard), + directory: Directory.Data, + encoding: Encoding.UTF8, + data: "\n" + items.map(v => JSON.stringify(v)).join("\n"), }) add = (items: T[]) => Promise.all( - Array.from(groupBy(this.options.getShard, items)).map(([shard, chunk]) => + Array.from(groupBy(this.getShardIdFromItem, items)).map(([shard, chunk]) => this.addToShard(shard, chunk), ), ) + + removeFromShard = async (shard: string, ids: Set) => + this.setShard( + shard, + reject(item => ids.has(this.options.getId(item)), await this.getShard(shard)), + ) + + remove = (ids: Iterable) => + Promise.all( + Array.from(groupBy(this.getShardId, ids)).map(([shard, chunk]) => + this.removeFromShard(shard, new Set(chunk)), + ), + ) } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index d86a119e..02e51edf 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -147,9 +147,7 @@ document.documentElement.style["font-size"] = `${$userSettingsValues.font_size}rem` }) - let unsubscribeStorage: () => void - - const ready = call(async () => { + const unsubscribeStorage = call(async () => { // Sync stuff to localstorage await Promise.all([ sync({ @@ -170,7 +168,7 @@ ]) // Sync stuff to indexeddb - unsubscribeStorage = await storage.syncDataStores() + return await storage.syncDataStores() }) // Default socket policies @@ -190,7 +188,7 @@ unsubscribeSignerLog() unsubscribeTheme() unsubscribeSettings() - unsubscribeStorage?.() + unsubscribeStorage.then(call) defaultSocketPolicies.splice(-additionalPolicies.length) }) @@ -201,8 +199,8 @@ {/if} -{#await ready} -
+{#await unsubscribeStorage} + {:then}