From a2d519766de320e0fb84589924f2fb6997fcf41a Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Tue, 30 Sep 2025 15:05:44 -0700 Subject: [PATCH] Bump version, drop storage stuff --- docs/app/storage.md | 32 --- package.json | 4 +- packages/app/package.json | 3 +- packages/app/src/index.ts | 2 - packages/app/src/storage.ts | 124 --------- packages/app/src/storageAdapters.ts | 266 -------------------- packages/content/package.json | 2 +- packages/editor/package.json | 2 +- packages/feeds/package.json | 2 +- packages/lib/package.json | 2 +- packages/net/package.json | 2 +- packages/relay/__tests__/repository.test.ts | 16 +- packages/relay/package.json | 2 +- packages/router/package.json | 2 +- packages/signer/package.json | 2 +- packages/store/package.json | 2 +- packages/util/package.json | 2 +- packages/util/src/Events.ts | 12 +- scripts/bump.sh | 5 - vitest.setup.ts | 1 - 20 files changed, 31 insertions(+), 454 deletions(-) delete mode 100644 docs/app/storage.md delete mode 100644 packages/app/src/storage.ts delete mode 100644 packages/app/src/storageAdapters.ts delete mode 100755 scripts/bump.sh delete mode 100644 vitest.setup.ts diff --git a/docs/app/storage.md b/docs/app/storage.md deleted file mode 100644 index b478c63..0000000 --- a/docs/app/storage.md +++ /dev/null @@ -1,32 +0,0 @@ -# Storage - -The storage system provides IndexedDB persistence for stores and repositories. - -Initialize this early in your application lifecycle to ensure data consistency. - -```typescript -import {initStorage, defaultStorageAdapters} from '@welshman/app' - -// Use default storage adapters, which track important metadata events, -// relays, handles, zappers, etc. -await initStorage("my-db", 1, { - ...defaultStorageAdapters, - custom: { - keyPath: "key", - init: async () => console.log(await getAll("custom")), - sync: () => { - // Set up a listener for changes, using bulkPut to save records. - // Return an unsubscribe function for cleanup - }, - }, -}) -``` - -The storage system: - -- Persists data across page reloads -- Throttles writes for performance -- Syncs bidirectionally -- Supports custom adapters - -Initialize storage before making any subscriptions or loading data to ensure proper caching behavior. diff --git a/package.json b/package.json index 89675ac..e683746 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@welshman", "private": true, - "version": "0.4.7", + "version": "0.5.0", "workspaces": [ "packages/*" ], @@ -12,6 +12,7 @@ "lint": "eslint .", "test": "vitest", "docs": "typedoc && vitepress build docs", + "bump": "pnpm exec ts-node scripts/bump.ts", "prepare": "husky" }, "devDependencies": { @@ -19,7 +20,6 @@ "eslint": "~9.23.0", "eslint-config-prettier": "^10.1.1", "eslint-plugin-prettier": "~5.2.5", - "fake-indexeddb": "^6.0.0", "globals": "~16.0.0", "happy-dom": "^17.4.4", "husky": "^9.1.7", diff --git a/packages/app/package.json b/packages/app/package.json index 9638a48..e9b0989 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/app", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of svelte stores for use in building nostr client applications.", @@ -30,7 +30,6 @@ "@welshman/store": "workspace:*", "@welshman/util": "workspace:*", "fuse.js": "^7.0.0", - "idb": "^8.0.0", "svelte": "^4.2.18", "throttle-debounce": "^5.0.2" }, diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts index 8b3e873..845a55d 100644 --- a/packages/app/src/index.ts +++ b/packages/app/src/index.ts @@ -14,8 +14,6 @@ export * from "./relaySelections.js" export * from "./inboxRelaySelections.js" export * from "./search.js" export * from "./session.js" -export * from "./storage.js" -export * from "./storageAdapters.js" export * from "./sync.js" export * from "./tags.js" export * from "./thunk.js" diff --git a/packages/app/src/storage.ts b/packages/app/src/storage.ts deleted file mode 100644 index ba272de..0000000 --- a/packages/app/src/storage.ts +++ /dev/null @@ -1,124 +0,0 @@ -import {openDB, deleteDB} from "idb" -import {IDBPDatabase} from "idb" -import {writable} from "svelte/store" -import {Unsubscriber} from "svelte/store" -import {call, defer} from "@welshman/lib" -import {withGetter} from "@welshman/store" - -export type StorageAdapterOptions = { - throttle?: number - migrate?: (items: any[]) => any[] -} - -export type StorageAdapter = { - keyPath: string - init: () => Promise - sync: () => Unsubscriber -} - -export let db: IDBPDatabase | undefined - -export const ready = defer() - -export const dead = withGetter(writable(false)) - -export const unsubscribers: Unsubscriber[] = [] - -export const getAll = async (name: string) => { - await ready - - const tx = db!.transaction(name, "readwrite") - const store = tx.objectStore(name) - const result = await store.getAll() - - await tx.done - - return result -} - -export const bulkPut = async (name: string, data: any[]) => { - await ready - - const tx = db!.transaction(name, "readwrite") - const store = tx.objectStore(name) - - await Promise.all( - data.map(item => { - try { - store.put(item) - } catch (e) { - console.error(e, item) - } - }), - ) - - await tx.done -} - -export const bulkDelete = async (name: string, ids: string[]) => { - await ready - - const tx = db!.transaction(name, "readwrite") - const store = tx.objectStore(name) - - await Promise.all(ids.map(id => store.delete(id))) - await tx.done -} - -export const initStorage = async ( - name: string, - version: number, - adapters: Record, -) => { - if (!window.indexedDB) return - - window.addEventListener("beforeunload", () => closeStorage()) - - if (db) { - throw new Error("Db initialized multiple times") - } - - db = await openDB(name, version, { - upgrade(db: IDBPDatabase) { - const names = Object.keys(adapters) - - for (const name of db.objectStoreNames) { - if (!names.includes(name)) { - db.deleteObjectStore(name) - } - } - - for (const [name, {keyPath}] of Object.entries(adapters)) { - try { - db.createObjectStore(name, {keyPath}) - } catch (e) { - console.warn(e) - } - } - }, - }) - - ready.resolve() - - await Promise.all( - Object.values(adapters).map(async adapter => { - await adapter.init() - - unsubscribers.push(adapter.sync()) - }), - ) -} - -export const closeStorage = async () => { - dead.set(true) - unsubscribers.forEach(call) - await db?.close() -} - -export const clearStorage = async () => { - if (db) { - await closeStorage() - await deleteDB(db.name) - db = undefined // force initStorage to run again in tests - } -} diff --git a/packages/app/src/storageAdapters.ts b/packages/app/src/storageAdapters.ts deleted file mode 100644 index a15b4a8..0000000 --- a/packages/app/src/storageAdapters.ts +++ /dev/null @@ -1,266 +0,0 @@ -import {derived} from "svelte/store" -import {batch, sortBy, call, fromPairs} from "@welshman/lib" -import { - PROFILE, - FOLLOWS, - MUTES, - RELAYS, - INBOX_RELAYS, - getPubkeyTagValues, - getListTags, - TrustedEvent, -} from "@welshman/util" -import {throttled, withGetter} from "@welshman/store" -import {Tracker} from "@welshman/net" -import {freshness} from "@welshman/store" -import {Repository, RepositoryUpdate} from "@welshman/relay" -import {getAll, bulkPut, bulkDelete} from "./storage.js" -import {relays} from "./relays.js" -import {handles, onHandle} from "./handles.js" -import {zappers, onZapper} from "./zappers.js" -import {plaintext} from "./plaintext.js" -import {repository, tracker} from "./core.js" -import {sessions} from "./session.js" -import {userFollows} from "./user.js" - -export type RelaysStorageAdapterOptions = { - name: string -} - -export class RelaysStorageAdapter { - keyPath = "url" - - constructor(readonly options: RelaysStorageAdapterOptions) {} - - async init() { - relays.set(await getAll(this.options.name)) - } - - sync() { - return throttled(3000, relays).subscribe($relays => bulkPut(this.options.name, $relays)) - } -} - -export type HandlesStorageAdapterOptions = { - name: string -} - -export class HandlesStorageAdapter { - keyPath = "nip05" - - constructor(readonly options: HandlesStorageAdapterOptions) {} - - async init() { - handles.set(await getAll(this.options.name)) - } - - sync() { - return onHandle(batch(300, $handles => bulkPut(this.options.name, $handles))) - } -} - -export type ZappersStorageAdapterOptions = { - name: string -} - -export class ZappersStorageAdapter { - keyPath = "lnurl" - - constructor(readonly options: ZappersStorageAdapterOptions) {} - - async init() { - zappers.set(await getAll(this.options.name)) - } - - sync() { - return onZapper(batch(300, $zappers => bulkPut(this.options.name, $zappers))) - } -} - -export type FreshnessStorageAdapterOptions = { - name: string -} - -export class FreshnessStorageAdapter { - keyPath = "key" - - constructor(readonly options: FreshnessStorageAdapterOptions) {} - - async init() { - const items = await getAll(this.options.name) - - freshness.set(fromPairs(items.map(item => [item.key, item.value]))) - } - - sync() { - const interval = setInterval(() => { - bulkPut( - this.options.name, - Object.entries(freshness.get()).map(([key, value]) => ({key, value})), - ) - }, 10_000) - - return () => clearInterval(interval) - } -} - -export type PlaintextStorageAdapterOptions = { - name: string -} - -export class PlaintextStorageAdapter { - keyPath = "key" - - constructor(readonly options: PlaintextStorageAdapterOptions) {} - - async init() { - const items = await getAll(this.options.name) - - plaintext.set(fromPairs(items.map(item => [item.key, item.value]))) - } - - sync() { - const interval = setInterval(() => { - bulkPut( - this.options.name, - Object.entries(plaintext.get()).map(([key, value]) => ({key, value})), - ) - }, 10_000) - - return () => clearInterval(interval) - } -} - -export type TrackerStorageAdapterOptions = { - name: string - tracker: Tracker -} - -export class TrackerStorageAdapter { - keyPath = "id" - - constructor(readonly options: TrackerStorageAdapterOptions) {} - - async init() { - const relaysById = new Map>() - - for (const {id, relays} of await getAll(this.options.name)) { - relaysById.set(id, new Set(relays)) - } - - this.options.tracker.load(relaysById) - } - - sync() { - const updateOne = (id: string, relay: string) => - bulkPut(this.options.name, [{id, relays: Array.from(this.options.tracker.getRelays(id))}]) - - const updateAll = () => - bulkPut( - this.options.name, - Array.from(this.options.tracker.relaysById.entries()).map(([id, relays]) => ({ - id, - relays: Array.from(relays), - })), - ) - - this.options.tracker.on("add", updateOne) - this.options.tracker.on("remove", updateOne) - this.options.tracker.on("load", updateAll) - this.options.tracker.on("clear", updateAll) - - return () => { - this.options.tracker.off("add", updateOne) - this.options.tracker.off("remove", updateOne) - this.options.tracker.off("load", updateAll) - this.options.tracker.off("clear", updateAll) - } - } -} - -export type EventsStorageAdapterOptions = { - name: string - limit: number - repository: Repository - rankEvent: (event: TrustedEvent) => number -} - -export class EventsStorageAdapter { - keyPath = "id" - eventCount = 0 - - constructor(readonly options: EventsStorageAdapterOptions) {} - - async init() { - const events = await getAll(this.options.name) - - this.eventCount = events.length - - this.options.repository.load(events) - } - - sync() { - const {name, limit, rankEvent} = this.options - - const onUpdate = async ({added, removed}: RepositoryUpdate) => { - // Only add events we want to keep - const keep = added.filter(e => rankEvent(e) > 0) - - // Add new events - if (keep.length > 0) { - await bulkPut(name, keep) - } - - // If we're well above our retention limit, drop lowest-ranked events - if (this.eventCount > limit * 1.5) { - removed = new Set(removed) - - for (const event of sortBy(e => -rankEvent(e), await getAll(name)).slice(limit)) { - removed.add(event.id) - } - } - - if (removed.size > 0) { - await bulkDelete(name, Array.from(removed)) - } - - // Keep track of our total number of events. This isn't strictly accurate, but it's close enough - this.eventCount = this.eventCount + keep.length - removed.size - } - - this.options.repository.on("update", onUpdate) - - return () => this.options.repository.off("update", onUpdate) - } -} - -export const defaultStorageAdapters = { - relays: new RelaysStorageAdapter({name: "relays"}), - handles: new HandlesStorageAdapter({name: "handles"}), - zappers: new ZappersStorageAdapter({name: "zappers"}), - freshness: new FreshnessStorageAdapter({name: "freshness"}), - plaintext: new PlaintextStorageAdapter({name: "plaintext"}), - tracker: new TrackerStorageAdapter({name: "tracker", tracker}), - events: new EventsStorageAdapter( - call(() => { - const userFollowPubkeys = withGetter( - derived(userFollows, l => new Set(getPubkeyTagValues(getListTags(l)))), - ) - - return { - repository, - name: "events", - limit: 10_000, - rankEvent: (e: TrustedEvent) => { - const $sessions = sessions.get() - const metaKinds = [PROFILE, FOLLOWS, MUTES, RELAYS, INBOX_RELAYS] - - if ($sessions[e.pubkey] || e.tags.some(t => $sessions[t[1]])) return 1 - if (metaKinds.includes(e.kind) && userFollowPubkeys.get()?.has(e.pubkey)) return 1 - - return 0 - }, - } - }), - ), -} diff --git a/packages/content/package.json b/packages/content/package.json index a2b3e47..2bd8d4e 100644 --- a/packages/content/package.json +++ b/packages/content/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/content", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of utilities for parsing nostr note content.", diff --git a/packages/editor/package.json b/packages/editor/package.json index bc4e5b6..b8ce578 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/editor", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A batteries-included nostr editor.", diff --git a/packages/feeds/package.json b/packages/feeds/package.json index 3312919..fde8e3a 100644 --- a/packages/feeds/package.json +++ b/packages/feeds/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/feeds", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "Utilities for building dynamic nostr feeds.", diff --git a/packages/lib/package.json b/packages/lib/package.json index e95a5a9..35b4c9c 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/lib", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of utilities.", diff --git a/packages/net/package.json b/packages/net/package.json index 0fca113..28b6a35 100644 --- a/packages/net/package.json +++ b/packages/net/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/net", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "Utilities for connecting with nostr relays.", diff --git a/packages/relay/__tests__/repository.test.ts b/packages/relay/__tests__/repository.test.ts index 0254a6c..9f545d3 100644 --- a/packages/relay/__tests__/repository.test.ts +++ b/packages/relay/__tests__/repository.test.ts @@ -1,12 +1,18 @@ import {describe, it, vi, expect, beforeEach} from "vitest" -import {now, randomId} from "@welshman/lib" +import {now, choice, range} from "@welshman/lib" import {getAddress, makeEvent, TrustedEvent, DELETE, MUTES} from "@welshman/util" import {Repository} from "../src/repository" +const randomHex = () => + Array.from(range(0, 64)) + .map(() => choice(Array.from("0123456789abcdef"))) + .join("") + const createEvent = (kind: number, extra = {}) => ({ ...makeEvent(kind), - pubkey: randomId(), - id: randomId(), + pubkey: randomHex(), + id: randomHex(), + sig: "fake", ...extra, }) @@ -55,7 +61,7 @@ describe("Repository", () => { }) it("should handle replaceable events", () => { - const pubkey = randomId() + const pubkey = randomHex() const event1 = createEvent(MUTES, {created_at: now() - 100, pubkey}) const event2 = createEvent(MUTES, {created_at: now(), pubkey}) @@ -177,7 +183,7 @@ describe("Repository", () => { }) it("should query by tags", () => { - const pubkey = randomId() + const pubkey = randomHex() const event = createEvent(1, {tags: [["p", pubkey]]}) repo.publish(event) diff --git a/packages/relay/package.json b/packages/relay/package.json index 220dd5a..249a9d6 100644 --- a/packages/relay/package.json +++ b/packages/relay/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/relay", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "An in-memory nostr relay implementation.", diff --git a/packages/router/package.json b/packages/router/package.json index 562dc66..070941c 100644 --- a/packages/router/package.json +++ b/packages/router/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/router", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of utilities for nostr relay selection.", diff --git a/packages/signer/package.json b/packages/signer/package.json index 37ee25e..5ce497d 100644 --- a/packages/signer/package.json +++ b/packages/signer/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/signer", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A nostr signer implemenation supporting several login methods.", diff --git a/packages/store/package.json b/packages/store/package.json index 2a272e7..e5bd01c 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/store", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of utilities based on svelte/store for use with welshman", diff --git a/packages/util/package.json b/packages/util/package.json index 87a180c..e9a66d9 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,6 +1,6 @@ { "name": "@welshman/util", - "version": "0.4.7", + "version": "0.5.0", "author": "hodlbod", "license": "MIT", "description": "A collection of nostr-related utilities.", diff --git a/packages/util/src/Events.ts b/packages/util/src/Events.ts index 195e823..3731f93 100644 --- a/packages/util/src/Events.ts +++ b/packages/util/src/Events.ts @@ -89,6 +89,7 @@ export const verifyEvent = (() => { // Type guards export const isEventTemplate = (e: EventTemplate): e is EventTemplate => { + if (!e) return false if (e.kind % 1 !== 0) return false if (typeof e.content !== "string") return false @@ -96,18 +97,19 @@ export const isEventTemplate = (e: EventTemplate): e is EventTemplate => { } export const isStampedEvent = (e: StampedEvent): e is StampedEvent => - Boolean(isEventTemplate(e) && e.created_at >= 0) + Boolean(isEventTemplate(e) && e.created_at >= 0 && e.created_at % 1 === 0) export const isOwnedEvent = (e: OwnedEvent): e is OwnedEvent => - Boolean(isStampedEvent(e) && e.pubkey) + Boolean(isStampedEvent(e) && typeof e.pubkey === "string" && e.pubkey.length === 64) -export const isHashedEvent = (e: HashedEvent): e is HashedEvent => Boolean(isOwnedEvent(e) && e.id) +export const isHashedEvent = (e: HashedEvent): e is HashedEvent => + Boolean(isOwnedEvent(e) && typeof e.id === "string" && e.id.length === 64) export const isSignedEvent = (e: TrustedEvent): e is SignedEvent => - Boolean(isHashedEvent(e) && e.sig) + Boolean(isHashedEvent(e) && typeof e.sig === "string" && e.sig.length > 0) export const isUnwrappedEvent = (e: TrustedEvent): e is UnwrappedEvent => - Boolean(isHashedEvent(e) && e.wrap) + Boolean(isHashedEvent(e) && e.wrap && isSignedEvent(e.wrap)) export const isTrustedEvent = (e: TrustedEvent): e is TrustedEvent => isSignedEvent(e) || isUnwrappedEvent(e) diff --git a/scripts/bump.sh b/scripts/bump.sh deleted file mode 100755 index 28dd750..0000000 --- a/scripts/bump.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -# Run the TypeScript script using ts-node -pnpm exec ts-node scripts/bump.ts diff --git a/vitest.setup.ts b/vitest.setup.ts deleted file mode 100644 index b054ed9..0000000 --- a/vitest.setup.ts +++ /dev/null @@ -1 +0,0 @@ -import "fake-indexeddb/auto"