Add filestorage adapters

src/app/core/commands.ts

@@ -111,7 +111,7 @@ import {
 } from "@app/core/state"
 import {loadAlertStatuses} from "@app/core/requests"
 import {platform, platformName, getPushInfo} from "@app/util/push"
-import {preferencesStorageProvider} from "@src/lib/storage"
+import {clearFileStorage, preferencesStorageProvider} from "@src/lib/storage"
 
 // Utils
 
@@ -158,6 +158,8 @@ export const logout = async () => {
 
   localStorage.clear()
   await preferencesStorageProvider.clear()
+
+  await clearFileStorage()
 }
 
 // Synchronization

src/lib/storage.ts

@@ -1,5 +1,15 @@
 import {type StorageProvider} from "@welshman/store"
 import {Preferences} from "@capacitor/preferences"
+import type {Unsubscriber} from "svelte/store"
+import {Encoding, Filesystem, type Directory} from "@capacitor/filesystem"
+import {EventsStorageProvider} from "@lib/storage/events"
+import {FreshnessStorageProvider} from "@lib/storage/freshness"
+import {HandlesStorageProvider} from "@lib/storage/handles"
+import {PlaintextStorageProvider} from "@lib/storage/plaintext"
+import {RelaysStorageProvider} from "@lib/storage/relays"
+import {TrackerStorageProvider} from "@lib/storage/tracker"
+import {ZappersStorageProvider} from "@lib/storage/zappers"
+import {repository, tracker, unsubscribers} from "@welshman/app"
 
 export class PreferencesStorageProvider implements StorageProvider {
   get = async <T>(key: string): Promise<T | undefined> => {
@@ -26,3 +36,60 @@ export class PreferencesStorageProvider implements StorageProvider {
 
 // singleton instance of PreferencesStorageProvider
 export const preferencesStorageProvider = new PreferencesStorageProvider()
+
+export interface FilesystemStorageProvider {
+  initializeState(): Promise<void>
+  sync(): Unsubscriber
+}
+
+export const getAllFromFile = async <T>(
+  filepath: string,
+  directory: Directory,
+  encoding: Encoding,
+): Promise<T[]> => {
+  try {
+    const contents = (
+      await Filesystem.readFile({
+        path: filepath,
+        directory,
+        encoding,
+      })
+    ).data.toString()
+
+    if (!contents || contents == "") {
+      return []
+    }
+
+    return JSON.parse(contents)
+  } catch (err) {
+    // file doesn't exist
+    return []
+  }
+}
+
+export const defaultStorageProviders = {
+  relays: new RelaysStorageProvider(),
+  handles: new RelaysStorageProvider(),
+  zappers: new ZappersStorageProvider(),
+  freshness: new FreshnessStorageProvider(),
+  plaintext: new PlaintextStorageProvider(),
+  tracker: new TrackerStorageProvider({tracker}),
+  events: new EventsStorageProvider({limit: 10_000, repository, rankEvent: () => 1}),
+}
+
+export const initFileStorage = async (storageProviders: Record<string, FilesystemStorageProvider>) => {
+  await Promise.all(Object.values(storageProviders).map(async provider => {
+    await provider.initializeState()
+    unsubscribers.push(provider.sync())
+  }))
+}
+
+export const clearFileStorage = async (): Promise<void> => {
+  await EventsStorageProvider.clearStorage()
+  await FreshnessStorageProvider.clearStorage()
+  await HandlesStorageProvider.clearStorage()
+  await PlaintextStorageProvider.clearStorage()
+  await RelaysStorageProvider.clearStorage()
+  await TrackerStorageProvider.clearStorage()
+  await ZappersStorageProvider.clearStorage()
+}

src/lib/storage/events.ts

@@ -0,0 +1,102 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import type {TrustedEvent} from "@welshman/util"
+import type {Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import type {Repository, RepositoryUpdate} from "@welshman/relay"
+import {on, sortBy} from "@welshman/lib"
+
+export class EventsStorageProvider implements FilesystemStorageProvider {
+  static filepath = "events.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+  limit: number
+  repository: Repository
+  rankEvent: (event: TrustedEvent) => number
+  eventCount: number = 0
+  isDeleting = false
+
+  constructor({
+    limit,
+    repository,
+    rankEvent,
+  }: {
+    limit: number
+    repository: Repository
+    rankEvent: (event: TrustedEvent) => number
+  }) {
+    this.limit = limit
+    this.repository = repository
+    this.rankEvent = rankEvent
+  }
+
+  async initializeState(): Promise<void> {
+    const events = await this.getAll()
+    this.eventCount = events.length
+    this.repository.load(events)
+  }
+
+  sync(): Unsubscriber {
+    const onUpdate = async ({added, removed}: RepositoryUpdate) => {
+      // Only add events we want to keep
+      const keep = added.filter(e => this.rankEvent(e) > 0)
+
+      // Add new events
+      if (keep.length > 0) {
+        await this.updateEvents(keep)
+      }
+
+      // If we're well above our retention limit, drop lowest-ranked events
+      if (!this.isDeleting && this.eventCount > this.limit * 1.5) {
+        try {
+          this.isDeleting = true
+
+          for (const event of sortBy(e => -this.rankEvent(e), await this.getAll()).slice(
+            this.limit,
+          )) {
+            removed.add(event.id)
+          }
+
+          if (removed.size > 0) {
+            await this.deleteEvents(Array.from(removed))
+          }
+        } finally {
+          this.isDeleting = false
+        }
+      }
+
+      // Keep track of our total number of events. This isn't strictly accurate, but it's close enough
+      this.eventCount = this.eventCount + keep.length - removed.size
+    }
+
+    return on(this.repository, "update", onUpdate)
+  }
+
+  async getAll(): Promise<TrustedEvent[]> {
+    return await getAllFromFile(EventsStorageProvider.filepath, EventsStorageProvider.directory, EventsStorageProvider.encoding)
+  }
+
+  async writeAll(events: TrustedEvent[]) {
+    await Filesystem.writeFile({
+      path: EventsStorageProvider.filepath,
+      directory: EventsStorageProvider.directory,
+      encoding: EventsStorageProvider.encoding,
+      data: JSON.stringify(events),
+    })
+  }
+
+  async updateEvents(events: TrustedEvent[]) {
+    const existing = await this.getAll()
+    const updated = existing.concat(events)
+    await this.writeAll(updated)
+  }
+
+  async deleteEvents(ids: string[]) {
+    const existing = await this.getAll()
+    const updated = existing.filter(e => !ids.includes(e.id))
+    await this.writeAll(updated)
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: EventsStorageProvider.filepath, directory: EventsStorageProvider.directory})
+  }
+}

src/lib/storage/freshness.ts

@@ -0,0 +1,45 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import type {Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import {fromPairs} from "@welshman/lib"
+import {freshness} from "@welshman/store"
+
+type KV = {key: string; value: any}
+
+export class FreshnessStorageProvider implements FilesystemStorageProvider {
+  static filepath = "freshness.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+
+  async initializeState(): Promise<void> {
+    const items = await this.getAll()
+    freshness.set(fromPairs(items.map(item => [item.key, item.value])))
+  }
+
+  sync(): Unsubscriber {
+    const interval = setInterval(() => {
+      this.writeAll(freshness.get())
+    }, 10_000)
+
+    return () => clearInterval(interval)
+  }
+
+  async getAll(): Promise<KV[]> {
+    return await getAllFromFile(FreshnessStorageProvider.filepath, FreshnessStorageProvider.directory, FreshnessStorageProvider.encoding)
+  }
+
+  async writeAll(items: Record<string, any>) {
+    const kvs = Object.entries(items).map(([key, value]) => ({key, value}))
+
+    await Filesystem.writeFile({
+      path: FreshnessStorageProvider.filepath,
+      directory: FreshnessStorageProvider.directory,
+      encoding: FreshnessStorageProvider.encoding,
+      data: JSON.stringify(kvs),
+    })
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: FreshnessStorageProvider.filepath, directory: FreshnessStorageProvider.directory})
+  }
+}

src/lib/storage/handles.ts

@@ -0,0 +1,40 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import {get, type Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import {batch} from "@welshman/lib"
+import {handles, onHandle, type Handle} from "@welshman/app"
+
+export class HandlesStorageProvider implements FilesystemStorageProvider {
+  static filepath = "handles.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+
+  async initializeState(): Promise<void> {
+    handles.set(await this.getAll())
+  }
+
+  sync(): Unsubscriber {
+    return onHandle(batch(300, () => this.saveState()))
+  }
+
+  async getAll(): Promise<Handle[]> {
+    return await getAllFromFile(HandlesStorageProvider.filepath, HandlesStorageProvider.directory, HandlesStorageProvider.encoding)
+  }
+
+  async writeAll(handles: Handle[]) {
+    await Filesystem.writeFile({
+      path: HandlesStorageProvider.filepath,
+      directory: HandlesStorageProvider.directory,
+      encoding: HandlesStorageProvider.encoding,
+      data: JSON.stringify(handles),
+    })
+  }
+
+  async saveState() {
+    await this.writeAll(get(handles))
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: HandlesStorageProvider.filepath, directory: HandlesStorageProvider.directory})
+  }
+}

src/lib/storage/plaintext.ts

@@ -0,0 +1,45 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import type {Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import {fromPairs} from "@welshman/lib"
+import {plaintext} from "@welshman/app"
+
+type KV = {key: string; value: any}
+
+export class PlaintextStorageProvider implements FilesystemStorageProvider {
+  static filepath = "plaintext.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+
+  async initializeState(): Promise<void> {
+    const items = await this.getAll()
+    plaintext.set(fromPairs(items.map(item => [item.key, item.value])))
+  }
+
+  sync(): Unsubscriber {
+    const interval = setInterval(() => {
+      this.writeAll(plaintext.get())
+    }, 10_000)
+
+    return () => clearInterval(interval)
+  }
+
+  async getAll(): Promise<KV[]> {
+    return await getAllFromFile(PlaintextStorageProvider.filepath, PlaintextStorageProvider.directory, PlaintextStorageProvider.encoding)
+  }
+
+  async writeAll(items: Record<string, any>) {
+    const kvs = Object.entries(items).map(([key, value]) => ({key, value}))
+
+    await Filesystem.writeFile({
+      path: PlaintextStorageProvider.filepath,
+      directory: PlaintextStorageProvider.directory,
+      encoding: PlaintextStorageProvider.encoding,
+      data: JSON.stringify(kvs),
+    })
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: PlaintextStorageProvider.filepath, directory: PlaintextStorageProvider.directory})
+  }
+}

src/lib/storage/relays.ts

@@ -0,0 +1,40 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import type {Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import {relays, type Relay} from "@welshman/app"
+import {throttled} from "@welshman/store"
+
+export class RelaysStorageProvider implements FilesystemStorageProvider {
+  static filepath = "relays.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+
+  async initializeState(): Promise<void> {
+    relays.set(await this.getAll())
+  }
+
+  sync(): Unsubscriber {
+    return throttled(3000, relays).subscribe(() => this.saveState())
+  }
+
+  async getAll(): Promise<Relay[]> {
+    return await getAllFromFile(RelaysStorageProvider.filepath, RelaysStorageProvider.directory, RelaysStorageProvider.encoding)
+  }
+
+  async writeAll(relays: Relay[]) {
+    await Filesystem.writeFile({
+      path: RelaysStorageProvider.filepath,
+      directory: RelaysStorageProvider.directory,
+      encoding: RelaysStorageProvider.encoding,
+      data: JSON.stringify(relays),
+    })
+  }
+
+  async saveState() {
+    await this.writeAll(relays.get())
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: RelaysStorageProvider.filepath, directory: RelaysStorageProvider.directory})
+  }
+}

src/lib/storage/tracker.ts

@@ -0,0 +1,80 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import type {Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import type {Tracker} from "@welshman/net"
+import {call, on} from "@welshman/lib"
+
+type Entry = {id: string; relays: string[]}
+
+export class TrackerStorageProvider implements FilesystemStorageProvider {
+  static filepath = "tracker.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+  tracker: Tracker
+
+  constructor({tracker}: {tracker: Tracker}) {
+    this.tracker = tracker
+  }
+
+  async initializeState(): Promise<void> {
+    const relaysByid = new Map<string, Set<string>>()
+
+    for (const {id, relays} of await this.getAll()) {
+      relaysByid.set(id, new Set(relays))
+    }
+
+    this.tracker.load(relaysByid)
+  }
+
+  sync(): Unsubscriber {
+    const updateOne = async (id: string, relay: string) => {
+      const relays = new Set(await this.getAll())
+      relays.add({id, relays: Array.from(this.tracker.getRelays(id))})
+      await this.writeAll([...relays])
+    }
+
+    const updateAll = async () => {
+      await this.writeAll(Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({
+        id,
+        relays: Array.from(relays),
+      })))
+    }
+
+    const unsubscribers = [
+      on(this.tracker, "add", updateOne),
+      on(this.tracker, "remove", updateOne),
+      on(this.tracker, "load", updateAll),
+      on(this.tracker, "clear", updateAll),
+    ]
+
+    return () => {
+      unsubscribers.forEach(call)
+    }
+  }
+
+  async getAll(): Promise<Entry[]> {
+    return await getAllFromFile(TrackerStorageProvider.filepath, TrackerStorageProvider.directory, TrackerStorageProvider.encoding)
+  }
+
+  async writeAll(relays: Entry[]) {
+    await Filesystem.writeFile({
+      path: TrackerStorageProvider.filepath,
+      directory: TrackerStorageProvider.directory,
+      encoding: TrackerStorageProvider.encoding,
+      data: JSON.stringify(relays),
+    })
+  }
+
+  async saveState() {
+    await this.writeAll(
+      Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({
+        id,
+        relays: Array.from(relays),
+      })),
+    )
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: TrackerStorageProvider.filepath, directory: TrackerStorageProvider.directory})
+  }
+}

src/lib/storage/zappers.ts

@@ -0,0 +1,41 @@
+import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
+import {get, type Unsubscriber} from "svelte/store"
+import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
+import {onZapper, zappers} from "@welshman/app"
+import {batch} from "@welshman/lib"
+import type {Zapper} from "@welshman/util"
+
+export class ZappersStorageProvider implements FilesystemStorageProvider {
+  static filepath = "zappers.json"
+  static directory = Directory.Data
+  static encoding = Encoding.UTF8
+
+  async initializeState(): Promise<void> {
+    zappers.set(await this.getAll())
+  }
+
+  sync(): Unsubscriber {
+    return onZapper(batch(300, () => this.saveState()))
+  }
+
+  async getAll(): Promise<Zapper[]> {
+    return await getAllFromFile(ZappersStorageProvider.filepath, ZappersStorageProvider.directory, ZappersStorageProvider.encoding)
+  }
+
+  async writeAll(zappers: Zapper[]) {
+    await Filesystem.writeFile({
+      path: ZappersStorageProvider.filepath,
+      directory: ZappersStorageProvider.directory,
+      encoding: ZappersStorageProvider.encoding,
+      data: JSON.stringify(zappers),
+    })
+  }
+
+  async saveState() {
+    await this.writeAll(get(zappers))
+  }
+
+  static async clearStorage(): Promise<void> {
+    await Filesystem.deleteFile({path: ZappersStorageProvider.filepath, directory: ZappersStorageProvider.directory})
+  }
+}

src/routes/+layout.svelte

@@ -62,7 +62,6 @@
   import {
     loadRelay,
     db,
-    initStorage,
     repository,
     pubkey,
     session,
@@ -70,7 +69,6 @@
     signer,
     signerLog,
     dropSession,
-    defaultStorageAdapters,
     loginWithNip01,
     loginWithNip46,
     EventsStorageAdapter,
@@ -85,7 +83,7 @@
   import * as net from "@welshman/net"
   import * as app from "@welshman/app"
   import {nsecDecode} from "@lib/util"
-  import {preferencesStorageProvider} from "@lib/storage"
+  import {defaultStorageProviders, initFileStorage, preferencesStorageProvider} from "@lib/storage"
   import AppContainer from "@app/components/AppContainer.svelte"
   import ModalContainer from "@app/components/ModalContainer.svelte"
   import {setupTracking} from "@app/util/tracking"
@@ -111,6 +109,7 @@
   import * as appState from "@app/core/state"
   import {badgeCount, handleBadgeCountChanges} from "@app/util/notifications"
   import NewNotificationSound from "@src/app/components/NewNotificationSound.svelte"
+  import {EventsStorageProvider} from "@lib/storage/events"
 
   // Migration: old nostrtalk instance used different sessions
   if ($session && !$signer) {
@@ -280,10 +279,8 @@
         storage: preferencesStorageProvider,
       })
 
-      await initStorage("flotilla", 8, {
-        ...defaultStorageAdapters,
-        events: new EventsStorageAdapter({
-          name: "events",
+      await initFileStorage({...defaultStorageProviders,
+        events: new EventsStorageProvider({
           limit: 10_000,
           repository,
           rankEvent: (e: TrustedEvent) => {