Re-work storage to optimize file access

src/lib/storage.ts

@@ -1,3 +1,4 @@
+import {flatten, identity, groupBy} from "@welshman/lib"
 import {type StorageProvider} from "@welshman/store"
 import {Preferences} from "@capacitor/preferences"
 import {Encoding, Filesystem, Directory} from "@capacitor/filesystem"
@@ -30,79 +31,92 @@ export class PreferencesStorageProvider implements StorageProvider {
 
 export const preferencesStorageProvider = new PreferencesStorageProvider()
 
-export class CollectionStorageProvider implements StorageProvider {
+export type CollectionOptions<T> = {
+  table: string
+  shards: string[]
+  getShard: (item: T) => string
+}
+
+export class Collection<T> {
   p = Promise.resolve()
 
-  get = async <T>(key: string): Promise<T[]> => {
+  constructor(readonly options: CollectionOptions<T>) {}
+
+  static clearAll = async (): Promise<void> => {
+    const res = await Filesystem.readdir({
+      path: "",
+      directory: Directory.Data,
+    })
+
+    await Promise.all(
+      res.files.map(file =>
+        Filesystem.deleteFile({
+          path: file.name,
+          directory: Directory.Data,
+        }),
+      ),
+    )
+  }
+
+  #then = async (f: () => Promise<void>) => {
+    this.p = this.p.then(f).catch(e => {
+      console.error(e)
+    })
+
+    await this.p
+  }
+
+  #path = (shard: string) => `collection_${this.options.table}_${shard}.json`
+
+  getShard = async (shard: string): Promise<T[]> => {
     try {
       const file = await Filesystem.readFile({
-        path: key + ".json",
+        path: this.#path(shard),
         directory: Directory.Data,
         encoding: Encoding.UTF8,
       })
 
-      const items: T[] = []
-      for (const line of file.data.toString().split("\n")) {
-        try {
-          items.push(JSON.parse(line))
-        } catch (e) {
-          // pass
-        }
-      }
-
-      return items
+      // Speed things up by parsing only once
+      return JSON.parse("[" + file.data.toString().split("\n").filter(identity).join(",") + "]")
     } catch (err) {
       // file doesn't exist, or isn't valid json
       return []
     }
   }
 
-  set = async <T>(key: string, value: T[]): Promise<void> => {
-    this.p = this.p.then(async () => {
+  get = async (): Promise<T[]> => flatten(await Promise.all(this.options.shards.map(this.getShard)))
+
+  setShard = (shard: string, items: T[]) =>
+    this.#then(async () => {
       await Filesystem.writeFile({
-        path: key + ".json",
+        path: this.#path(shard),
         directory: Directory.Data,
         encoding: Encoding.UTF8,
-        data: value.map(v => JSON.stringify(v)).join("\n"),
+        data: items.map(v => JSON.stringify(v)).join("\n"),
       })
     })
 
-    await this.p
-  }
+  set = (items: T[]) =>
+    Promise.all(
+      Array.from(groupBy(this.options.getShard, items)).map(([shard, chunk]) =>
+        this.setShard(shard, chunk),
+      ),
+    )
 
-  add = async <T>(key: string, value: T[]): Promise<void> => {
-    this.p = this.p.then(async () => {
+  addToShard = (shard: string, items: T[]) =>
+    this.#then(async () => {
       await Filesystem.appendFile({
-        path: key + ".json",
+        path: this.#path(shard),
         directory: Directory.Data,
         encoding: Encoding.UTF8,
-        data: "\n" + value.map(v => JSON.stringify(v)).join("\n"),
+        data: "\n" + items.map(v => JSON.stringify(v)).join("\n"),
       })
     })
 
-    await this.p
-  }
-
-  clear = async (): Promise<void> => {
-    this.p = this.p.then(async () => {
-      try {
-        const res = await Filesystem.readdir({path: "./", directory: Directory.Data})
-
-        await Promise.all(
-          res.files.map(file =>
-            Filesystem.deleteFile({
-              path: file.name + ".json",
-              directory: Directory.Data,
-            }),
-          ),
-        )
-      } catch (e) {
-        // Directory might not have been created yet
-      }
-    })
-
-    await this.p
-  }
+  add = (items: T[]) =>
+    Promise.all(
+      Array.from(groupBy(this.options.getShard, items)).map(([shard, chunk]) =>
+        this.addToShard(shard, chunk),
+      ),
+    )
 }
-
-export const collectionStorageProvider = new CollectionStorageProvider()