Improve feed loader, wait for db before executing reads/updates, make taskQueue subscribable, add race, rename auth methods, fix failed wasm verify, fix localhost urls

packages/app/src/feeds.ts

@@ -1,6 +1,6 @@
-import {nthEq, now} from "@welshman/lib"
+import {nthEq, partition, race, now} from "@welshman/lib"
 import {createEvent, getPubkeyTagValues} from "@welshman/util"
-import {MultiRequest, RequestEvent} from "@welshman/net"
+import {MultiRequest, Tracker, RequestEvent, request} from "@welshman/net"
 import {Scope, FeedController, RequestOpts, FeedOptions, DVMOpts, Feed} from "@welshman/feeds"
 import {makeDvmRequest, DVMEvent} from "@welshman/dvm"
 import {makeSecret, Nip01Signer} from "@welshman/signer"
@@ -8,20 +8,68 @@ import {pubkey, signer} from "./session.js"
 import {Router, addMinimalFallbacks, getFilterSelections} from "./router.js"
 import {loadRelaySelections} from "./relaySelections.js"
 import {wotGraph, maxWot, getFollows, getNetwork, getFollowers} from "./wot.js"
+import {repository} from "./core.js"
 
-export const request = async ({filters = [{}], relays = [], onEvent}: RequestOpts) => {
-  if (relays.length > 0) {
-    await new Promise<void>(resolve => {
-      const sub = new MultiRequest({filters, relays, timeout: 5000, autoClose: true})
-
-      sub.on(RequestEvent.Event, onEvent)
-      sub.on(RequestEvent.Close, resolve)
-    })
-  } else {
-    await Promise.all(getFilterSelections(filters).map(opts => request({...opts, onEvent})))
-  }
+export type FeedRequestHandlerOptions = {
+  signal?: AbortSignal
 }
 
+export const makeFeedRequestHandler = ({signal}: FeedRequestHandlerOptions) =>
+  async ({filters = [{}], relays = [], onEvent}: RequestOpts) => {
+    const tracker = new Tracker()
+    const requestOptions = {}
+
+    if (relays.length > 0) {
+      await new Promise(resolve => {
+        const req = request({tracker, signal, autoClose: true, relays, filters})
+
+        req.on(RequestEvent.Event, onEvent)
+        req.on(RequestEvent.Close, resolve)
+      })
+    } else {
+      const requests: MultiRequest[] = []
+      const [withSearch, withoutSearch] = partition(f => Boolean(f.search), filters)
+
+      if (withSearch.length > 0) {
+        requests.push(
+          request({
+            tracker, signal, autoClose: true,
+            filters: withSearch,
+            relays: Router.get().Search().getUrls(),
+          }),
+        )
+      }
+
+      if (withoutSearch.length > 0) {
+        requests.push(
+          ...getFilterSelections(filters).flatMap(options =>
+            request({tracker, signal, autoClose: true, ...options}),
+          ),
+        )
+      }
+
+      // Break out selections by relay so we can complete early after a certain number
+      // of requests complete for faster load times
+      await race(
+        withSearch.length > 0 ? 0.1 : 0.8,
+        requests.map(
+          req =>
+            new Promise(resolve => {
+              req.on(RequestEvent.Event, onEvent)
+              req.on(RequestEvent.Close, resolve)
+            }),
+        ),
+      )
+
+      // Wait until after we've queried the network to access our local cache. This results in less
+      // snappy response times, but is necessary to prevent stale stuff that the user has already seen
+      // from showing up at the top of the feed
+      for (const event of repository.query(filters)) {
+        onEvent(event)
+      }
+    }
+  }
+
 export const requestDVM = async ({kind, onEvent, ...request}: DVMOpts) => {
   // Make sure we know what relays to use for target dvms
   if (request.tags && !request.relays) {
@@ -101,11 +149,14 @@ export const getPubkeysForWOTRange = (min: number, max: number) => {
 
 type _FeedOptions = Partial<Omit<FeedOptions, "feed">> & {feed: Feed}
 
-export const createFeedController = (options: _FeedOptions) =>
-  new FeedController({
+export const createFeedController = (options: _FeedOptions) => {
+  const request = makeFeedRequestHandler(options)
+
+  return new FeedController({
     request,
     requestDVM,
     getPubkeysForScope,
     getPubkeysForWOTRange,
     ...options,
   })
+}

packages/app/src/storage.ts

@@ -2,7 +2,7 @@ import {openDB, deleteDB} from "idb"
 import {IDBPDatabase} from "idb"
 import {writable} from "svelte/store"
 import {Unsubscriber} from "svelte/store"
-import {call} from "@welshman/lib"
+import {call, defer} from "@welshman/lib"
 import {withGetter} from "@welshman/store"
 
 export type StorageAdapterOptions = {
@@ -18,11 +18,15 @@ export type StorageAdapter = {
 
 export let db: IDBPDatabase | undefined
 
+export const ready = defer<void>()
+
 export const dead = withGetter(writable(false))
 
 export const subs: Unsubscriber[] = []
 
 export const getAll = async (name: string) => {
+  await ready
+
   const tx = db!.transaction(name, "readwrite")
   const store = tx.objectStore(name)
   const result = await store.getAll()
@@ -33,6 +37,8 @@ export const getAll = async (name: string) => {
 }
 
 export const bulkPut = async (name: string, data: any[]) => {
+  await ready
+
   const tx = db!.transaction(name, "readwrite")
   const store = tx.objectStore(name)
 
@@ -50,6 +56,8 @@ export const bulkPut = async (name: string, data: any[]) => {
 }
 
 export const bulkDelete = async (name: string, ids: string[]) => {
+  await ready
+
   const tx = db!.transaction(name, "readwrite")
   const store = tx.objectStore(name)
 
@@ -90,6 +98,8 @@ export const initStorage = async (
     },
   })
 
+  ready.resolve()
+
   await Promise.all(Object.values(adapters).map(adapter => adapter.init()))
 
   const unsubscribers = Object.values(adapters).map(adapter => adapter.sync())

packages/app/src/thunk.ts

@@ -1,9 +1,11 @@
+import type {Subscriber} from "svelte/store"
 import {Writable, Readable, writable, derived, get} from "svelte/store"
 import {
   Deferred,
   fromPairs,
   TaskQueue,
   dissoc,
+  remove,
   identity,
   uniq,
   defer,