Approach request optimization differently

packages/app/src/core.ts

@@ -1,7 +1,7 @@
 import {isNil} from "@welshman/lib"
 import {Repository, Relay, LOCAL_RELAY_URL, getFilterResultCardinality} from "@welshman/util"
 import type {TrustedEvent, Filter} from "@welshman/util"
-import {Tracker, subscribe as baseSubscribe, mergeSubscriptions} from "@welshman/net"
+import {Tracker, subscribe as baseSubscribe} from "@welshman/net"
 import type {SubscribeRequest} from "@welshman/net"
 import {createEventStore} from "@welshman/store"
 import type {Router} from './router'
@@ -52,29 +52,23 @@ export const subscribe = (request: PartialSubscribeRequest) => {
     }
   }
 
+  // Make sure to query our local relay too
   const delay = AppContext.requestDelay
   const timeout = AppContext.requestTimeout
+  const sub = baseSubscribe({delay, authTimeout: timeout, relays: [], ...request})
 
-  return mergeSubscriptions(
+  sub.emitter.on("event", (url: string, e: TrustedEvent) => {
-    AppContext.splitRequest!(request).map(req => {
+    repository.publish(e)
-      // Make sure to query our local relay too
+  })
-      const relays = [...req.relays, LOCAL_RELAY_URL]
-      const sub = baseSubscribe({delay, authTimeout: timeout, ...req, relays})
 
-      sub.emitter.on("event", (url: string, e: TrustedEvent) => {
+  // Keep cached results async so the caller can set up handlers
-        repository.publish(e)
+  setTimeout(() => {
-      })
+    for (const event of events) {
+      sub.emitter.emit("event", LOCAL_RELAY_URL, event)
+    }
+  })
 
-      // Keep cached results async so the caller can set up handlers
+  return sub
-      setTimeout(() => {
-        for (const event of events) {
-          sub.emitter.emit("event", LOCAL_RELAY_URL, event)
-        }
-      })
-
-      return sub
-    })
-  )
 }
 
 export const load = (request: PartialSubscribeRequest) =>

packages/app/src/index.ts

@@ -16,19 +16,29 @@ export * from './topics'
 export * from './util'
 export * from './zappers'
 
-import {NetworkContext} from "@welshman/net"
+import {partition} from "@welshman/lib"
-import {type TrustedEvent} from "@welshman/util"
+import {type Subscription, NetworkContext, defaultOptimizeSubscriptions} from "@welshman/net"
+import {type TrustedEvent, unionFilters} from "@welshman/util"
 import {tracker, repository, AppContext} from './core'
-import {splitRequest, makeRouter} from './router'
+import {makeRouter, getFilterSelections} from './router'
 import {onAuth} from './session'
 
+export function* optimizeSubscriptions(subs: Subscription[]) {
+  const [withRelays, withoutRelays] = partition(sub => sub.request.relays.length > 0, subs)
+
+  yield* defaultOptimizeSubscriptions(withRelays)
+  yield* getFilterSelections(
+    unionFilters(withoutRelays.flatMap(sub => sub.request.filters))
+  )
+}
+
 Object.assign(NetworkContext, {
   onAuth,
   onEvent: (url: string, event: TrustedEvent) => tracker.track(event.id, url),
   isDeleted: (url: string, event: TrustedEvent) => repository.isDeleted(event),
+  optimizeSubscriptions,
 })
 
 Object.assign(AppContext, {
-  splitRequest,
   router: makeRouter(),
 })

packages/app/src/router.ts

@@ -3,7 +3,6 @@ import {Tags, getFilterId, unionFilters, isShareableRelayUrl, isCommunityAddress
 import type {TrustedEvent, Filter} from '@welshman/util'
 import {NetworkContext, ConnectionStatus} from '@welshman/net'
 import {AppContext} from './core'
-import type {PartialSubscribeRequest} from './core'
 import {pubkey} from './session'
 import {relaySelectionsByPubkey, getReadRelayUrls, getWriteRelayUrls, getRelayUrls} from './relaySelections'
 import {relays, relaysByUrl} from './relays'
@@ -443,7 +442,7 @@ export const makeRouter = (options: Partial<RouterOptions> = {}) =>
 // Infer relay selections from filters
 
 export type RelayFilters = {
-  relay: string
+  relays: string[]
   filters: Filter[]
 }
 
@@ -561,7 +560,7 @@ export const getFilterSelections = (filters: Filter[]): RelayFilters[] => {
     .getSelections()
     .map(({values, relay}) => ({
       filters: values.map(id => filtersById.get(id)!),
-      relay,
+      relays: [relay],
     }))
 
   // Pubkey-based selections can get really big. Use the most popular relays for the long tail
@@ -575,11 +574,3 @@ export const getFilterSelections = (filters: Filter[]): RelayFilters[] => {
 
   return keep
 }
-
-export const splitRequest = (req: PartialSubscribeRequest) => {
-  if ((req.relays || []).length > 0) return [req]
-
-  return getFilterSelections(req.filters)
-    .map(({relay, filters}) => ({...req, filters, relays: [relay]}))
-}
-

packages/lib/src/Tools.ts

@@ -197,6 +197,19 @@ export const isPojo = (obj: any) => {
 export const equals = (a: any, b: any) => {
   if (a === b) return true
 
+  if (a instanceof Set && b instanceof Set) {
+    a = Array.from(a)
+    b = Array.from(b)
+  }
+
+  if (a instanceof Set) {
+    if (!(b instanceof Set) || a.size !== b.size) {
+      return false
+    }
+
+    return Array.from(a).every(x => b.has(x))
+  }
+
   if (Array.isArray(a)) {
     if (!Array.isArray(b) || a.length !== b.length) {
       return false

packages/net/src/Context.ts

@@ -1,8 +1,10 @@
-import {matchFilters, hasValidSignature} from '@welshman/util'
+import {uniq} from '@welshman/lib'
+import {matchFilters, unionFilters, hasValidSignature} from '@welshman/util'
 import type {Filter, SignedEvent} from '@welshman/util'
 import {Pool} from "./Pool"
 import {Executor} from "./Executor"
 import {Relays} from "./target/Relays"
+import type {Subscription} from "./Subscribe"
 
 export const defaultPool = new Pool()
 
@@ -21,6 +23,15 @@ const defaultHasValidSignature = (url: string, event: SignedEvent) => hasValidSi
 
 const defaultMatchFilters = (url: string, filters: Filter[], event: SignedEvent) => matchFilters(filters, event)
 
+export function* defaultOptimizeSubscriptions(subs: Subscription[]) {
+  for (const relay of uniq(subs.flatMap(sub => sub.request.relays || []))) {
+    const relaySubs = subs.filter(sub => sub.request.relays.includes(relay))
+    const filters = unionFilters(relaySubs.flatMap(sub => sub.request.filters))
+
+    yield {relays: [relay], filters}
+  }
+}
+
 export const NetworkContext = {
   pool: defaultPool,
   getExecutor: defaultGetExecutor,
@@ -30,4 +41,5 @@ export const NetworkContext = {
   isDeleted: defaultIsDeleted,
   hasValidSignature: defaultHasValidSignature,
   matchFilters: defaultMatchFilters,
+  optimizeSubscriptions: defaultOptimizeSubscriptions,
 }

packages/net/src/Subscribe.ts

@@ -66,6 +66,7 @@ export const calculateSubscriptionGroup = (sub: Subscription) => {
   const parts: string[] = []
 
   if (sub.request.timeout) parts.push(`timeout:${sub.request.timeout}`)
+  if (sub.request.authTimeout) parts.push(`authTimeout:${sub.request.authTimeout}`)
   if (sub.request.closeOnEose) parts.push('closeOnEose')
 
   return parts.join('|')
@@ -123,28 +124,31 @@ export const mergeSubscriptions = (subs: Subscription[]) => {
   return mergedSub
 }
 
-export const optimizeSubscriptions = (subs: Subscription[]) =>
+export const optimizeSubscriptions = (subs: Subscription[]) => {
-  Array.from(groupBy(calculateSubscriptionGroup, subs).values())
+  return Array.from(groupBy(calculateSubscriptionGroup, subs).values())
     .flatMap(group => {
-      const completedRelays = new Set()
+      const timeout = max(group.map(sub => sub.request.timeout || 0))
+      const authTimeout = max(group.map(sub => sub.request.authTimeout || 0))
+      const closeOnEose = group.every(sub => sub.request.closeOnEose)
+      const completedSubs = new Set<string>()
+      const abortedSubs = new Set<string>()
+      const closedSubs = new Set<string>()
+      const eosedSubs = new Set<string>()
       const mergedSubs = []
 
-      for (const relay of uniq(group.flatMap((sub: Subscription) => sub.request.relays))) {
+      for (const {relays, filters} of NetworkContext.optimizeSubscriptions(group)) {
-        const abortedSubs = new Set()
+        const mergedSub = makeSubscription({filters,
-        const callerSubs = group.filter((sub: Subscription) => sub.request.relays.includes(relay))
+          relays,
-        const mergedSub = makeSubscription({
+          timeout,
-          relays: [relay],
+          authTimeout,
-          closeOnEose: callerSubs.every(sub => sub.request.closeOnEose),
+          closeOnEose
-          timeout: max(callerSubs.map(sub => sub.request.timeout || 0)),
-          authTimeout: max(callerSubs.map(sub => sub.request.authTimeout || 0)),
-          filters: unionFilters(callerSubs.flatMap((sub: Subscription) => sub.request.filters)),
         })
 
-        for (const {id, controller, request} of callerSubs) {
+        for (const {id, controller, request} of group) {
           const onAbort = () => {
             abortedSubs.add(id)
 
-            if (abortedSubs.size === callerSubs.length) {
+            if (abortedSubs.size === group.length) {
               mergedSub.close()
             }
           }
@@ -154,7 +158,7 @@ export const optimizeSubscriptions = (subs: Subscription[]) =>
         }
 
         mergedSub.emitter.on(SubscriptionEvent.Event, (url: string, event: SignedEvent) => {
-          for (const sub of callerSubs) {
+          for (const sub of group) {
             if (!sub.tracker.track(event.id, url) && matchFilters(sub.request.filters, event)) {
               sub.emitter.emit(SubscriptionEvent.Event, url, event)
             }
@@ -162,53 +166,48 @@ export const optimizeSubscriptions = (subs: Subscription[]) =>
         })
 
         // Pass events back to caller
-        const propagateEvent = (type: SubscriptionEvent, checkFilter: boolean) =>
+        const propagateEvent = (type: SubscriptionEvent) =>
           mergedSub.emitter.on(type,  (url: string, event: SignedEvent) => {
-            for (const sub of callerSubs) {
+            for (const sub of group) {
-              if (!checkFilter || matchFilters(sub.request.filters, event)) {
+              if (matchFilters(sub.request.filters, event)) {
                 sub.emitter.emit(type, url, event)
               }
             }
           })
 
-        propagateEvent(SubscriptionEvent.Duplicate, true)
+        propagateEvent(SubscriptionEvent.Duplicate)
-        propagateEvent(SubscriptionEvent.DeletedEvent, false)
+        propagateEvent(SubscriptionEvent.DeletedEvent)
-        propagateEvent(SubscriptionEvent.FailedFilter, false)
+        propagateEvent(SubscriptionEvent.InvalidSignature)
-        propagateEvent(SubscriptionEvent.InvalidSignature, true)
 
-        // Propagate eose
+        const propagateFinality = (type: SubscriptionEvent, subIds: Set<string>) =>
-        mergedSub.emitter.on(SubscriptionEvent.Eose, (url: string) => {
+          mergedSub.emitter.on(type, (...args: any[]) => {
-          for (const sub of callerSubs) {
+            subIds.add(mergedSub.id)
-            sub.emitter.emit(SubscriptionEvent.Eose, url)
-          }
-        })
 
-        // Propagate close
+            // Wait for all subscriptions to complete before reporting finality to the caller.
-        mergedSub.emitter.on(SubscriptionEvent.Close, (url: string) => {
+            // This is sub-optimal, but because we're outsourcing filter/relay optimization
-          for (const sub of callerSubs) {
+            // we can't make any assumptions about which caller subscriptions have completed
-            sub.emitter.emit(SubscriptionEvent.Close, url)
+            // at any given time.
-          }
+            if (subIds.size === group.length) {
-        })
+              for (const sub of group) {
-
+                sub.emitter.emit(type, ...args)
-        // Propagate subscription completion. Since we split subs by relay, we need to wait
+              }
-        // until all relays are completed before we notify
-        mergedSub.emitter.on(SubscriptionEvent.Complete, () => {
-          completedRelays.add(relay)
-
-          for (const sub of callerSubs) {
-            if (sub.request.relays.every(url => completedRelays.has(url))) {
-              sub.emitter.emit(SubscriptionEvent.Complete)
             }
-          }
 
-          mergedSub.emitter.removeAllListeners()
+            if (type === SubscriptionEvent.Complete) {
-        })
+              mergedSub.emitter.removeAllListeners()
+            }
+          })
+
+        propagateFinality(SubscriptionEvent.Eose, eosedSubs)
+        propagateFinality(SubscriptionEvent.Close, closedSubs)
+        propagateFinality(SubscriptionEvent.Complete, completedSubs)
 
         mergedSubs.push(mergedSub)
       }
 
       return mergedSubs
     })
+}
 
 export const executeSubscription = (sub: Subscription) => {
   const {request, emitter, tracker, controller} = sub
@@ -328,10 +327,6 @@ export const executeSubscriptionBatched = (() => {
 export const subscribe = (request: SubscribeRequest) => {
   const subscription: Subscription = makeSubscription({delay: 50, ...request})
 
-  if (request.relays.length === 0) {
-    console.warn("Attempted to execute a subscription with zero relays")
-  }
-
   if (request.delay === 0) {
     executeSubscription(subscription)
   } else {