Trim filters, fix some issues with feed loaders

packages/feeds/loader.ts

@@ -1,6 +1,6 @@
 import {inc, max, min, now} from '@welshman/lib'
 import type {TrustedEvent, Filter} from '@welshman/util'
-import {EPOCH, guessFilterDelta} from '@welshman/util'
+import {EPOCH, trimFilters, guessFilterDelta} from '@welshman/util'
 import type {Feed, RequestItem, FeedOptions} from './core'
 import {FeedType} from './core'
 import {FeedCompiler} from './compiler'
@@ -100,26 +100,26 @@ export class FeedLoader<E extends TrustedEvent> {
 
       await this.options.request({
         relays,
-        filters: requestFilters,
+        filters: trimFilters(requestFilters),
         onEvent: (event: E) => {
           count += 1
-          until = Math.min(until, event.created_at)
+          until = Math.min(until, event.created_at - 1)
           onEvent?.(event)
         },
       })
 
+      if (since === minSince) {
+        onExhausted?.()
+      }
+
       // Relays can't be relied upon to return events in descending order, do exponential
       // windowing to ensure we get the most recent stuff on first load, but eventually find it all
-      if (count === 0) {
+      if (count < limit) {
-        delta *= 10
+        delta = delta * Math.round(10 - 9 * (Math.log(count + 1) / Math.log(limit + 1)))
         until = since
       }
 
       since = Math.max(minSince, until - delta)
-
-      if (since === minSince) {
-        onExhausted?.()
-      }
     }
   }
 

packages/lib/Tools.ts

@@ -63,6 +63,26 @@ export function* range(a: number, b: number, step = 1) {
   }
 }
 
+export const mapKeys = <T extends Record<string, any>>(f: (v: string) => string, x: T) => {
+  const r: Record<string, any> = {}
+
+  for (const [k, v] of Object.entries(x)) {
+    r[f(k)] = v
+  }
+
+  return r as T
+}
+
+export const mapVals = <T extends Record<string, any>>(f: (v: any) => any, x: T) => {
+  const r: Record<string, any> = {}
+
+  for (const [k, v] of Object.entries(x)) {
+    r[k] = f(v)
+  }
+
+  return r as T
+}
+
 export const between = (low: number, high: number, n: number) => n > low && n < high
 
 export const randomId = (): string => Math.random().toString().slice(2)

packages/util/Filters.ts

@@ -1,6 +1,6 @@
 import {Event} from 'nostr-tools'
 import {matchFilter as nostrToolsMatchFilter} from 'nostr-tools'
-import {prop, avg, hash, groupBy, randomId, uniq} from '@welshman/lib'
+import {prop, mapVals, shuffle, avg, hash, groupBy, randomId, uniq} from '@welshman/lib'
 import type {HashedEvent, TrustedEvent} from './Events'
 import {isReplaceableKind} from './Kinds'
 import {Address, getAddress} from './Address'
@@ -213,3 +213,8 @@ export const getFilterResultCardinality = (filter: Filter) => {
 
   return null
 }
+
+export const trimFilter = (filter: Filter) =>
+  mapVals(v => Array.isArray(v) && v.length > 1000 ? shuffle(v).slice(0, 1000) : v, filter)
+
+export const trimFilters = (filters: Filter[]) => filters.map(trimFilter)