From fdce131d27c1ac2f91ed78e592061907dce4606f Mon Sep 17 00:00:00 2001
From: Jon Staab <shtaab@gmail.com>
Date: Wed, 22 May 2024 10:30:29 -0700
Subject: [PATCH] Trim filters, fix some issues with feed loaders

---
 packages/feeds/loader.ts | 18 +++++++++---------
 packages/lib/Tools.ts    | 20 ++++++++++++++++++++
 packages/util/Filters.ts |  7 ++++++-
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/packages/feeds/loader.ts b/packages/feeds/loader.ts
index 0addfa2..6f1393e 100644
--- a/packages/feeds/loader.ts
+++ b/packages/feeds/loader.ts
@@ -1,6 +1,6 @@
 import {inc, max, min, now} from '@welshman/lib'
 import type {TrustedEvent, Filter} from '@welshman/util'
-import {EPOCH, guessFilterDelta} from '@welshman/util'
+import {EPOCH, trimFilters, guessFilterDelta} from '@welshman/util'
 import type {Feed, RequestItem, FeedOptions} from './core'
 import {FeedType} from './core'
 import {FeedCompiler} from './compiler'
@@ -100,26 +100,26 @@ export class FeedLoader<E extends TrustedEvent> {
 
       await this.options.request({
         relays,
-        filters: requestFilters,
+        filters: trimFilters(requestFilters),
         onEvent: (event: E) => {
           count += 1
-          until = Math.min(until, event.created_at)
+          until = Math.min(until, event.created_at - 1)
           onEvent?.(event)
         },
       })
 
+      if (since === minSince) {
+        onExhausted?.()
+      }
+
       // Relays can't be relied upon to return events in descending order, do exponential
       // windowing to ensure we get the most recent stuff on first load, but eventually find it all
-      if (count === 0) {
-        delta *= 10
+      if (count < limit) {
+        delta = delta * Math.round(10 - 9 * (Math.log(count + 1) / Math.log(limit + 1)))
         until = since
       }
 
       since = Math.max(minSince, until - delta)
-
-      if (since === minSince) {
-        onExhausted?.()
-      }
     }
   }
 
diff --git a/packages/lib/Tools.ts b/packages/lib/Tools.ts
index f7c6e54..2f77965 100644
--- a/packages/lib/Tools.ts
+++ b/packages/lib/Tools.ts
@@ -63,6 +63,26 @@ export function* range(a: number, b: number, step = 1) {
   }
 }
 
+export const mapKeys = <T extends Record<string, any>>(f: (v: string) => string, x: T) => {
+  const r: Record<string, any> = {}
+
+  for (const [k, v] of Object.entries(x)) {
+    r[f(k)] = v
+  }
+
+  return r as T
+}
+
+export const mapVals = <T extends Record<string, any>>(f: (v: any) => any, x: T) => {
+  const r: Record<string, any> = {}
+
+  for (const [k, v] of Object.entries(x)) {
+    r[k] = f(v)
+  }
+
+  return r as T
+}
+
 export const between = (low: number, high: number, n: number) => n > low && n < high
 
 export const randomId = (): string => Math.random().toString().slice(2)
diff --git a/packages/util/Filters.ts b/packages/util/Filters.ts
index d43f770..e468a77 100644
--- a/packages/util/Filters.ts
+++ b/packages/util/Filters.ts
@@ -1,6 +1,6 @@
 import {Event} from 'nostr-tools'
 import {matchFilter as nostrToolsMatchFilter} from 'nostr-tools'
-import {prop, avg, hash, groupBy, randomId, uniq} from '@welshman/lib'
+import {prop, mapVals, shuffle, avg, hash, groupBy, randomId, uniq} from '@welshman/lib'
 import type {HashedEvent, TrustedEvent} from './Events'
 import {isReplaceableKind} from './Kinds'
 import {Address, getAddress} from './Address'
@@ -213,3 +213,8 @@ export const getFilterResultCardinality = (filter: Filter) => {
 
   return null
 }
+
+export const trimFilter = (filter: Filter) =>
+  mapVals(v => Array.isArray(v) && v.length > 1000 ? shuffle(v).slice(0, 1000) : v, filter)
+
+export const trimFilters = (filters: Filter[]) => filters.map(trimFilter)