From 5b73c507b7c17eeabd8feea5deb4bc09f6162e7a Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Tue, 10 Jun 2025 14:15:45 -0700 Subject: [PATCH] Update feed docs --- docs/feeds/compiler.md | 155 ++++++++--------- docs/feeds/controller.md | 199 +++++++++------------ docs/feeds/core.md | 362 +++++++-------------------------------- docs/feeds/utils.md | 21 ++- 4 files changed, 227 insertions(+), 510 deletions(-) diff --git a/docs/feeds/compiler.md b/docs/feeds/compiler.md index b7f41f6..22cdfa7 100644 --- a/docs/feeds/compiler.md +++ b/docs/feeds/compiler.md @@ -1,101 +1,82 @@ # Feed Compiler -The `FeedCompiler` class is responsible for transforming feed definitions into executable relay requests. It handles the complex task of converting various feed types into optimized filters and relay selections. +The `FeedCompiler` class transforms feed definitions into optimized `RequestItem[]` arrays containing filters and relay selections for efficient event fetching. -## Overview +## Types ```typescript -class FeedCompiler { - constructor(readonly options: FeedOptions) +export type FeedCompilerOptions = { + signer?: ISigner + signal?: AbortSignal + context?: AdapterContext + getPubkeysForScope: (scope: Scope) => string[] + getPubkeysForWOTRange: (minWOT: number, maxWOT: number) => string[] +} +``` +## FeedCompiler Class + +```typescript +export class FeedCompiler { + constructor(readonly options: FeedCompilerOptions) + + // Check if a feed can be compiled canCompile(feed: Feed): boolean - compile(feed: Feed): Promise + + // Compile a feed into request items + async compile(feed: Feed): Promise } ``` -## Feed Compilation Process +## Compilation Logic + +### Basic Feed Types + +- **ID feeds** → `{filters: [{ids: [...]}]}` +- **Kind feeds** → `{filters: [{kinds: [...]}]}` +- **Author feeds** → `{filters: [{authors: [...]}]}` +- **Tag feeds** → `{filters: [{[key]: [...values]}]}` +- **Address feeds** → Converts to ID filters using `getIdFilters()` +- **Relay feeds** → `{relays: [...urls]}` +- **Global feeds** → `{filters: [{}]}` + +### Time-based Feeds + +- **CreatedAt feeds** → Processes `since`/`until` with optional relative timestamps +- **Scope feeds** → Resolves to author filters using `getPubkeysForScope()` +- **WOT feeds** → Resolves to author filters using `getPubkeysForWOTRange()` +- **Search feeds** → `{filters: [{search: "term"}]}` + +### Complex Feed Types + +- **DVM feeds** → Requests DVM responses and converts result tags to feeds +- **List feeds** → Fetches list events and converts their tags to feeds +- **Label feeds** → Fetches label events (kind 1985) and converts tags to feeds + +### Set Operations + +- **Union feeds** → Merges all sub-feed results, optimizing by relay +- **Intersection feeds** → Finds overlapping filters and relays across sub-feeds + +## Usage -The compiler transforms feed definitions into `RequestItem[]`, where each item contains: ```typescript -type RequestItem = { - relays?: string[] // Specific relays to query - filters?: Filter[] // Nostr filters to apply -} -``` - -## Examples - -### Basic Feed Compilation -```typescript -const compiler = new FeedCompiler(options) - -// Simple author feed -const feed = [FeedType.Author, "pubkey1", "pubkey2"] -const requests = await compiler.compile(feed) -// => [{ filters: [{ authors: ["pubkey1", "pubkey2"] }] }] -``` - -### Complex Feed Compilation -```typescript -// Complex feed with multiple operations -const feed = [ - FeedType.Intersection, - [FeedType.Kind, 1], - [ - FeedType.Union, - [FeedType.Scope, Scope.Follows], - [FeedType.List, { addresses: ["trending"] }] - ] -] - -const requests = await compiler.compile(feed) -// Compiles to optimized filters for relay queries -``` - -### DVM Integration -```typescript -const feed = [ - FeedType.DVM, - { - kind: 5300, - mappings: [ - ["p", [FeedType.Author]], - ["t", [FeedType.Tag, "#t"]] - ] - } -] - -const requests = await compiler.compile(feed) -// Queries DVM and compiles resulting tags into feeds -``` - -## Implementation Notes - -### Optimization Strategies - -1. **Filter Merging**: Similar filters are combined when possible - ```typescript - // Before: [{ authors: ["a"] }, { authors: ["b"] }] - // After: [{ authors: ["a", "b"] }] - ``` - -2. **Relay Grouping**: Requests are grouped by relay where possible - ```typescript - // Filters are organized by relay to minimize connections - filtersByRelay: Map - ``` - -3. **Deduplication**: Duplicate values are removed using `uniq` - ```typescript - uniq(scopes.flatMap(this.options.getPubkeysForScope)) - ``` - -### Error Handling - -The compiler includes various safety checks: -```typescript -canCompile(feed: Feed): boolean { - // Checks if feed type is supported - // Recursively checks sub-feeds +import { FeedCompiler, makeAuthorFeed, makeKindFeed } from '@welshman/feeds' + +const compiler = new FeedCompiler({ + getPubkeysForScope: (scope) => [...], // Your scope resolution logic + getPubkeysForWOTRange: (min, max) => [...], // Your WOT logic + context: adapterContext, + signal: abortSignal +}) + +// Compile a simple feed +const feed = makeAuthorFeed("pubkey1", "pubkey2") +const requests = await compiler.compile(feed) +// => [{filters: [{authors: ["pubkey1", "pubkey2"]}]}] + +// Check if feed can be compiled +if (compiler.canCompile(feed)) { + const requests = await compiler.compile(feed) } ``` diff --git a/docs/feeds/controller.md b/docs/feeds/controller.md index afc4008..00b5c74 100644 --- a/docs/feeds/controller.md +++ b/docs/feeds/controller.md @@ -1,132 +1,93 @@ # Feed Controller -The `FeedController` class is responsible for managing and executing feed queries in a performant and organized manner. It handles the compilation of feed definitions into executable queries and manages the loading of events based on those queries. +The `FeedController` class manages feed execution with advanced loading strategies including pagination, windowing, and set operations. It compiles feeds into requests and handles event streaming with deduplication. + +## Types + +```typescript +export type FeedControllerOptions = FeedCompilerOptions & { + feed: Feed + tracker?: Tracker + onEvent?: (event: TrustedEvent) => void + onExhausted?: () => void + useWindowing?: boolean +} +``` + +## FeedController Class + +```typescript +export class FeedController { + compiler: FeedCompiler + + constructor(readonly options: FeedControllerOptions) + + // Get compiled request items (memoized) + getRequestItems(): Promise + + // Get loader function (memoized) + getLoader(): Promise<(limit: number) => Promise> + + // Load events with specified limit + load(limit: number): Promise +} +``` + +## Loading Strategies + +### Request-based Loading + +For feeds that can be compiled to `RequestItem[]`: +- **Pagination**: Automatically handles `since`/`until` windowing +- **Deduplication**: Prevents duplicate events across multiple requests +- **Exhaustion tracking**: Detects when all requests are exhausted + +### Set Operation Loading + +For feeds requiring special handling: + +#### Union Feeds +- Loads events from all sub-feeds in parallel +- Deduplicates events by ID across sub-feeds +- Signals exhaustion when all sub-feeds are exhausted + +#### Intersection Feeds +- Loads events from all sub-feeds in parallel +- Only emits events that appear in ALL sub-feeds +- Uses count tracking to determine intersection + +#### Difference Feeds +- Loads events from first feed (included) and remaining feeds (excluded) +- Emits events from first feed that don't appear in other feeds +- Maintains skip set for excluded events + +## Windowing Strategy + +When `useWindowing: true`: +- **Initial window**: Starts from recent events with estimated delta +- **Exponential backoff**: Increases window size when few events found +- **Timeline traversal**: Moves backward through time systematically +- **Performance optimization**: Gets recent events first + +Windowing is best used when you don't trust relays to give you results ordered by `created_at` descending. Windowing should not be used when treating relays as algorithm feeds. ## Usage ```typescript -import { FeedController } from '@welshman/feeds' +import { FeedController, makeAuthorFeed } from '@welshman/feeds' const controller = new FeedController({ - feed: yourFeedDefinition, - request: async ({ filters, relays, onEvent }) => { - // Your implementation for fetching events - }, - requestDVM: async ({ kind, tags, relays, onEvent }) => { - // Your implementation for DVM requests - }, - getPubkeysForScope: (scope) => { - // Return pubkeys for given scope - return ['pubkey1', 'pubkey2'] - }, - getPubkeysForWOTRange: (min, max) => { - // Return pubkeys within WOT range - return ['pubkey1', 'pubkey2'] - }, - onEvent: (event) => { - // Handle received events - }, - onExhausted: () => { - // Called when no more events are available - }, - useWindowing: true, // Optional: enable time-window based loading -}) -``` - -## API Reference - -### Constructor - -```typescript -constructor(options: FeedOptions) -``` - -Creates a new feed controller with the given options: -- `feed`: The feed definition to execute -- `request`: Function to fetch events from relays -- `requestDVM`: Function to fetch events from DVMs -- `getPubkeysForScope`: Function to get pubkeys for a scope -- `getPubkeysForWOTRange`: Function to get pubkeys within a WOT range -- `onEvent`: Optional callback for received events -- `onExhausted`: Optional callback when feed is exhausted -- `useWindowing`: Optional flag to enable time-window based loading - -### Methods - -#### `load(limit: number): Promise` -```typescript -const controller = new FeedController(options) -await controller.load(10) // Load 10 events -``` -Loads events from the feed up to the specified limit. - -#### `getLoader(): Promise<(limit: number) => Promise>` -Gets the loader function for this feed. Usually called internally by `load()`. - -#### `getRequestItems(): Promise` -Gets the compiled request items for this feed. Usually called internally. - -## Advanced Features - -### Time Windowing - -When `useWindowing` is enabled, the controller uses a time-based window approach to load events: - -```typescript -const controller = new FeedController({ - ...options, - useWindowing: true -}) -``` - -This is useful for: -- Loading recent events first -- Handling large datasets efficiently -- Progressive loading of historical data - - -## Examples - -### Basic Loading - -```typescript -const controller = new FeedController(options) -await controller.load(20) // Load 20 events -``` - -### Custom Loading Strategy - -```typescript -const controller = new FeedController({ - ...options, + feed: makeAuthorFeed("pubkey1", "pubkey2"), useWindowing: true, - onEvent: (event) => { - console.log('Received event:', event.id) - }, - onExhausted: () => { - console.log('No more events available') - } + onEvent: (event) => console.log('New event:', event.id), + onExhausted: () => console.log('No more events'), + getPubkeysForScope: (scope) => [...], + getPubkeysForWOTRange: (min, max) => [...] }) -// Load events in batches -async function loadAllEvents() { - while (!exhausted) { - await controller.load(10) - await new Promise(resolve => setTimeout(resolve, 1000)) - } -} -``` +// Load first batch of events +await controller.load(50) -### Error Handling - -```typescript -try { - await controller.load(10) -} catch (error) { - if (error.message.includes('relay')) { - // Handle relay errors - } else { - // Handle other errors - } -} +// Load more events +await controller.load(50) ``` diff --git a/docs/feeds/core.md b/docs/feeds/core.md index af00b09..600ed73 100644 --- a/docs/feeds/core.md +++ b/docs/feeds/core.md @@ -1,341 +1,97 @@ -# Feed Types and Core Definitions +# Feed Core Types -This module defines the core types and structures used to build Nostr feeds. -It provides a type-safe way to define complex feed compositions using various filtering mechanisms and set operations. +Core type definitions for the feed system, providing structured ways to query and filter Nostr events. ## Feed Types ```typescript -enum FeedType { - Address = "address", // Filter by event addresses - Author = "author", // Filter by author pubkeys - CreatedAt = "created_at", // Filter by timestamp - DVM = "dvm", // Data Vending Machine based feed - Difference = "difference", // Set difference operation - ID = "id", // Filter by event IDs - Intersection = "intersection", // Set intersection operation - Global = "global", // Global feed (no filters) - Kind = "kind", // Filter by event kinds - List = "list", // List-based feed - Label = "label", // Label-based feed - WOT = "wot", // Web of Trust based feed - Relay = "relay", // Relay-specific feed - Scope = "scope", // Scoped feed (followers, network) - Search = "search", // Search-based feed - Tag = "tag", // Filter by specific tags - Union = "union" // Set union operation +export enum FeedType { + Address = "address", + Author = "author", + CreatedAt = "created_at", + DVM = "dvm", + Difference = "difference", + ID = "id", + Intersection = "intersection", + Global = "global", + Kind = "kind", + List = "list", + Label = "label", + WOT = "wot", + Relay = "relay", + Scope = "scope", + Search = "search", + Tag = "tag", + Union = "union", +} + +export enum Scope { + Followers = "followers", + Follows = "follows", + Network = "network", + Self = "self", } ``` -## Scope Types +## Item Types ```typescript -enum Scope { - Followers = "followers", // People who follow the user - Follows = "follows", // People the user follows - Network = "network", // Extended network - Self = "self" // The signed in user -} -``` - -## Feed Definitions - -Each feed type has its own structure: - -### Basic Filter Feeds - -```typescript -type AddressFeed = [type: FeedType.Address, ...addresses: string[]] -type AuthorFeed = [type: FeedType.Author, ...pubkeys: string[]] -type IDFeed = [type: FeedType.ID, ...ids: string[]] -type KindFeed = [type: FeedType.Kind, ...kinds: number[]] -type TagFeed = [type: FeedType.Tag, key: string, ...values: string[]] -``` - -### Time-based Feeds - -```typescript -type CreatedAtItem = { - since?: number - until?: number - relative?: string[] // For relative time references -} -type CreatedAtFeed = [type: FeedType.CreatedAt, ...items: CreatedAtItem[]] -``` - -### Advanced Filter Feeds - -```typescript -// DVM-based feed -type DVMItem = { +export type DVMItem = { kind: number tags?: string[][] relays?: string[] mappings?: TagFeedMapping[] } -type DVMFeed = [type: FeedType.DVM, ...items: DVMItem[]] -// List-based feed -type ListItem = { +export type ListItem = { addresses: string[] mappings?: TagFeedMapping[] } -type ListFeed = [type: FeedType.List, ...items: ListItem[]] -// Label-based feed -type LabelItem = { +export type LabelItem = { relays?: string[] authors?: string[] [key: `#${string}`]: string[] mappings?: TagFeedMapping[] } -type LabelFeed = [type: FeedType.Label, ...items: LabelItem[]] -// Web of Trust feed -type WOTItem = { +export type WOTItem = { min?: number max?: number } -type WOTFeed = [type: FeedType.WOT, ...items: WOTItem[]] -``` -## Tag Feed Mapping +export type CreatedAtItem = { + since?: number + until?: number + relative?: string[] +} -`TagFeedMapping` is a mechanism to convert event tags into feed definitions. It's particularly useful when working with DVMs, Lists, and Labels where you want to interpret tags in a specific way. - -```typescript -type TagFeedMapping = [string, Feed] -``` - -### Usage -```typescript -// Example mappings -const mappings: TagFeedMapping[] = [ - // Convert 'p' tags into author feeds - ["p", [FeedType.Author]], - - // Convert 't' tags into hashtag filters - ["t", [FeedType.Tag, "#t"]], - - // Convert 'e' tags into event ID feeds - ["e", [FeedType.ID]], - - // Convert 'r' tags into relay feeds - ["r", [FeedType.Relay]] -] - -// Using mappings in a DVM feed -const dvmFeed: Feed = [ - FeedType.DVM, - { - kind: 5300, - mappings: mappings - } -] - -// Using mappings in a List feed -const listFeed: Feed = [ - FeedType.List, - { - addresses: ["list_id"], - mappings: mappings - } -] -``` - -### Default Mappings -The system comes with default mappings for common tags: -```typescript -const defaultTagFeedMappings: TagFeedMapping[] = [ - ["a", [FeedType.Address]], // Address references - ["e", [FeedType.ID]], // Event references - ["p", [FeedType.Author]], // Person/Pubkey references - ["r", [FeedType.Relay]], // Relay references - ["t", [FeedType.Tag, "#t"]], // Hashtags -] -``` - -## Set Operation Feeds - -### Union Feed -A Union feed combines multiple feeds with an OR operation. Events matching any of the constituent feeds will be included. - -```typescript -type UnionFeed = [type: FeedType.Union, ...feeds: Feed[]] - -// Example: Events from either Alice OR Bob -const unionFeed: UnionFeed = [ - FeedType.Union, - [FeedType.Author, "alice_pubkey"], - [FeedType.Author, "bob_pubkey"] -] - -// Example: Events from a list OR matching a search term -const complexUnion: UnionFeed = [ - FeedType.Union, - [FeedType.List, { addresses: ["trending_list"] }], - [FeedType.Search, "bitcoin"] -] -``` - -### Intersection Feed -An Intersection feed combines multiple feeds with an AND operation. Only events that match all constituent feeds will be included. - -```typescript -type IntersectionFeed = [type: FeedType.Intersection, ...feeds: Feed[]] - -// Example: Text notes (kind 1) from trusted authors -const intersectionFeed: IntersectionFeed = [ - FeedType.Intersection, - [FeedType.Kind, 1], - [FeedType.WOT, { min: 0.5 }] -] - -// Example: Recent posts from followed users -const timeAndScope: IntersectionFeed = [ - FeedType.Intersection, - [FeedType.CreatedAt, { since: Date.now() - 86400000 }], // Last 24h - [FeedType.Scope, Scope.Follows] -] -``` - -### Difference Feed -A Difference feed excludes events from the second feed from the first feed (NOT operation). - -```typescript -type DifferenceFeed = [type: FeedType.Difference, ...feeds: Feed[]] - -// Example: Posts from everyone except blocked users -const differenceFeed: DifferenceFeed = [ - FeedType.Difference, - [FeedType.Global], // All events - [FeedType.List, { addresses: ["blocked_users"] }] // Except from blocked users -] - -// Example: Posts from follows except reposts -const noReposts: DifferenceFeed = [ - FeedType.Difference, - [FeedType.Scope, Scope.Follows], - [FeedType.Kind, 6] // Kind 6 is repost -] -``` - -### Complex Combinations - -You can nest set operations to create sophisticated feed definitions: - -```typescript -// Posts that are either: -// - from trusted authors AND about bitcoin -// - OR from a curated list -const complexFeed: Feed = [ - FeedType.Union, - [ - FeedType.Intersection, - [FeedType.WOT, { min: 0.7 }], - [FeedType.Search, "bitcoin"] - ], - [FeedType.List, { addresses: ["curated_content"] }] -] - -// Posts that are: -// - from follows -// - AND (from the last 24h OR highly rated by DVMs) -// - AND NOT marked as sensitive content -const advancedFeed: Feed = [ - FeedType.Difference, - [ - FeedType.Intersection, - [FeedType.Scope, Scope.Follows], - [ - FeedType.Union, - [FeedType.CreatedAt, { since: Date.now() - 86400000 }], - [FeedType.DVM, { kind: 5300, pubkey: "rating_dvm" }] - ] - ], - [FeedType.Label, { authors: ["content_warning_dvm"] }] -] -``` - -## Feed Controller Options - -The `FeedOptions` interface defines the configuration required to execute a feed: - -```typescript -interface FeedOptions { - // The feed definition to execute - feed: Feed - - // Function to request events from relays - request: (opts: RequestOpts) => Promise - - // Function to request events from DVMs - requestDVM: (opts: DVMOpts) => Promise - - // Function to get pubkeys for a given scope - getPubkeysForScope: (scope: Scope) => string[] - - // Function to get pubkeys within a WOT range - getPubkeysForWOTRange: (minWOT: number, maxWOT: number) => string[] - - // Event handler - onEvent?: (event: TrustedEvent) => void - - // Called when feed is exhausted - onExhausted?: () => void - - // Enable time-window based loading - useWindowing?: boolean - - // Optional abort controller - abortController?: AbortController +export type RequestItem = { + relays?: string[] + filters?: Filter[] } ``` -## Examples +## Feed Definitions -### Simple Author Feed ```typescript -const authorFeed: Feed = [FeedType.Author, "pubkey1", "pubkey2"] -``` +export type AddressFeed = [type: FeedType.Address, ...addresses: string[]] +export type AuthorFeed = [type: FeedType.Author, ...pubkeys: string[]] +export type CreatedAtFeed = [type: FeedType.CreatedAt, ...items: CreatedAtItem[]] +export type DVMFeed = [type: FeedType.DVM, ...items: DVMItem[]] +export type DifferenceFeed = [type: FeedType.Difference, ...feeds: Feed[]] +export type IDFeed = [type: FeedType.ID, ...ids: string[]] +export type IntersectionFeed = [type: FeedType.Intersection, ...feeds: Feed[]] +export type GlobalFeed = [type: FeedType.Global, ...feeds: Feed[]] +export type KindFeed = [type: FeedType.Kind, ...kinds: number[]] +export type ListFeed = [type: FeedType.List, ...items: ListItem[]] +export type LabelFeed = [type: FeedType.Label, ...items: LabelItem[]] +export type WOTFeed = [type: FeedType.WOT, ...items: WOTItem[]] +export type RelayFeed = [type: FeedType.Relay, ...urls: string[]] +export type ScopeFeed = [type: FeedType.Scope, ...scopes: Scope[]] +export type SearchFeed = [type: FeedType.Search, ...searches: string[]] +export type TagFeed = [type: FeedType.Tag, key: string, ...values: string[]] +export type UnionFeed = [type: FeedType.Union, ...feeds: Feed[]] -### Time-filtered Feed -```typescript -const recentFeed: Feed = [ - FeedType.CreatedAt, - { - since: Date.now() - 24 * 60 * 60 * 1000, // Last 24 hours - relative: ["since"] - } -] +export type Feed = /* union of all feed types */ ``` - -### Complex Feed Composition -```typescript -const complexFeed: Feed = [ - FeedType.Intersection, - [FeedType.Kind, 1], // Text notes - [FeedType.WOT, { min: 0.5 }], // Trusted authors - [ - FeedType.Union, - [FeedType.Scope, Scope.Follows], // From follows - [FeedType.List, { addresses: ["list_id"] }] // Or from list - ] -] -``` - -### DVM Feed with Mappings -```typescript -const dvmFeed: Feed = [ - FeedType.DVM, - { - kind: 5300, - mappings: [ - ["p", [FeedType.Author]], // Map 'p' tags to authors - ["t", [FeedType.Tag, "#t"]] // Map 't' tags to hashtags - ] - } -] -``` - -This core module provides the foundation for building complex, type-safe feed definitions that can be executed by the [feed controller](/feeds/controller). diff --git a/docs/feeds/utils.md b/docs/feeds/utils.md index 7dc45af..2ff1e07 100644 --- a/docs/feeds/utils.md +++ b/docs/feeds/utils.md @@ -45,7 +45,8 @@ const feed: Feed = makeDVMFeed({ kind: 5300 }) if (isDVMFeed(feed)) { // feed is now typed as DVMFeed - const [kind] = feed.slice(1) + const [item] = getFeedArgs(feed) + const kind = item.kind } if (hasSubFeeds(feed)) { @@ -122,6 +123,24 @@ walkFeed(feed, (node) => { }) ``` +## Feed Simplification + +Flatten nested feeds of the same type: + +```typescript +// Simplifies nested feeds of the same type +export declare const simplifyFeed: (feed: Feed) => Feed + +// Example: flatten nested union feeds +const nested = makeUnionFeed( + makeAuthorFeed("pubkey1"), + makeUnionFeed(makeKindFeed(1), makeKindFeed(6)) +) + +const simplified = simplifyFeed(nested) +// Result: [FeedType.Union, [FeedType.Author, "pubkey1"], [FeedType.Kind, 1], [FeedType.Kind, 6]] +``` + ## Type Extraction Get typed arguments from feeds: