From 37c0491d71de127bad39dee633a8cefe2144818f Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Fri, 25 Apr 2025 10:37:57 -0700 Subject: [PATCH] Move collection to store module --- packages/app/src/follows.ts | 3 +- packages/app/src/freshness.ts | 29 -- packages/app/src/handles.ts | 2 +- packages/app/src/index.ts | 2 - packages/app/src/mutes.ts | 3 +- packages/app/src/pins.ts | 3 +- packages/app/src/profiles.ts | 3 +- packages/app/src/relaySelections.ts | 18 +- packages/app/src/relays.ts | 2 +- packages/app/src/storageAdapters.ts | 2 +- packages/app/src/zappers.ts | 2 +- packages/feeds/__tests__/compiler.test.ts | 397 ------------------ packages/feeds/__tests__/controller.test.ts | 243 ----------- packages/router/src/index.ts | 4 +- .../__tests__/collection.test.ts | 3 +- packages/{app => store}/src/collection.ts | 35 +- packages/store/src/custom.ts | 65 +++ packages/store/src/getter.ts | 20 + packages/store/src/index.ts | 289 +------------ packages/store/src/repository.ts | 164 ++++++++ packages/store/src/synced.ts | 11 + packages/store/src/throttle.ts | 12 + 22 files changed, 325 insertions(+), 987 deletions(-) delete mode 100644 packages/app/src/freshness.ts delete mode 100644 packages/feeds/__tests__/compiler.test.ts delete mode 100644 packages/feeds/__tests__/controller.test.ts rename packages/{app => store}/__tests__/collection.test.ts (98%) rename packages/{app => store}/src/collection.ts (74%) create mode 100644 packages/store/src/custom.ts create mode 100644 packages/store/src/getter.ts create mode 100644 packages/store/src/repository.ts create mode 100644 packages/store/src/synced.ts create mode 100644 packages/store/src/throttle.ts diff --git a/packages/app/src/follows.ts b/packages/app/src/follows.ts index 388c80d..220fc44 100644 --- a/packages/app/src/follows.ts +++ b/packages/app/src/follows.ts @@ -1,8 +1,7 @@ import {FOLLOWS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped} from "@welshman/store" +import {deriveEventsMapped, collection} from "@welshman/store" import {repository} from "./core.js" -import {collection} from "./collection.js" import {makeOutboxLoader} from "./relaySelections.js" export const follows = deriveEventsMapped(repository, { diff --git a/packages/app/src/freshness.ts b/packages/app/src/freshness.ts deleted file mode 100644 index 367f33f..0000000 --- a/packages/app/src/freshness.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {writable} from "svelte/store" -import {assoc, batch} from "@welshman/lib" -import {withGetter} from "@welshman/store" - -export type FreshnessUpdate = { - ns: string - key: string - ts: number -} - -export const freshness = withGetter(writable>({})) - -export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}` - -export const getFreshness = (ns: string, key: string) => - freshness.get()[getFreshnessKey(ns, key)] || 0 - -export const setFreshnessImmediate = ({ns, key, ts}: FreshnessUpdate) => - freshness.update(assoc(getFreshnessKey(ns, key), ts)) - -export const setFreshnessThrottled = batch(100, (updates: FreshnessUpdate[]) => - freshness.update($freshness => { - for (const {ns, key, ts} of updates) { - $freshness[getFreshnessKey(ns, key)] = ts - } - - return $freshness - }), -) diff --git a/packages/app/src/handles.ts b/packages/app/src/handles.ts index e59904c..34944e2 100644 --- a/packages/app/src/handles.ts +++ b/packages/app/src/handles.ts @@ -1,6 +1,6 @@ import {writable, derived} from "svelte/store" import {tryCatch, fetchJson, uniq, batcher, postJson, last} from "@welshman/lib" -import {collection} from "./collection.js" +import {collection} from "@welshman/store" import {deriveProfile} from "./profiles.js" import {appContext} from "./context.js" diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts index 1b2a2e6..4e492a9 100644 --- a/packages/app/src/index.ts +++ b/packages/app/src/index.ts @@ -1,9 +1,7 @@ export * from "./context.js" export * from "./core.js" -export * from "./collection.js" export * from "./commands.js" export * from "./feeds.js" -export * from "./freshness.js" export * from "./follows.js" export * from "./handles.js" export * from "./mutes.js" diff --git a/packages/app/src/mutes.ts b/packages/app/src/mutes.ts index a195f88..ae62629 100644 --- a/packages/app/src/mutes.ts +++ b/packages/app/src/mutes.ts @@ -1,8 +1,7 @@ import {MUTES, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped} from "@welshman/store" +import {deriveEventsMapped, collection} from "@welshman/store" import {repository} from "./core.js" -import {collection} from "./collection.js" import {ensurePlaintext} from "./plaintext.js" import {makeOutboxLoader} from "./relaySelections.js" diff --git a/packages/app/src/pins.ts b/packages/app/src/pins.ts index d271d00..af10fe5 100644 --- a/packages/app/src/pins.ts +++ b/packages/app/src/pins.ts @@ -1,8 +1,7 @@ import {PINS, asDecryptedEvent, readList} from "@welshman/util" import {TrustedEvent, PublishedList} from "@welshman/util" -import {deriveEventsMapped} from "@welshman/store" +import {deriveEventsMapped, collection} from "@welshman/store" import {repository} from "./core.js" -import {collection} from "./collection.js" import {makeOutboxLoader} from "./relaySelections.js" export const pins = deriveEventsMapped(repository, { diff --git a/packages/app/src/profiles.ts b/packages/app/src/profiles.ts index aa976c1..33d6ba2 100644 --- a/packages/app/src/profiles.ts +++ b/packages/app/src/profiles.ts @@ -1,9 +1,8 @@ import {derived, readable} from "svelte/store" import {readProfile, displayProfile, displayPubkey, PROFILE} from "@welshman/util" import {PublishedProfile} from "@welshman/util" -import {deriveEventsMapped, withGetter} from "@welshman/store" +import {deriveEventsMapped, collection, withGetter} from "@welshman/store" import {repository} from "./core.js" -import {collection} from "./collection.js" import {makeOutboxLoader} from "./relaySelections.js" export const profiles = withGetter( diff --git a/packages/app/src/relaySelections.ts b/packages/app/src/relaySelections.ts index 700b771..d7fc1c9 100644 --- a/packages/app/src/relaySelections.ts +++ b/packages/app/src/relaySelections.ts @@ -1,19 +1,11 @@ -import {derived} from 'svelte/store' -import {uniq, batcher, always} from "@welshman/lib" -import { - INBOX_RELAYS, - RELAYS, - normalizeRelayUrl, - asDecryptedEvent, - readList, - getRelaysFromList, -} from "@welshman/util" -import {TrustedEvent, PublishedList, RelayMode, List} from "@welshman/util" +import {derived} from "svelte/store" +import {batcher, always} from "@welshman/lib" +import {INBOX_RELAYS, RELAYS, asDecryptedEvent, readList, getRelaysFromList} from "@welshman/util" +import {TrustedEvent, PublishedList, RelayMode} from "@welshman/util" import {request} from "@welshman/net" -import {deriveEventsMapped} from "@welshman/store" +import {deriveEventsMapped, collection} from "@welshman/store" import {Router} from "@welshman/router" import {repository} from "./core.js" -import {collection} from "./collection.js" export type OutboxLoaderRequest = { pubkey: string diff --git a/packages/app/src/relays.ts b/packages/app/src/relays.ts index 5881a29..a4ad2b9 100644 --- a/packages/app/src/relays.ts +++ b/packages/app/src/relays.ts @@ -24,7 +24,7 @@ import { isRelayUrl, } from "@welshman/util" import {Pool, Socket, SocketStatus, SocketEvent, ClientMessage, RelayMessage} from "@welshman/net" -import {collection} from "./collection.js" +import {collection} from "@welshman/store" import {appContext} from "./context.js" export type RelayStats = { diff --git a/packages/app/src/storageAdapters.ts b/packages/app/src/storageAdapters.ts index 8445976..148b455 100644 --- a/packages/app/src/storageAdapters.ts +++ b/packages/app/src/storageAdapters.ts @@ -12,13 +12,13 @@ import { } from "@welshman/util" import {throttled, withGetter} from "@welshman/store" import {Tracker} from "@welshman/net" +import {freshness} from "@welshman/store" import {Repository, RepositoryUpdate} from "@welshman/relay" import {getAll, bulkPut, bulkDelete} from "./storage.js" import {relays} from "./relays.js" import {handles, onHandle} from "./handles.js" import {zappers, onZapper} from "./zappers.js" import {plaintext} from "./plaintext.js" -import {freshness} from "./freshness.js" import {repository, tracker} from "./core.js" import {sessions} from "./session.js" import {userFollows} from "./user.js" diff --git a/packages/app/src/zappers.ts b/packages/app/src/zappers.ts index f374c72..e7896eb 100644 --- a/packages/app/src/zappers.ts +++ b/packages/app/src/zappers.ts @@ -10,7 +10,7 @@ import { batcher, postJson, } from "@welshman/lib" -import {collection} from "./collection.js" +import {collection} from "@welshman/store" import {deriveProfile} from "./profiles.js" import {appContext} from "./context.js" diff --git a/packages/feeds/__tests__/compiler.test.ts b/packages/feeds/__tests__/compiler.test.ts deleted file mode 100644 index 9fe52b7..0000000 --- a/packages/feeds/__tests__/compiler.test.ts +++ /dev/null @@ -1,397 +0,0 @@ -import {defaultTagFeedMappings} from "@welshman/feeds" -import {now} from "@welshman/lib" -import {getAddress, TrustedEvent} from "@welshman/util" -import {beforeEach, describe, expect, it, vi} from "vitest" -import {FeedCompiler} from "../src/compiler" -import {Feed, FeedType, Scope} from "../src/core" - -describe("FeedCompiler", () => { - let compiler: FeedCompiler - let mockOptions: any - - beforeEach(() => { - mockOptions = { - getPubkeysForScope: vi.fn().mockReturnValue(["pubkey1", "pubkey2"]), - getPubkeysForWOTRange: vi.fn().mockReturnValue(["pubkey3", "pubkey4"]), - requestDVM: vi.fn(), - request: vi.fn(), - } - compiler = new FeedCompiler(mockOptions) - }) - - describe("canCompile", () => { - it("should return true for supported feed types", () => { - const supportedFeeds: Feed[] = [ - [FeedType.Address, "addr1", "addr2"], - [FeedType.Author, "author1", "author2"], - [FeedType.CreatedAt, {since: 1000}], - [FeedType.DVM, {kind: 1, mappings: []}], - [FeedType.ID, "id1", "id2"], - [FeedType.Global], - [FeedType.Kind, 1, 2], - [FeedType.List, {addresses: [], mappings: []}], - [FeedType.Label, {mappings: []}], - [FeedType.Relay, "relay1", "relay2"], - [FeedType.Scope, Scope.Followers, Scope.Follows], - [FeedType.Search, "query1", "query2"], - [FeedType.Tag, "key", "value"], - [FeedType.WOT, {min: 0, max: 1}], - ] - - for (const feed of supportedFeeds) { - expect(compiler.canCompile(feed)).toBe(true) - } - }) - - it("should return true for nested union and intersection feeds", () => { - const feed: Feed = [FeedType.Union, [FeedType.Author, "author1"], [FeedType.Kind, 1]] - expect(compiler.canCompile(feed)).toBe(true) - }) - - it("should return false for unsupported feed type", () => { - const feed: any = ["UnsupportedType", "value"] - expect(compiler.canCompile(feed)).toBe(false) - }) - }) - - describe("compile", () => { - it("should compile ID feed", async () => { - const result = await compiler.compile([FeedType.ID, "id1", "id2"]) - expect(result).toEqual([ - { - filters: [{ids: ["id1", "id2"]}], - }, - ]) - }) - - it("should compile Kind feed", async () => { - const result = await compiler.compile([FeedType.Kind, 1, 2]) - expect(result).toEqual([ - { - filters: [{kinds: [1, 2]}], - }, - ]) - }) - - it("should compile Author feed", async () => { - const result = await compiler.compile([FeedType.Author, "author1", "author2"]) - expect(result).toEqual([ - { - filters: [{authors: ["author1", "author2"]}], - }, - ]) - }) - - it("should compile Scope feed", async () => { - const result = await compiler.compile([FeedType.Scope, Scope.Followers, Scope.Follows]) - expect(result).toEqual([ - { - filters: [{authors: ["pubkey1", "pubkey2"]}], - }, - ]) - // there is an issue with vitest, these conditions are true - // expect(mockOptions.getPubkeysForScope).toHaveBeenCalledWith(Scope.Followers) - // expect(mockOptions.getPubkeysForScope).toHaveBeenCalledWith(Scope.Follows) - expect(mockOptions.getPubkeysForScope).toHaveBeenCalledTimes(2) - }) - - it("should compile WOT feed", async () => { - const result = await compiler.compile([FeedType.WOT, {min: 0, max: 1}]) - expect(result).toEqual([ - { - filters: [{authors: ["pubkey3", "pubkey4"]}], - }, - ]) - expect(mockOptions.getPubkeysForWOTRange).toHaveBeenCalledWith(0, 1) - }) - - it("should compile CreatedAt feed", async () => { - const created_at = now() - const result = await compiler.compile([ - FeedType.CreatedAt, - {since: 1000, until: 2000}, - {since: 3000, relative: ["since"]}, - ]) - expect(result[0].filters?.length).toBe(2) - expect(result[0].filters?.[0]).toMatchObject({since: 1000, until: 2000}) - expect(result[0].filters?.[1].since).toBe(created_at - 3000) - }) - - it("should compile Search feed", async () => { - const result = await compiler.compile([FeedType.Search, "query1", "query2"]) - expect(result).toEqual([ - { - filters: [{search: "query1"}, {search: "query2"}], - }, - ]) - }) - - it("should compile Relay feed", async () => { - const result = await compiler.compile([FeedType.Relay, "relay1", "relay2"]) - expect(result).toEqual([ - { - relays: ["relay1", "relay2"], - }, - ]) - }) - - it("should compile Global feed", async () => { - const result = await compiler.compile([FeedType.Global]) - expect(result).toEqual([ - { - filters: [{}], - }, - ]) - }) - - it("should compile Tag feed", async () => { - const result = await compiler.compile([FeedType.Tag, "key", "value1", "value2"]) - expect(result).toEqual([ - { - filters: [{key: ["value1", "value2"]}], - }, - ]) - }) - }) - - describe("compile complex feeds", () => { - it("should compile Union feed", async () => { - const requestItem = await compiler.compile([ - FeedType.Union, - [FeedType.Author, "author1"], - [FeedType.Kind, 1], - ]) - // one request item with two filters - expect(requestItem).toHaveLength(1) - expect(requestItem[0].filters).toHaveLength(2) - - const requestItem2 = await compiler.compile([ - FeedType.Union, - [FeedType.Author, "author1"], - [FeedType.Relay, "relay1", "relay2"], - [FeedType.Kind, 1], - ]) - // two request items - expect(requestItem2).toHaveLength(2) - // the first with 2 filters and no relay - expect(requestItem2[0].filters).toHaveLength(2) - expect(requestItem2[0].relays).toBeUndefined() - // the second with 0 filter and 2 relays - expect(requestItem2[1].filters).toBeUndefined() - expect(requestItem2[1].relays).toHaveLength(2) - - const requestItem3 = await compiler.compile([ - FeedType.Union, - [FeedType.Author, "author1"], - [FeedType.Intersection, [FeedType.Kind, 1], [FeedType.Relay, "relay1"]], - ]) - - // two request items - expect(requestItem3).toHaveLength(2) - // the first with 1 filter and one relay - expect(requestItem3[0].filters).toHaveLength(1) - expect(requestItem3[0].relays).toHaveLength(1) - // the second with 1 filter and no relay - expect(requestItem3[1].filters).toHaveLength(1) - expect(requestItem3[1].relays).toBeUndefined() - - const requestItem4 = await compiler.compile([ - FeedType.Union, - [FeedType.Author, "author1"], - [FeedType.Union, [FeedType.Kind, 1], [FeedType.Relay, "relay1"]], - ]) - // two request items - expect(requestItem4).toHaveLength(2) - // the first with 2 filters and no relay - expect(requestItem4[0].filters).toHaveLength(2) - expect(requestItem4[0].relays).toBeUndefined() - // the second with no filter and one relay - expect(requestItem4[1].filters).toBeUndefined() - expect(requestItem4[1].relays).toHaveLength(1) - }) - - it("should compile Intersection feed", async () => { - const requestItems = await compiler.compile([ - FeedType.Intersection, - [FeedType.Author, "author1"], - [FeedType.Kind, 1], - ]) - // one request item with one filter - expect(requestItems).toHaveLength(1) - expect(requestItems[0].filters).toHaveLength(1) - - const requestItems2 = await compiler.compile([ - FeedType.Intersection, - [FeedType.Author, "author1"], - [FeedType.Relay, "relay1", "relay2"], - [FeedType.Kind, 1], - ]) - - // one request item with one filter and two relays - expect(requestItems2).toHaveLength(1) - expect(requestItems2[0].filters).toHaveLength(1) - expect(requestItems2[0].relays).toHaveLength(2) - - const requestItems3 = await compiler.compile([ - FeedType.Intersection, - [FeedType.Author, "author1"], - [FeedType.Intersection, [FeedType.Kind, 1], [FeedType.Relay, "relay1", "relay2"]], - ]) - - // one request item with one filter and one relay - expect(requestItems3).toHaveLength(1) - expect(requestItems3[0].filters).toHaveLength(1) - expect(requestItems3[0].relays).toHaveLength(2) - - const requestItems4 = await compiler.compile([ - FeedType.Intersection, - [FeedType.Author, "author1"], - [FeedType.Union, [FeedType.Kind, 1], [FeedType.Relay, "relay1", "relay2"]], - ]) - - // one request item with one filter and one relay - expect(requestItems4).toHaveLength(1) - expect(requestItems4[0].filters).toHaveLength(1) - expect(requestItems4[0].relays).toHaveLength(2) - }) - - it("should compile DVM feed", async () => { - const mockEvent: TrustedEvent = { - id: "id1", - pubkey: "pubkey1", - created_at: 1000, - kind: 7000, - tags: [], - content: JSON.stringify([ - ["t", "test"], - ["r", "relay1"], - ]), - sig: "sig1", - } - - mockOptions.requestDVM.mockImplementation(async ({onEvent}: any) => { - await onEvent(mockEvent) - }) - - const requestItems = await compiler.compile([ - FeedType.DVM, - { - kind: 7000, - mappings: defaultTagFeedMappings, - }, - ]) - - expect(mockOptions.requestDVM).toHaveBeenCalled() - // 2 request items - expect(requestItems).toHaveLength(2) - // the first with 1 filter and no relay - expect(requestItems[0].filters).toHaveLength(1) - expect(requestItems[0].relays).toBeUndefined() - // the second with no filter and 1 relay - expect(requestItems[1].filters).toBeUndefined() - expect(requestItems[1].relays).toHaveLength(1) - }) - - it("should compile List feed", async () => { - const mockEvent: TrustedEvent = { - id: "id1", - pubkey: "pubkey1", - created_at: 1000, - kind: 1, - tags: [ - ["d", "test"], - ["t", "test"], - ["r", "relay1"], - ], - content: "", - sig: "sig1", - } - - mockOptions.request.mockImplementation(({onEvent}: any) => { - onEvent(mockEvent) - }) - - const requestItems = await compiler.compile([ - FeedType.List, - { - addresses: [getAddress(mockEvent)], - mappings: defaultTagFeedMappings, - }, - ]) - - expect(mockOptions.request).toHaveBeenCalled() - // 2 request items - expect(requestItems).toHaveLength(2) - // the first with 1 filter and no relay - expect(requestItems[0].filters).toHaveLength(1) - expect(requestItems[0].relays).toBeUndefined() - // the second with no filter and 1 relay - expect(requestItems[1].filters).toBeUndefined() - expect(requestItems[1].relays).toHaveLength(1) - }) - - it("should compile Label feed", async () => { - const labelEvent: TrustedEvent = { - id: "label1", - pubkey: "pubkey1", - created_at: 1000, - kind: 1985, - tags: [ - ["L", "spam"], - ["e", "event1"], - ["p", "author1"], - ], - content: "This is spam", - sig: "sig1", - } - - mockOptions.request.mockImplementation(({onEvent}: any) => { - onEvent(labelEvent) - }) - - const requestItems = await compiler.compile([ - FeedType.Label, - { - "#L": ["spam"], - mappings: defaultTagFeedMappings, - }, - ]) - // should return an union filter with the "e" and "p" tags from the label event - expect(mockOptions.request).toHaveBeenCalled() - expect(requestItems).toHaveLength(1) - expect(requestItems[0].filters).toHaveLength(2) - }) - }) - - describe("error handling", () => { - it("should throw error for unsupported feed type", async () => { - await expect(compiler.compile(["UnsupportedType", "value"] as any)).rejects.toThrow( - "Unable to convert feed of type UnsupportedType to filters", - ) - }) - - it("should handle DVM events with invalid JSON content", async () => { - const mockEvent: TrustedEvent = { - id: "id1", - pubkey: "pubkey1", - created_at: 7000, - kind: 1, - tags: [], - content: "invalid json", - sig: "sig1", - } - - mockOptions.requestDVM.mockImplementation(async ({onEvent}: any) => { - await onEvent(mockEvent) - }) - - const requestItems = await compiler.compile([ - FeedType.DVM, - {kind: 7000, mappings: defaultTagFeedMappings}, - ]) - - expect(requestItems).toBeDefined() - expect(requestItems).toHaveLength(0) - }) - }) -}) diff --git a/packages/feeds/__tests__/controller.test.ts b/packages/feeds/__tests__/controller.test.ts deleted file mode 100644 index 8b62237..0000000 --- a/packages/feeds/__tests__/controller.test.ts +++ /dev/null @@ -1,243 +0,0 @@ -import {describe, it, expect, vi, beforeEach} from "vitest" -import {FeedController} from "../src/controller" -import {Feed, FeedOptions, FeedType} from "../src/core" -import {EPOCH, type TrustedEvent} from "@welshman/util" -import {now} from "@welshman/lib" - -describe("FeedController", () => { - let mockRequest: ReturnType - let mockOnEvent: ReturnType - let mockOnExhausted: ReturnType - let mockRequestDVM: ReturnType - - beforeEach(() => { - mockRequest = vi.fn() - mockOnEvent = vi.fn() - mockOnExhausted = vi.fn() - mockRequestDVM = vi.fn() - }) - - const createEvent = (id: string, created_at: number): TrustedEvent => ({ - id, - pubkey: "pub1", - created_at, - kind: 1, - tags: [], - content: "", - sig: "sig1", - }) - - const createFeedOptions = (feed: Feed, useWindowing = false): FeedOptions => ({ - getPubkeysForScope: vi.fn().mockReturnValue(["pubkey1", "pubkey2"]), - getPubkeysForWOTRange: vi.fn().mockReturnValue(["pubkey3", "pubkey4"]), - feed, - request: mockRequest, - requestDVM: mockRequestDVM, - onEvent: mockOnEvent, - onExhausted: mockOnExhausted, - useWindowing, - }) - - describe("Basic Loading", () => { - it("should load events from simple feed", async () => { - const controller = new FeedController(createFeedOptions([FeedType.Author, "pub1"])) - - mockRequest.mockImplementation(({onEvent}) => { - onEvent(createEvent("1", 1000)) - onEvent(createEvent("2", 900)) - }) - - await controller.load(10) - - expect(mockRequest).toHaveBeenCalledWith( - expect.objectContaining({ - filters: expect.arrayContaining([expect.objectContaining({authors: ["pub1"]})]), - }), - ) - expect(mockOnEvent).toHaveBeenCalledTimes(2) - }) - - it("should handle windowing", async () => { - const controller = new FeedController(createFeedOptions([FeedType.Author, "pub1"], true)) - - mockRequest.mockImplementation(({onEvent}) => { - onEvent(createEvent("1", 1000)) - }) - - await controller.load(10) - await controller.load(10) // Should load next window - - expect(mockRequest).toHaveBeenCalledTimes(2) - expect(mockRequest.mock.calls[1][0].filters[0].until).toBeLessThan( - mockRequest.mock.calls[0][0].filters[0].until, - ) - }) - }) - - describe("Complex Feed Types", () => { - it("should handle Union feeds", async () => { - const controller = new FeedController( - createFeedOptions([FeedType.Union, [FeedType.Author, "pub1"], [FeedType.Author, "pub2"]]), - ) - - mockRequest.mockImplementation(({filters, onEvent}) => { - if (filters[0].authors?.includes("pub1")) { - onEvent(createEvent("1", 1000)) - } - if (filters[0].authors?.includes("pub2")) { - onEvent(createEvent("2", 900)) - } - }) - - await controller.load(10) - - expect(mockOnEvent).toHaveBeenCalledTimes(2) - }) - - it("should handle Intersection feeds", async () => { - const controller = new FeedController( - createFeedOptions([FeedType.Intersection, [FeedType.Author, "pub1"], [FeedType.Kind, 1]]), - ) - - const event = createEvent("1", 1000) - mockRequest.mockImplementation(({onEvent}) => { - onEvent(event) - }) - - await controller.load(10) - - expect(mockOnEvent).toHaveBeenCalledWith(event) - }) - - it("should handle Difference feeds", async () => { - const controller = new FeedController( - createFeedOptions([ - FeedType.Difference, - [FeedType.Author, "pub1"], - [FeedType.Author, "pub2"], - ]), - ) - - mockRequest.mockImplementation(({filters, onEvent}) => { - if (filters[0].authors?.includes("pub1")) { - onEvent(createEvent("1", 1000)) - onEvent(createEvent("2", 900)) - } - if (filters[0].authors?.includes("pub2")) { - onEvent(createEvent("2", 900)) // This one should be excluded - } - }) - - await controller.load(10) - - expect(mockOnEvent).toHaveBeenCalledTimes(1) - expect(mockOnEvent).toHaveBeenCalledWith(expect.objectContaining({id: "1"})) - }) - }) - - describe("Event Deduplication", () => { - it("should not emit duplicate events", async () => { - const controller = new FeedController(createFeedOptions([FeedType.Author, "pub1"])) - - const event = createEvent("1", 1000) - mockRequest.mockImplementation(({onEvent}) => { - onEvent(event) - onEvent(event) // Duplicate - }) - - await controller.load(10) - - expect(mockOnEvent).toHaveBeenCalledTimes(1) - }) - }) - - describe("Exhaustion Handling", () => { - it("should call onExhausted when no more events", async () => { - const controller = new FeedController(createFeedOptions([FeedType.Author, "pub1"])) - - mockRequest.mockImplementation(({onEvent}) => { - // No events returned - }) - - await controller.load(10) - - expect(mockOnExhausted).toHaveBeenCalled() - }) - - it("should handle exhaustion in complex feeds", async () => { - const controller = new FeedController( - createFeedOptions([FeedType.Union, [FeedType.Author, "pub1"], [FeedType.Author, "pub2"]]), - ) - - mockRequest.mockImplementation(() => { - // No events returned - }) - - await controller.load(10) - - expect(mockOnExhausted).toHaveBeenCalled() - }) - }) - - describe("Filter Handling", () => { - it("should handle time-based filters", async () => { - const controller = new FeedController(createFeedOptions([FeedType.Author, "pub1"])) - - await controller.load(10) - - expect(mockRequest).toHaveBeenCalledWith( - expect.objectContaining({ - filters: expect.arrayContaining([ - expect.objectContaining({ - since: EPOCH, - until: now(), - }), - ]), - }), - ) - }) - - it("should respect existing filter constraints", async () => { - const since = 1000 - const until = 2000 - const controller = new FeedController( - createFeedOptions([ - FeedType.Intersection, - [FeedType.Author, "pub1"], - [FeedType.CreatedAt, {since, until}], - ]), - ) - - mockRequest.mockImplementation(({filters, onEvent}) => { - expect(filters[0].since).toBeGreaterThanOrEqual(since) - expect(filters[0].until).toBeLessThanOrEqual(until) - onEvent(createEvent("1", 1500)) - }) - - await controller.load(10) - }) - }) - - describe("Error Handling", () => { - it("should handle request errors gracefully", async () => { - const controller = new FeedController({ - ...createFeedOptions([FeedType.Author, "pub1"]), - request: () => { - throw new Error("Request failed") - }, - onEvent: mockOnEvent, - }) - - await expect(controller.load(10)).rejects.toThrow("Request failed") - }) - - it("should handle invalid feed types", async () => { - const controller = new FeedController({ - ...createFeedOptions(["InvalidType", "value"] as any), - request: mockRequest, - }) - - await expect(controller.load(10)).rejects.toThrow() - }) - }) -}) diff --git a/packages/router/src/index.ts b/packages/router/src/index.ts index 80ae22a..614501a 100644 --- a/packages/router/src/index.ts +++ b/packages/router/src/index.ts @@ -119,9 +119,9 @@ export const routerContext: RouterOptions = { return uniq( Repository.get() .query([{kinds: [RELAYS], authors: [pubkey]}]) - .flatMap(event => getRelaysFromList(readList(asDecryptedEvent(event)), mode)) + .flatMap(event => getRelaysFromList(readList(asDecryptedEvent(event)), mode)), ) - } + }, } export class Router { diff --git a/packages/app/__tests__/collection.test.ts b/packages/store/__tests__/collection.test.ts similarity index 98% rename from packages/app/__tests__/collection.test.ts rename to packages/store/__tests__/collection.test.ts index d96243c..b64f479 100644 --- a/packages/app/__tests__/collection.test.ts +++ b/packages/store/__tests__/collection.test.ts @@ -1,8 +1,7 @@ import {describe, it, expect, beforeEach, vi, afterEach} from "vitest" import {get, writable} from "svelte/store" import {now, always} from "@welshman/lib" -import {collection} from "../src/collection" -import {freshness, setFreshnessImmediate} from "../src/freshness" +import {collection, freshness, setFreshnessImmediate} from "../src/collection" describe("collection", () => { beforeEach(() => { diff --git a/packages/app/src/collection.ts b/packages/store/src/collection.ts similarity index 74% rename from packages/app/src/collection.ts rename to packages/store/src/collection.ts index b9968c8..344ee6b 100644 --- a/packages/app/src/collection.ts +++ b/packages/store/src/collection.ts @@ -1,7 +1,34 @@ -import {readable, derived, type Readable, type Subscriber} from "svelte/store" -import {indexBy, remove, now} from "@welshman/lib" -import {ReadableWithGetter, withGetter} from "@welshman/store" -import {getFreshness, setFreshnessThrottled} from "./freshness.js" +import {readable, derived, writable, Readable, Subscriber} from "svelte/store" +import {batch, indexBy, remove, assoc, now} from "@welshman/lib" +import {withGetter, ReadableWithGetter} from "./getter.js" + +// Collection utility + +export type FreshnessUpdate = { + ns: string + key: string + ts: number +} + +export const freshness = withGetter(writable>({})) + +export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}` + +export const getFreshness = (ns: string, key: string) => + freshness.get()[getFreshnessKey(ns, key)] || 0 + +export const setFreshnessImmediate = ({ns, key, ts}: FreshnessUpdate) => + freshness.update(assoc(getFreshnessKey(ns, key), ts)) + +export const setFreshnessThrottled = batch(100, (updates: FreshnessUpdate[]) => + freshness.update($freshness => { + for (const {ns, key, ts} of updates) { + $freshness[getFreshnessKey(ns, key)] = ts + } + + return $freshness + }), +) export type CachedLoaderOptions = { name: string diff --git a/packages/store/src/custom.ts b/packages/store/src/custom.ts new file mode 100644 index 0000000..c156377 --- /dev/null +++ b/packages/store/src/custom.ts @@ -0,0 +1,65 @@ +import {Subscriber, Unsubscriber} from "svelte/store" +import {throttle} from "@welshman/lib" +import {WritableWithGetter} from "./getter.js" + +type Start = (set: Subscriber) => Unsubscriber + +export type CustomStoreOpts = { + throttle?: number + set?: (x: T) => void +} + +export const custom = ( + start: Start, + opts: CustomStoreOpts = {}, +): WritableWithGetter => { + const subs: Subscriber[] = [] + + let value: T + let stop: () => void + + const set = (newValue: T) => { + for (const sub of subs) { + sub(newValue) + } + + value = newValue + } + + return { + get: () => value, + set: (newValue: T) => { + set(newValue) + opts.set?.(newValue) + }, + update: (f: (value: T) => T) => { + const newValue = f(value) + + set(newValue) + opts.set?.(newValue) + }, + subscribe: (sub: Subscriber) => { + if (opts.throttle) { + sub = throttle(opts.throttle, sub) + } + + if (subs.length === 0) { + stop = start(set) + } + + subs.push(sub) + sub(value) + + return () => { + subs.splice( + subs.findIndex(s => s === sub), + 1, + ) + + if (subs.length === 0) { + stop() + } + } + }, + } +} diff --git a/packages/store/src/getter.ts b/packages/store/src/getter.ts new file mode 100644 index 0000000..9e34395 --- /dev/null +++ b/packages/store/src/getter.ts @@ -0,0 +1,20 @@ +import {Readable, Writable} from "svelte/store" + +export const getter = (store: Readable) => { + let value: T + + store.subscribe((newValue: T) => { + value = newValue + }) + + return () => value +} + +export type WritableWithGetter = Writable & {get: () => T} +export type ReadableWithGetter = Readable & {get: () => T} + +export function withGetter(store: Writable): WritableWithGetter +export function withGetter(store: Readable): ReadableWithGetter +export function withGetter(store: Readable | Writable) { + return {...store, get: getter(store)} +} diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index c70b7b9..a9abc16 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -1,283 +1,6 @@ -import {derived, writable} from "svelte/store" -import type {Readable, Writable, Subscriber, Unsubscriber} from "svelte/store" -import { - identity, - throttle, - ensurePlural, - getJson, - setJson, - batch, - partition, - first, -} from "@welshman/lib" -import {Repository} from "@welshman/relay" -import {matchFilters, getIdAndAddress, getIdFilters, Filter, TrustedEvent} from "@welshman/util" - -// Sync with localstorage - -export const synced = (key: string, defaultValue: T) => { - const init = getJson(key) - const store = writable(init === undefined ? defaultValue : init) - - store.subscribe((value: T) => setJson(key, value)) - - return store -} - -// Getters - -export const getter = (store: Readable) => { - let value: T - - store.subscribe((newValue: T) => { - value = newValue - }) - - return () => value -} - -export type WritableWithGetter = Writable & {get: () => T} -export type ReadableWithGetter = Readable & {get: () => T} - -export function withGetter(store: Writable): WritableWithGetter -export function withGetter(store: Readable): ReadableWithGetter -export function withGetter(store: Readable | Writable) { - return {...store, get: getter(store)} -} - -// Throttle - -export const throttled = >(delay: number, store: S) => { - if (delay) { - const {subscribe} = store - - store = {...store, subscribe: (f: Subscriber) => subscribe(throttle(delay, f))} - } - - return store -} - -// Custom store - -type Start = (set: Subscriber) => Unsubscriber - -export type CustomStoreOpts = { - throttle?: number - set?: (x: T) => void -} - -export const custom = ( - start: Start, - opts: CustomStoreOpts = {}, -): WritableWithGetter => { - const subs: Subscriber[] = [] - - let value: T - let stop: () => void - - const set = (newValue: T) => { - for (const sub of subs) { - sub(newValue) - } - - value = newValue - } - - return { - get: () => value, - set: (newValue: T) => { - set(newValue) - opts.set?.(newValue) - }, - update: (f: (value: T) => T) => { - const newValue = f(value) - - set(newValue) - opts.set?.(newValue) - }, - subscribe: (sub: Subscriber) => { - if (opts.throttle) { - sub = throttle(opts.throttle, sub) - } - - if (subs.length === 0) { - stop = start(set) - } - - subs.push(sub) - sub(value) - - return () => { - subs.splice( - subs.findIndex(s => s === sub), - 1, - ) - - if (subs.length === 0) { - stop() - } - } - }, - } -} - -// Event related stores - -export type DeriveEventsMappedOptions = { - filters: Filter[] - eventToItem: (event: TrustedEvent) => T | T[] | Promise | undefined - itemToEvent: (item: T) => TrustedEvent - throttle?: number - includeDeleted?: boolean -} - -export const deriveEventsMapped = ( - repository: Repository, - { - filters, - eventToItem, - itemToEvent, - throttle = 0, - includeDeleted = false, - }: DeriveEventsMappedOptions, -) => - custom( - setter => { - let data: T[] = [] - const deferred = new Set() - - const defer = (event: TrustedEvent, promise: Promise) => { - deferred.add(event.id) - - void promise.then(items => { - if (deferred.has(event.id)) { - deferred.delete(event.id) - - for (const item of ensurePlural(items)) { - data.push(item) - } - - setter(data) - } - }) - } - - for (const event of repository.query(filters, {includeDeleted})) { - const items = eventToItem(event) - - if (!items) { - continue - } - - if (items instanceof Promise) { - defer(event, items) - } else { - for (const item of ensurePlural(items)) { - data.push(item) - } - } - } - - setter(data) - - const onUpdate = batch(300, (updates: {added: TrustedEvent[]; removed: Set}[]) => { - const removed = new Set() - const added = new Map() - - // Apply updates in order - for (const update of updates) { - for (const event of update.added.values()) { - added.set(event.id, event) - removed.delete(event.id) - } - - for (const id of update.removed) { - removed.add(id) - added.delete(id) - deferred.delete(id) - } - } - - let dirty = false - for (const event of added.values()) { - if (matchFilters(filters, event)) { - const items = eventToItem(event) - - if (items instanceof Promise) { - defer(event, items) - } else if (items) { - dirty = true - - for (const item of ensurePlural(items)) { - data.push(item as T) - } - } - } - } - - if (!includeDeleted && removed.size > 0) { - const [deleted, ok] = partition( - (item: T) => getIdAndAddress(itemToEvent(item)).some((id: string) => removed.has(id)), - data, - ) - - if (deleted.length > 0) { - dirty = true - data = ok - } - } - - if (dirty) { - setter(data) - } - }) - - repository.on("update", onUpdate) - - return () => repository.off("update", onUpdate) - }, - {throttle}, - ) - -export type DeriveEventsOptions = Omit< - DeriveEventsMappedOptions, - "itemToEvent" | "eventToItem" -> - -export const deriveEvents = (repository: Repository, opts: DeriveEventsOptions) => - deriveEventsMapped(repository, { - ...opts, - eventToItem: identity, - itemToEvent: identity, - }) - -export const deriveEvent = (repository: Repository, idOrAddress: string) => - derived( - deriveEvents(repository, { - filters: getIdFilters([idOrAddress]), - includeDeleted: true, - }), - first, - ) - -export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) => - custom(setter => { - setter(repository.isDeleted(event)) - - const onUpdate = batch(300, () => setter(repository.isDeleted(event))) - - repository.on("update", onUpdate) - - return () => repository.off("update", onUpdate) - }) - -export const deriveIsDeletedByAddress = (repository: Repository, event: TrustedEvent) => - custom(setter => { - setter(repository.isDeletedByAddress(event)) - - const onUpdate = batch(300, () => setter(repository.isDeletedByAddress(event))) - - repository.on("update", onUpdate) - - return () => repository.off("update", onUpdate) - }) +export * from "./synced.js" +export * from "./getter.js" +export * from "./throttle.js" +export * from "./custom.js" +export * from "./repository.js" +export * from "./collection.js" diff --git a/packages/store/src/repository.ts b/packages/store/src/repository.ts new file mode 100644 index 0000000..cf059b6 --- /dev/null +++ b/packages/store/src/repository.ts @@ -0,0 +1,164 @@ +import {derived} from "svelte/store" +import {identity, ensurePlural, batch, partition, first} from "@welshman/lib" +import {Repository} from "@welshman/relay" +import {matchFilters, getIdAndAddress, getIdFilters, Filter, TrustedEvent} from "@welshman/util" +import {custom} from "./custom.js" + +export type DeriveEventsMappedOptions = { + filters: Filter[] + eventToItem: (event: TrustedEvent) => T | T[] | Promise | undefined + itemToEvent: (item: T) => TrustedEvent + throttle?: number + includeDeleted?: boolean +} + +export const deriveEventsMapped = ( + repository: Repository, + { + filters, + eventToItem, + itemToEvent, + throttle = 0, + includeDeleted = false, + }: DeriveEventsMappedOptions, +) => + custom( + setter => { + let data: T[] = [] + const deferred = new Set() + + const defer = (event: TrustedEvent, promise: Promise) => { + deferred.add(event.id) + + void promise.then(items => { + if (deferred.has(event.id)) { + deferred.delete(event.id) + + for (const item of ensurePlural(items)) { + data.push(item) + } + + setter(data) + } + }) + } + + for (const event of repository.query(filters, {includeDeleted})) { + const items = eventToItem(event) + + if (!items) { + continue + } + + if (items instanceof Promise) { + defer(event, items) + } else { + for (const item of ensurePlural(items)) { + data.push(item) + } + } + } + + setter(data) + + const onUpdate = batch(300, (updates: {added: TrustedEvent[]; removed: Set}[]) => { + const removed = new Set() + const added = new Map() + + // Apply updates in order + for (const update of updates) { + for (const event of update.added.values()) { + added.set(event.id, event) + removed.delete(event.id) + } + + for (const id of update.removed) { + removed.add(id) + added.delete(id) + deferred.delete(id) + } + } + + let dirty = false + for (const event of added.values()) { + if (matchFilters(filters, event)) { + const items = eventToItem(event) + + if (items instanceof Promise) { + defer(event, items) + } else if (items) { + dirty = true + + for (const item of ensurePlural(items)) { + data.push(item as T) + } + } + } + } + + if (!includeDeleted && removed.size > 0) { + const [deleted, ok] = partition( + (item: T) => getIdAndAddress(itemToEvent(item)).some((id: string) => removed.has(id)), + data, + ) + + if (deleted.length > 0) { + dirty = true + data = ok + } + } + + if (dirty) { + setter(data) + } + }) + + repository.on("update", onUpdate) + + return () => repository.off("update", onUpdate) + }, + {throttle}, + ) + +export type DeriveEventsOptions = Omit< + DeriveEventsMappedOptions, + "itemToEvent" | "eventToItem" +> + +export const deriveEvents = (repository: Repository, opts: DeriveEventsOptions) => + deriveEventsMapped(repository, { + ...opts, + eventToItem: identity, + itemToEvent: identity, + }) + +export const deriveEvent = (repository: Repository, idOrAddress: string) => + derived( + deriveEvents(repository, { + filters: getIdFilters([idOrAddress]), + includeDeleted: true, + }), + first, + ) + +export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) => + custom(setter => { + setter(repository.isDeleted(event)) + + const onUpdate = batch(300, () => setter(repository.isDeleted(event))) + + repository.on("update", onUpdate) + + return () => repository.off("update", onUpdate) + }) + +export const deriveIsDeletedByAddress = (repository: Repository, event: TrustedEvent) => + custom(setter => { + setter(repository.isDeletedByAddress(event)) + + const onUpdate = batch(300, () => setter(repository.isDeletedByAddress(event))) + + repository.on("update", onUpdate) + + return () => repository.off("update", onUpdate) + }) diff --git a/packages/store/src/synced.ts b/packages/store/src/synced.ts new file mode 100644 index 0000000..d411ec1 --- /dev/null +++ b/packages/store/src/synced.ts @@ -0,0 +1,11 @@ +import {writable} from "svelte/store" +import {getJson, setJson} from "@welshman/lib" + +export const synced = (key: string, defaultValue: T) => { + const init = getJson(key) + const store = writable(init === undefined ? defaultValue : init) + + store.subscribe((value: T) => setJson(key, value)) + + return store +} diff --git a/packages/store/src/throttle.ts b/packages/store/src/throttle.ts new file mode 100644 index 0000000..b48317e --- /dev/null +++ b/packages/store/src/throttle.ts @@ -0,0 +1,12 @@ +import {Readable, Subscriber} from "svelte/store" +import {throttle} from "@welshman/lib" + +export const throttled = >(delay: number, store: S) => { + if (delay) { + const {subscribe} = store + + store = {...store, subscribe: (f: Subscriber) => subscribe(throttle(delay, f))} + } + + return store +}