Move collection to store module

This commit is contained in:
Jon Staab
2025-04-25 10:37:57 -07:00
parent d14ae2ce77
commit 37c0491d71
22 changed files with 325 additions and 987 deletions
+240
View File
@@ -0,0 +1,240 @@
import {describe, it, expect, beforeEach, vi, afterEach} from "vitest"
import {get, writable} from "svelte/store"
import {now, always} from "@welshman/lib"
import {collection, freshness, setFreshnessImmediate} from "../src/collection"
describe("collection", () => {
beforeEach(() => {
vi.useFakeTimers()
vi.clearAllMocks()
})
afterEach(() => {
vi.resetModules()
vi.useRealTimers()
freshness.set({})
})
describe("basic functionality", () => {
it("should create a collection with indexStore", () => {
const items = [{id: "1", value: "test"}]
const store = writable(items)
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
expect(col.indexStore.get().get("1")).toEqual(items[0])
})
it("should update indexStore when store changes", () => {
const store = writable<Array<{id: string; value: string}>>([])
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
const newItem = {id: "1", value: "test"}
store.set([newItem])
expect(get(col.indexStore).get("1")).toEqual(newItem)
})
})
describe("loadItem", () => {
it("should return stale item if no loader provided", async () => {
const items = [{id: "1", value: "test"}]
const store = writable(items)
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
const result = await col.loadItem("1")
expect(result).toEqual(items[0])
})
it("should return undefined for non-existent items when no loader provided", async () => {
const store = writable<Array<{id: string}>>([])
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
const result = await col.loadItem("1")
expect(result).toBeUndefined()
})
it("should use loader to fetch new items", async () => {
const store = writable<Array<{id: string; value: string}>>([])
const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
await col.loadItem("1")
expect(mockLoad).toHaveBeenCalledWith("1", [])
})
it("should handle concurrent loading of the same item", async () => {
const store = writable<Array<{id: string; value: string}>>([])
const mockLoad = vi.fn().mockResolvedValue({id: "1", value: "loaded"})
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
// Start multiple concurrent loads
const loads = Promise.all([col.loadItem("1"), col.loadItem("1"), col.loadItem("1")])
await loads
// Should only call load once
expect(mockLoad).toHaveBeenCalledTimes(1)
})
it("should respect freshness checks", async () => {
await vi.advanceTimersByTimeAsync(1000)
const store = writable<Array<{id: string; value: string}>>([{id: "1", value: "stale"}])
const mockLoad = vi.fn()
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
// force freshness
setFreshnessImmediate({ns: "test", key: "1", ts: now()})
await col.loadItem("1")
// Should not call load because item is fresh
expect(mockLoad).toHaveBeenCalledTimes(0)
})
it("should reload stale items", async () => {
const mockLoad = vi.fn()
const store = writable([{id: "1", value: "test"}])
const col = collection({
name: "test",
store,
getKey: (item: any) => item.id,
load: mockLoad,
})
// load the item to set freshness
await col.loadItem("1")
await vi.advanceTimersByTimeAsync(4000 * 1000)
await col.loadItem("1")
expect(mockLoad).toHaveBeenCalledTimes(2)
})
it("should implement exponential backoff for failed attempts", async () => {
const store = writable<Array<{id: string; value: string}>>([])
const mockLoad = vi.fn().mockResolvedValue(undefined)
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
// First attempt
await col.loadItem("1")
expect(mockLoad).toHaveBeenCalledTimes(1)
//force freshness
setFreshnessImmediate({ns: "test", key: "1", ts: now()})
// Immediate retry should be throttled
await col.loadItem("1").catch(() => {})
expect(mockLoad).toHaveBeenCalledTimes(1)
})
})
describe("deriveItem", () => {
it("should return readable undefined for null keys", () => {
const store = writable<Array<{id: string}>>([])
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
const derived = col.deriveItem(undefined)
expect(get(derived)).toBeUndefined()
})
it("should create a derived store that updates with the source", () => {
const store = writable<Array<{id: string; value: string}>>([])
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: always(Promise.resolve()),
})
const derived = col.deriveItem("1")
expect(get(derived)).toBeUndefined()
// Update source store
store.set([{id: "1", value: "test"}])
expect(get(derived)).toEqual({id: "1", value: "test"})
})
it("should trigger load when deriving non-existent item", () => {
const store = writable<Array<{id: string; value: string}>>([])
const mockLoad = vi.fn()
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
col.deriveItem("1")
expect(mockLoad).toHaveBeenCalledWith("1", [])
})
})
describe("error handling", () => {
it("should handle loader failures gracefully", async () => {
const store = writable<Array<{id: string; value: string}>>([])
const mockLoad = vi.fn(() => {
return Promise.reject("load failed")
})
const col = collection({
name: "test",
store,
getKey: item => item.id,
load: mockLoad,
})
const result = await col.loadItem("1")
expect(result).toBeUndefined()
})
})
})
+141
View File
@@ -0,0 +1,141 @@
import {readable, derived, writable, Readable, Subscriber} from "svelte/store"
import {batch, indexBy, remove, assoc, now} from "@welshman/lib"
import {withGetter, ReadableWithGetter} from "./getter.js"
// Collection utility
export type FreshnessUpdate = {
ns: string
key: string
ts: number
}
export const freshness = withGetter(writable<Record<string, number>>({}))
export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}`
export const getFreshness = (ns: string, key: string) =>
freshness.get()[getFreshnessKey(ns, key)] || 0
export const setFreshnessImmediate = ({ns, key, ts}: FreshnessUpdate) =>
freshness.update(assoc(getFreshnessKey(ns, key), ts))
export const setFreshnessThrottled = batch(100, (updates: FreshnessUpdate[]) =>
freshness.update($freshness => {
for (const {ns, key, ts} of updates) {
$freshness[getFreshnessKey(ns, key)] = ts
}
return $freshness
}),
)
export type CachedLoaderOptions<T> = {
name: string
indexStore: ReadableWithGetter<Map<string, T>>
load: (key: string, relays: string[]) => Promise<any>
subscribers?: Subscriber<T>[]
}
export const makeCachedLoader = <T>({
name,
load,
indexStore,
subscribers = [],
}: CachedLoaderOptions<T>) => {
const pending = new Map<string, Promise<T | void>>()
const loadAttempts = new Map<string, number>()
return async (key: string, relays: string[] = []) => {
const stale = indexStore.get().get(key)
// If we have no loader function, nothing we can do
if (!load) {
return stale
}
const freshness = getFreshness(name, key)
// If we have an item, reload if it's stale
if (stale && freshness > now() - 3600) {
return stale
}
// If we already are loading, await and return
if (pending.has(key)) {
return pending.get(key)!.then(() => indexStore.get().get(key))
}
const attempt = loadAttempts.get(key) || 0
// Use exponential backoff to throttle attempts
if (freshness > now() - Math.pow(2, attempt)) {
return stale
}
loadAttempts.set(key, attempt + 1)
setFreshnessThrottled({ns: name, key, ts: now()})
const promise = load(key, relays)
pending.set(key, promise)
try {
await promise
} catch (e) {
console.warn(`Failed to load ${name} item ${key}`, e)
} finally {
pending.delete(key)
}
const fresh = indexStore.get().get(key)
if (fresh) {
loadAttempts.delete(key)
for (const subscriber of subscribers) {
subscriber(fresh)
}
}
return fresh
}
}
export type CollectionOptions<T> = {
name: string
store: Readable<T[]>
getKey: (item: T) => string
load: (key: string, relays: string[]) => Promise<any>
}
export const collection = <T>({name, store, getKey, load}: CollectionOptions<T>) => {
const indexStore = withGetter(derived(store, $items => indexBy(getKey, $items)))
let subscribers: Subscriber<T>[] = []
const loadItem = makeCachedLoader({name, load, indexStore, subscribers})
const deriveItem = (key: string | undefined, relays: string[] = []) => {
if (!key) {
return readable(undefined)
}
// If we don't yet have the item, or it's stale, trigger a request for it. The derived
// store will update when it arrives
loadItem(key, relays)
return derived(indexStore, $index => $index.get(key))
}
const onItem = (cb: Subscriber<T>) => {
subscribers.push(cb)
return () => {
subscribers = remove(cb, subscribers)
}
}
return {indexStore, deriveItem, loadItem, onItem}
}
+65
View File
@@ -0,0 +1,65 @@
import {Subscriber, Unsubscriber} from "svelte/store"
import {throttle} from "@welshman/lib"
import {WritableWithGetter} from "./getter.js"
type Start<T> = (set: Subscriber<T>) => Unsubscriber
export type CustomStoreOpts<T> = {
throttle?: number
set?: (x: T) => void
}
export const custom = <T>(
start: Start<T>,
opts: CustomStoreOpts<T> = {},
): WritableWithGetter<T> => {
const subs: Subscriber<T>[] = []
let value: T
let stop: () => void
const set = (newValue: T) => {
for (const sub of subs) {
sub(newValue)
}
value = newValue
}
return {
get: () => value,
set: (newValue: T) => {
set(newValue)
opts.set?.(newValue)
},
update: (f: (value: T) => T) => {
const newValue = f(value)
set(newValue)
opts.set?.(newValue)
},
subscribe: (sub: Subscriber<T>) => {
if (opts.throttle) {
sub = throttle(opts.throttle, sub)
}
if (subs.length === 0) {
stop = start(set)
}
subs.push(sub)
sub(value)
return () => {
subs.splice(
subs.findIndex(s => s === sub),
1,
)
if (subs.length === 0) {
stop()
}
}
},
}
}
+20
View File
@@ -0,0 +1,20 @@
import {Readable, Writable} from "svelte/store"
export const getter = <T>(store: Readable<T>) => {
let value: T
store.subscribe((newValue: T) => {
value = newValue
})
return () => value
}
export type WritableWithGetter<T> = Writable<T> & {get: () => T}
export type ReadableWithGetter<T> = Readable<T> & {get: () => T}
export function withGetter<T>(store: Writable<T>): WritableWithGetter<T>
export function withGetter<T>(store: Readable<T>): ReadableWithGetter<T>
export function withGetter<T>(store: Readable<T> | Writable<T>) {
return {...store, get: getter<T>(store)}
}
+6 -283
View File
@@ -1,283 +1,6 @@
import {derived, writable} from "svelte/store"
import type {Readable, Writable, Subscriber, Unsubscriber} from "svelte/store"
import {
identity,
throttle,
ensurePlural,
getJson,
setJson,
batch,
partition,
first,
} from "@welshman/lib"
import {Repository} from "@welshman/relay"
import {matchFilters, getIdAndAddress, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
// Sync with localstorage
export const synced = <T>(key: string, defaultValue: T) => {
const init = getJson(key)
const store = writable<T>(init === undefined ? defaultValue : init)
store.subscribe((value: T) => setJson(key, value))
return store
}
// Getters
export const getter = <T>(store: Readable<T>) => {
let value: T
store.subscribe((newValue: T) => {
value = newValue
})
return () => value
}
export type WritableWithGetter<T> = Writable<T> & {get: () => T}
export type ReadableWithGetter<T> = Readable<T> & {get: () => T}
export function withGetter<T>(store: Writable<T>): WritableWithGetter<T>
export function withGetter<T>(store: Readable<T>): ReadableWithGetter<T>
export function withGetter<T>(store: Readable<T> | Writable<T>) {
return {...store, get: getter<T>(store)}
}
// Throttle
export const throttled = <T, S extends Readable<T>>(delay: number, store: S) => {
if (delay) {
const {subscribe} = store
store = {...store, subscribe: (f: Subscriber<T>) => subscribe(throttle(delay, f))}
}
return store
}
// Custom store
type Start<T> = (set: Subscriber<T>) => Unsubscriber
export type CustomStoreOpts<T> = {
throttle?: number
set?: (x: T) => void
}
export const custom = <T>(
start: Start<T>,
opts: CustomStoreOpts<T> = {},
): WritableWithGetter<T> => {
const subs: Subscriber<T>[] = []
let value: T
let stop: () => void
const set = (newValue: T) => {
for (const sub of subs) {
sub(newValue)
}
value = newValue
}
return {
get: () => value,
set: (newValue: T) => {
set(newValue)
opts.set?.(newValue)
},
update: (f: (value: T) => T) => {
const newValue = f(value)
set(newValue)
opts.set?.(newValue)
},
subscribe: (sub: Subscriber<T>) => {
if (opts.throttle) {
sub = throttle(opts.throttle, sub)
}
if (subs.length === 0) {
stop = start(set)
}
subs.push(sub)
sub(value)
return () => {
subs.splice(
subs.findIndex(s => s === sub),
1,
)
if (subs.length === 0) {
stop()
}
}
},
}
}
// Event related stores
export type DeriveEventsMappedOptions<T> = {
filters: Filter[]
eventToItem: (event: TrustedEvent) => T | T[] | Promise<T | T[]> | undefined
itemToEvent: (item: T) => TrustedEvent
throttle?: number
includeDeleted?: boolean
}
export const deriveEventsMapped = <T>(
repository: Repository,
{
filters,
eventToItem,
itemToEvent,
throttle = 0,
includeDeleted = false,
}: DeriveEventsMappedOptions<T>,
) =>
custom<T[]>(
setter => {
let data: T[] = []
const deferred = new Set()
const defer = (event: TrustedEvent, promise: Promise<T | T[]>) => {
deferred.add(event.id)
void promise.then(items => {
if (deferred.has(event.id)) {
deferred.delete(event.id)
for (const item of ensurePlural(items)) {
data.push(item)
}
setter(data)
}
})
}
for (const event of repository.query(filters, {includeDeleted})) {
const items = eventToItem(event)
if (!items) {
continue
}
if (items instanceof Promise) {
defer(event, items)
} else {
for (const item of ensurePlural(items)) {
data.push(item)
}
}
}
setter(data)
const onUpdate = batch(300, (updates: {added: TrustedEvent[]; removed: Set<string>}[]) => {
const removed = new Set()
const added = new Map()
// Apply updates in order
for (const update of updates) {
for (const event of update.added.values()) {
added.set(event.id, event)
removed.delete(event.id)
}
for (const id of update.removed) {
removed.add(id)
added.delete(id)
deferred.delete(id)
}
}
let dirty = false
for (const event of added.values()) {
if (matchFilters(filters, event)) {
const items = eventToItem(event)
if (items instanceof Promise) {
defer(event, items)
} else if (items) {
dirty = true
for (const item of ensurePlural(items)) {
data.push(item as T)
}
}
}
}
if (!includeDeleted && removed.size > 0) {
const [deleted, ok] = partition(
(item: T) => getIdAndAddress(itemToEvent(item)).some((id: string) => removed.has(id)),
data,
)
if (deleted.length > 0) {
dirty = true
data = ok
}
}
if (dirty) {
setter(data)
}
})
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
},
{throttle},
)
export type DeriveEventsOptions<T> = Omit<
DeriveEventsMappedOptions<T>,
"itemToEvent" | "eventToItem"
>
export const deriveEvents = <T>(repository: Repository, opts: DeriveEventsOptions<T>) =>
deriveEventsMapped<TrustedEvent>(repository, {
...opts,
eventToItem: identity,
itemToEvent: identity,
})
export const deriveEvent = (repository: Repository, idOrAddress: string) =>
derived(
deriveEvents(repository, {
filters: getIdFilters([idOrAddress]),
includeDeleted: true,
}),
first,
)
export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) =>
custom<boolean>(setter => {
setter(repository.isDeleted(event))
const onUpdate = batch(300, () => setter(repository.isDeleted(event)))
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
})
export const deriveIsDeletedByAddress = (repository: Repository, event: TrustedEvent) =>
custom<boolean>(setter => {
setter(repository.isDeletedByAddress(event))
const onUpdate = batch(300, () => setter(repository.isDeletedByAddress(event)))
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
})
export * from "./synced.js"
export * from "./getter.js"
export * from "./throttle.js"
export * from "./custom.js"
export * from "./repository.js"
export * from "./collection.js"
+164
View File
@@ -0,0 +1,164 @@
import {derived} from "svelte/store"
import {identity, ensurePlural, batch, partition, first} from "@welshman/lib"
import {Repository} from "@welshman/relay"
import {matchFilters, getIdAndAddress, getIdFilters, Filter, TrustedEvent} from "@welshman/util"
import {custom} from "./custom.js"
export type DeriveEventsMappedOptions<T> = {
filters: Filter[]
eventToItem: (event: TrustedEvent) => T | T[] | Promise<T | T[]> | undefined
itemToEvent: (item: T) => TrustedEvent
throttle?: number
includeDeleted?: boolean
}
export const deriveEventsMapped = <T>(
repository: Repository,
{
filters,
eventToItem,
itemToEvent,
throttle = 0,
includeDeleted = false,
}: DeriveEventsMappedOptions<T>,
) =>
custom<T[]>(
setter => {
let data: T[] = []
const deferred = new Set()
const defer = (event: TrustedEvent, promise: Promise<T | T[]>) => {
deferred.add(event.id)
void promise.then(items => {
if (deferred.has(event.id)) {
deferred.delete(event.id)
for (const item of ensurePlural(items)) {
data.push(item)
}
setter(data)
}
})
}
for (const event of repository.query(filters, {includeDeleted})) {
const items = eventToItem(event)
if (!items) {
continue
}
if (items instanceof Promise) {
defer(event, items)
} else {
for (const item of ensurePlural(items)) {
data.push(item)
}
}
}
setter(data)
const onUpdate = batch(300, (updates: {added: TrustedEvent[]; removed: Set<string>}[]) => {
const removed = new Set()
const added = new Map()
// Apply updates in order
for (const update of updates) {
for (const event of update.added.values()) {
added.set(event.id, event)
removed.delete(event.id)
}
for (const id of update.removed) {
removed.add(id)
added.delete(id)
deferred.delete(id)
}
}
let dirty = false
for (const event of added.values()) {
if (matchFilters(filters, event)) {
const items = eventToItem(event)
if (items instanceof Promise) {
defer(event, items)
} else if (items) {
dirty = true
for (const item of ensurePlural(items)) {
data.push(item as T)
}
}
}
}
if (!includeDeleted && removed.size > 0) {
const [deleted, ok] = partition(
(item: T) => getIdAndAddress(itemToEvent(item)).some((id: string) => removed.has(id)),
data,
)
if (deleted.length > 0) {
dirty = true
data = ok
}
}
if (dirty) {
setter(data)
}
})
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
},
{throttle},
)
export type DeriveEventsOptions<T> = Omit<
DeriveEventsMappedOptions<T>,
"itemToEvent" | "eventToItem"
>
export const deriveEvents = <T>(repository: Repository, opts: DeriveEventsOptions<T>) =>
deriveEventsMapped<TrustedEvent>(repository, {
...opts,
eventToItem: identity,
itemToEvent: identity,
})
export const deriveEvent = (repository: Repository, idOrAddress: string) =>
derived(
deriveEvents(repository, {
filters: getIdFilters([idOrAddress]),
includeDeleted: true,
}),
first,
)
export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) =>
custom<boolean>(setter => {
setter(repository.isDeleted(event))
const onUpdate = batch(300, () => setter(repository.isDeleted(event)))
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
})
export const deriveIsDeletedByAddress = (repository: Repository, event: TrustedEvent) =>
custom<boolean>(setter => {
setter(repository.isDeletedByAddress(event))
const onUpdate = batch(300, () => setter(repository.isDeletedByAddress(event)))
repository.on("update", onUpdate)
return () => repository.off("update", onUpdate)
})
+11
View File
@@ -0,0 +1,11 @@
import {writable} from "svelte/store"
import {getJson, setJson} from "@welshman/lib"
export const synced = <T>(key: string, defaultValue: T) => {
const init = getJson(key)
const store = writable<T>(init === undefined ? defaultValue : init)
store.subscribe((value: T) => setJson(key, value))
return store
}
+12
View File
@@ -0,0 +1,12 @@
import {Readable, Subscriber} from "svelte/store"
import {throttle} from "@welshman/lib"
export const throttled = <T, S extends Readable<T>>(delay: number, store: S) => {
if (delay) {
const {subscribe} = store
store = {...store, subscribe: (f: Subscriber<T>) => subscribe(throttle(delay, f))}
}
return store
}