import {readable, Readable} from "svelte/store" import {on, now, mapPop, Maybe, MaybeAsync, call, sortBy, first} from "@welshman/lib" import { matchFilters, getIdFilters, Filter, TrustedEvent, sortEventsAsc, sortEventsDesc, getIdOrAddress, } from "@welshman/util" import {Repository, RepositoryUpdate, Tracker} from "@welshman/net" import {deriveDeduplicated} from "./misc.js" // Events by id export type EventsById = Map export type EventsByIdOptions = { filters: Filter[] repository: Repository includeDeleted?: boolean } export const getEventsById = ({filters, repository, includeDeleted}: EventsByIdOptions) => { const eventsById = new Map() for (const event of repository.query(filters, {includeDeleted})) { eventsById.set(event.id, event) } return eventsById } export const deriveEventsById = ({filters, repository, includeDeleted}: EventsByIdOptions) => readable(new Map(), set => { const eventsById = getEventsById({filters, repository, includeDeleted}) set(eventsById) return on(repository, "update", ({added, removed}: RepositoryUpdate) => { let dirty = false for (const event of added) { if (matchFilters(filters, event)) { dirty = true eventsById.set(event.id, event) } } for (const id of removed) { if (mapPop(id, eventsById)) { dirty = true } } if (dirty) { set(eventsById) } }) }) export const deriveArray = (itemsByIdStore: Readable>) => deriveDeduplicated(itemsByIdStore, itemsById => Array.from(itemsById.values())) export const deriveEventsAsc = (eventsByIdStore: Readable) => deriveDeduplicated(eventsByIdStore, eventsById => sortEventsAsc(eventsById.values())) export const deriveEventsDesc = (eventsByIdStore: Readable) => deriveDeduplicated(eventsByIdStore, eventsById => sortEventsDesc(eventsById.values())) export const deriveEvents = (options: EventsByIdOptions) => deriveArray(deriveEventsById(options)) export type EventOptions = { repository: Repository includeDeleted?: boolean onDerive?: (filters: Filter[], ...args: any[]) => void } export const makeDeriveEvent = ({repository, includeDeleted = false, onDerive}: EventOptions) => { return (idOrAddress: string, ...args: any[]) => { const filters = getIdFilters([idOrAddress]) onDerive?.(filters, ...args) return readable>(undefined, set => { let event = first(repository.query(filters, {includeDeleted})) set(event) return on(repository, "update", ({added, removed}: RepositoryUpdate) => { if (!includeDeleted) { for (const id of removed) { if (event?.id === id) { set(undefined) } } } for (const newEvent of added) { if (matchFilters(filters, newEvent)) { event = newEvent set(event) } } }) }) } } // Events by id by url export type EventsByIdByUrl = Map export type EventsByIdByUrlOptions = EventsByIdOptions & { tracker: Tracker } export const getEventsByIdByUrl = ({ filters, tracker, repository, includeDeleted, }: EventsByIdByUrlOptions) => { const eventsByIdByUrl: EventsByIdByUrl = new Map() for (const event of repository.query(filters, {includeDeleted})) { for (const url of tracker.getRelays(event.id)) { let eventsById = eventsByIdByUrl.get(url) if (!eventsById) { eventsById = new Map() eventsByIdByUrl.set(url, eventsById) } eventsById.set(event.id, event) } } return eventsByIdByUrl } export const deriveEventsByIdByUrl = ({ filters, tracker, repository, includeDeleted, }: EventsByIdByUrlOptions) => readable(new Map(), set => { const eventsByIdByUrl = getEventsByIdByUrl({filters, tracker, repository, includeDeleted}) const addEvent = (url: string, event: TrustedEvent) => { if (!matchFilters(filters, event)) return false const eventsById = eventsByIdByUrl.get(url) if (eventsById?.has(event.id)) return false // Create a new map so we can detect which key changed const newEventsById = new Map(eventsById) newEventsById.set(event.id, event) eventsByIdByUrl.set(url, newEventsById) return true } const removeEvent = (url: string, id: string) => { const eventsById = eventsByIdByUrl.get(url) if (eventsById?.has(id)) { eventsById.delete(id) if (eventsById.size === 0) { eventsByIdByUrl.delete(url) } else { // Create a new map so we can detect which key changed eventsByIdByUrl.set(url, new Map(eventsById)) } return true } return false } set(eventsByIdByUrl) const unsubscribers = [ on(repository, "update", ({added, removed}: RepositoryUpdate) => { let dirty = false for (const event of added) { for (const url of tracker.getRelays(event.id)) { dirty = dirty || addEvent(url, event) } } for (const id of removed) { for (const url of tracker.getRelays(id)) { dirty = dirty || removeEvent(url, id) } } if (dirty) { set(eventsByIdByUrl) } }), on(tracker, "add", (id: string, url: string) => { const event = repository.getEvent(id) if (event && addEvent(url, event)) { set(eventsByIdByUrl) } }), on(tracker, "remove", (id: string, url: string) => { if (removeEvent(url, id)) { set(eventsByIdByUrl) } }), on(tracker, "load", () => { eventsByIdByUrl.clear() for (const event of repository.query(filters, {includeDeleted})) { for (const url of tracker.getRelays(event.id)) { addEvent(url, event) } } set(eventsByIdByUrl) }), on(tracker, "clear", () => { eventsByIdByUrl.clear() set(eventsByIdByUrl) }), ] return () => unsubscribers.forEach(call) }) export type EventsByIdForUrlOptions = EventsByIdOptions & { url: string tracker: Tracker } export const getEventsByIdForUrl = ({ url, filters, tracker, repository, includeDeleted, }: EventsByIdForUrlOptions) => { const initialIds = Array.from(tracker.getIds(url)) const initialFilters = filters.map(filter => ({ids: initialIds, ...filter})) const eventsById: EventsById = new Map() for (const event of repository.query(initialFilters, {includeDeleted})) { eventsById.set(event.id, event) } return eventsById } export const deriveEventsByIdForUrl = ({ url, filters, tracker, repository, includeDeleted, }: EventsByIdForUrlOptions) => { let eventsById = getEventsByIdForUrl({url, filters, tracker, repository, includeDeleted}) return readable(eventsById, set => { const reset = () => { eventsById = getEventsByIdForUrl({url, filters, tracker, repository, includeDeleted}) set(eventsById) } reset() const unsubscribers = [ on(repository, "update", ({added, removed}: RepositoryUpdate) => { let dirty = false for (const event of added) { if (tracker.hasRelay(event.id, url) && matchFilters(filters, event)) { eventsById.set(event.id, event) dirty = true } } for (const id of removed) { if (eventsById.has(id)) { eventsById.delete(id) dirty = true } } if (dirty) { set(eventsById) } }), on(tracker, "add", (id: string, trackedUrl: string) => { if (trackedUrl === url) { const event = repository.getEvent(id) if (event && tracker.hasRelay(id, trackedUrl) && matchFilters(filters, event)) { eventsById.set(id, event) set(eventsById) } } }), on(tracker, "remove", (id: string, trackedUrl: string) => { if (trackedUrl === url && eventsById.has(id)) { eventsById.delete(id) set(eventsById) } }), on(tracker, "load", reset), on(tracker, "clear", reset), ] return () => unsubscribers.forEach(call) }) } // Items by key export type ItemsByKey = Map export type EventToItem = (event: TrustedEvent) => MaybeAsync> export type GetItem = (key: string, ...args: any[]) => Maybe export type ItemsByKeyOptions = { getKey: (item: T) => string filters: Filter[] repository: Repository eventToItem: EventToItem includeDeleted?: boolean } export const deriveItemsByKey = ({ getKey, filters, repository, eventToItem, includeDeleted, }: ItemsByKeyOptions) => { const deferred = new Map>>() const itemsByKey = new Map() const idsByKey = new Map() const keysById = new Map() return readable(itemsByKey, set => { const addEvent = async (event: TrustedEvent) => { if (deferred.has(event.id)) return if (keysById.has(event.id)) return const itemOrPromise = eventToItem(event) if (itemOrPromise instanceof Promise) { deferred.set(event.id, itemOrPromise) } try { const item = await itemOrPromise if (item) { const key = getKey(item) itemsByKey.set(key, item) idsByKey.set(key, event.id) keysById.set(event.id, key) set(itemsByKey) } } finally { deferred.delete(event.id) } } for (const event of repository.query(filters, {includeDeleted})) { addEvent(event) } return on(repository, "update", ({added, removed}: RepositoryUpdate) => { for (const event of added) { if (matchFilters(filters, event)) { addEvent(event) } } if (!includeDeleted) { let dirty = false for (const id of removed) { const key = mapPop(id, keysById) if (key) { idsByKey.delete(key) itemsByKey.delete(key) dirty = true } } if (dirty) { set(itemsByKey) } } }) }) } export const deriveItems = (itemsByKeyStore: Readable>) => deriveDeduplicated(itemsByKeyStore, itemsByKey => Array.from(itemsByKey.values())) export const deriveItemsSorted = (sortFn: (item: T) => number, itemsStore: Readable) => deriveDeduplicated(itemsStore, items => sortBy(sortFn, items)) export const makeDeriveItem = ( itemsByKeyStore: Readable>, onDerive?: (key: string, ...args: any[]) => void, ) => { return (key?: string, ...args: any[]) => { if (!key) return readable(undefined) onDerive?.(key, ...args) return deriveDeduplicated(itemsByKeyStore, itemsByKey => itemsByKey.get(key)) } } // Item loaders export type LoadItem = (key: string, ...args: any[]) => Promise export const makeForceLoadItem = (loadItem: LoadItem, getItem: GetItem) => { return (key: string, ...args: any[]) => loadItem(key, ...args).then(() => getItem(key)) } export type MakeLoadItemOptions = { getFetched?: (key: string) => number setFetched?: (key: string, ts: number) => void timeout?: number } export const makeLoadItem = ( loadItem: LoadItem, getItem: GetItem, options: MakeLoadItemOptions = {}, ) => { const timeout = options.timeout || 3600 const fetched = new Map() const getFetched = options.getFetched || ((key: string) => fetched.get(key) || 0) const setFetched = options.setFetched || ((key: string, ts: number) => fetched.set(key, ts)) const pending = new Map>>() const attempts = new Map() return (key: string, ...args: any[]): Promise> => { const stale = getItem(key) const fetched = getFetched(key) // If we have an item, reload if it's relatively recent if (stale && fetched > now() - timeout) { return Promise.resolve(stale) } const pendingItem = pending.get(key) // If we already are loading, await and return if (pendingItem) { return Promise.resolve(pendingItem) } const attempt = attempts.get(key) || 0 // Use exponential backoff to throttle attempts if (fetched > now() - Math.pow(2, attempt)) { return Promise.resolve(stale) } attempts.set(key, attempt + 1) setFetched(key, now()) const promise = loadItem(key, ...args).then(() => getItem(key)) pending.set(key, promise) return call(async () => { let item try { item = await promise } catch (e) { console.warn(`Failed to load item ${key}`, e) } finally { pending.delete(key) } if (item) { attempts.delete(key) } return item }) } } // Miscellaneous other stuff export const deriveIsDeleted = (repository: Repository, event: TrustedEvent) => readable(false, set => { const idOrAddress = getIdOrAddress(event) set(repository.isDeleted(event)) return on(repository, "update", ({removed, added}: RepositoryUpdate) => { if (removed.has(event.id)) { set(true) } for (const event of added) { if (getIdOrAddress(event) === idOrAddress) { set(false) } } }) })