bring in khatru and eventstore.

eventstore/badger/count.go

@@ -0,0 +1,168 @@
+package badger
+
+import (
+	"context"
+	"encoding/binary"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip45/hyperloglog"
+)
+
+func (b *BadgerBackend) CountEvents(ctx context.Context, filter nostr.Filter) (int64, error) {
+	var count int64 = 0
+
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return 0, err
+	}
+
+	err = b.View(func(txn *badger.Txn) error {
+		// iterate only through keys and in reverse order
+		opts := badger.IteratorOptions{
+			Reverse: true,
+		}
+
+		// actually iterate
+		for _, q := range queries {
+			it := txn.NewIterator(opts)
+			defer it.Close()
+
+			for it.Seek(q.startingPoint); it.ValidForPrefix(q.prefix); it.Next() {
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !q.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						break
+					}
+				}
+
+				idx := make([]byte, 5)
+				idx[0] = rawEventStorePrefix
+				copy(idx[1:], key[idxOffset:])
+
+				if extraFilter == nil {
+					count++
+				} else {
+					// fetch actual event
+					item, err := txn.Get(idx)
+					if err != nil {
+						if err == badger.ErrDiscardedTxn {
+							return err
+						}
+						log.Printf("badger: count (%v) failed to get %d from raw event store: %s\n", q, idx, err)
+						return err
+					}
+
+					err = item.Value(func(val []byte) error {
+						evt := &nostr.Event{}
+						if err := bin.Unmarshal(val, evt); err != nil {
+							return err
+						}
+
+						// check if this matches the other filters that were not part of the index
+						if extraFilter.Matches(evt) {
+							count++
+						}
+
+						return nil
+					})
+					if err != nil {
+						log.Printf("badger: count value read error: %s\n", err)
+					}
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, err
+}
+
+func (b *BadgerBackend) CountEventsHLL(ctx context.Context, filter nostr.Filter, offset int) (int64, *hyperloglog.HyperLogLog, error) {
+	var count int64 = 0
+
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	hll := hyperloglog.New(offset)
+
+	err = b.View(func(txn *badger.Txn) error {
+		// iterate only through keys and in reverse order
+		opts := badger.IteratorOptions{
+			Reverse: true,
+		}
+
+		// actually iterate
+		for _, q := range queries {
+			it := txn.NewIterator(opts)
+			defer it.Close()
+
+			for it.Seek(q.startingPoint); it.ValidForPrefix(q.prefix); it.Next() {
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !q.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						break
+					}
+				}
+
+				idx := make([]byte, 5)
+				idx[0] = rawEventStorePrefix
+				copy(idx[1:], key[idxOffset:])
+
+				// fetch actual event
+				item, err := txn.Get(idx)
+				if err != nil {
+					if err == badger.ErrDiscardedTxn {
+						return err
+					}
+					log.Printf("badger: count (%v) failed to get %d from raw event store: %s\n", q, idx, err)
+					return err
+				}
+
+				err = item.Value(func(val []byte) error {
+					if extraFilter == nil {
+						hll.AddBytes(val[32:64])
+						count++
+						return nil
+					}
+
+					evt := &nostr.Event{}
+					if err := bin.Unmarshal(val, evt); err != nil {
+						return err
+					}
+					if extraFilter.Matches(evt) {
+						hll.Add(evt.PubKey)
+						count++
+						return nil
+					}
+
+					return nil
+				})
+				if err != nil {
+					log.Printf("badger: count value read error: %s\n", err)
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return count, hll, err
+}

eventstore/badger/delete.go

@@ -0,0 +1,72 @@
+package badger
+
+import (
+	"context"
+	"encoding/hex"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var serialDelete uint32 = 0
+
+func (b *BadgerBackend) DeleteEvent(ctx context.Context, evt *nostr.Event) error {
+	deletionHappened := false
+
+	err := b.Update(func(txn *badger.Txn) error {
+		var err error
+		deletionHappened, err = b.delete(txn, evt)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	// after deleting, run garbage collector (sometimes)
+	if deletionHappened {
+		serialDelete = (serialDelete + 1) % 256
+		if serialDelete == 0 {
+			if err := b.RunValueLogGC(0.8); err != nil && err != badger.ErrNoRewrite {
+				log.Println("badger gc errored:" + err.Error())
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *BadgerBackend) delete(txn *badger.Txn, evt *nostr.Event) (bool, error) {
+	idx := make([]byte, 1, 5)
+	idx[0] = rawEventStorePrefix
+
+	// query event by id to get its idx
+	idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+	prefix := make([]byte, 1+8)
+	prefix[0] = indexIdPrefix
+	copy(prefix[1:], idPrefix8)
+	opts := badger.IteratorOptions{
+		PrefetchValues: false,
+	}
+	it := txn.NewIterator(opts)
+	it.Seek(prefix)
+	if it.ValidForPrefix(prefix) {
+		idx = append(idx, it.Item().Key()[1+8:]...)
+	}
+	it.Close()
+
+	// if no idx was found, end here, this event doesn't exist
+	if len(idx) == 1 {
+		return false, nil
+	}
+
+	// calculate all index keys we have for this event and delete them
+	for k := range b.getIndexKeysForEvent(evt, idx[1:]) {
+		if err := txn.Delete(k); err != nil {
+			return false, err
+		}
+	}
+
+	// delete the raw event
+	return true, txn.Delete(idx)
+}

eventstore/badger/fuzz_test.go

@@ -0,0 +1,158 @@
+package badger
+
+import (
+	"cmp"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
+)
+
+func FuzzQuery(f *testing.F) {
+	ctx := context.Background()
+
+	f.Add(uint(200), uint(50), uint(13), uint(2), uint(2), uint(0), uint(1))
+	f.Fuzz(func(t *testing.T, total, limit, authors, timestampAuthorFactor, seedFactor, kinds, kindFactor uint) {
+		total++
+		authors++
+		seedFactor++
+		kindFactor++
+		if kinds == 1 {
+			kinds++
+		}
+		if limit == 0 {
+			return
+		}
+
+		// ~ setup db
+
+		bdb, err := badger.Open(badger.DefaultOptions("").WithInMemory(true))
+		if err != nil {
+			t.Fatalf("failed to create database: %s", err)
+			return
+		}
+		db := &BadgerBackend{}
+		db.DB = bdb
+
+		if err := db.runMigrations(); err != nil {
+			t.Fatalf("error: %s", err)
+			return
+		}
+
+		if err := db.DB.View(func(txn *badger.Txn) error {
+			it := txn.NewIterator(badger.IteratorOptions{
+				Prefix:  []byte{0},
+				Reverse: true,
+			})
+			it.Seek([]byte{1})
+			if it.Valid() {
+				key := it.Item().Key()
+				idx := key[1:]
+				serial := binary.BigEndian.Uint32(idx)
+				db.serial.Store(serial)
+			}
+			it.Close()
+			return nil
+		}); err != nil {
+			t.Fatalf("failed to initialize serial: %s", err)
+			return
+		}
+
+		db.MaxLimit = 500
+		defer db.Close()
+
+		// ~ start actual test
+
+		filter := nostr.Filter{
+			Authors: make([]string, authors),
+			Limit:   int(limit),
+		}
+		maxKind := 1
+		if kinds > 0 {
+			filter.Kinds = make([]int, kinds)
+			for i := range filter.Kinds {
+				filter.Kinds[i] = int(kindFactor) * i
+			}
+			maxKind = filter.Kinds[len(filter.Kinds)-1]
+		}
+
+		for i := 0; i < int(authors); i++ {
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, uint32(i%int(authors*seedFactor))+1)
+			pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
+			filter.Authors[i] = pk
+		}
+
+		expected := make([]*nostr.Event, 0, total)
+		for i := 0; i < int(total); i++ {
+			skseed := uint32(i%int(authors*seedFactor)) + 1
+			sk := make([]byte, 32)
+			binary.BigEndian.PutUint32(sk, skseed)
+
+			evt := &nostr.Event{
+				CreatedAt: nostr.Timestamp(skseed)*nostr.Timestamp(timestampAuthorFactor) + nostr.Timestamp(i),
+				Content:   fmt.Sprintf("unbalanced %d", i),
+				Tags:      nostr.Tags{},
+				Kind:      i % maxKind,
+			}
+			err := evt.Sign(hex.EncodeToString(sk))
+			require.NoError(t, err)
+
+			err = db.SaveEvent(ctx, evt)
+			require.NoError(t, err)
+
+			if filter.Matches(evt) {
+				expected = append(expected, evt)
+			}
+		}
+
+		slices.SortFunc(expected, nostr.CompareEventPtrReverse)
+		if len(expected) > int(limit) {
+			expected = expected[0:limit]
+		}
+
+		w := eventstore.RelayWrapper{Store: db}
+
+		start := time.Now()
+		// fmt.Println(filter)
+		res, err := w.QuerySync(ctx, filter)
+		end := time.Now()
+
+		require.NoError(t, err)
+		require.Equal(t, len(expected), len(res), "number of results is different than expected")
+
+		require.Less(t, end.Sub(start).Milliseconds(), int64(1500), "query took too long")
+		require.True(t, slices.IsSortedFunc(res, func(a, b *nostr.Event) int { return cmp.Compare(b.CreatedAt, a.CreatedAt) }), "results are not sorted")
+
+		nresults := len(expected)
+
+		getTimestamps := func(events []*nostr.Event) []nostr.Timestamp {
+			res := make([]nostr.Timestamp, len(events))
+			for i, evt := range events {
+				res[i] = evt.CreatedAt
+			}
+			return res
+		}
+
+		// fmt.Println(" expected   result")
+		// for i := range expected {
+		// 	fmt.Println(" ", expected[i].CreatedAt, expected[i].ID[0:8], "  ", res[i].CreatedAt, res[i].ID[0:8], "           ", i)
+		// }
+
+		require.Equal(t, expected[0].CreatedAt, res[0].CreatedAt, "first result is wrong")
+		require.Equal(t, expected[nresults-1].CreatedAt, res[nresults-1].CreatedAt, "last result is wrong")
+		require.Equal(t, getTimestamps(expected), getTimestamps(res))
+
+		for _, evt := range res {
+			require.True(t, filter.Matches(evt), "event %s doesn't match filter %s", evt, filter)
+		}
+	})
+}

eventstore/badger/helpers.go

@@ -0,0 +1,162 @@
+package badger
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"iter"
+	"strconv"
+	"strings"
+
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+func getTagIndexPrefix(tagValue string) ([]byte, int) {
+	var k []byte   // the key with full length for created_at and idx at the end, but not filled with these
+	var offset int // the offset -- i.e. where the prefix ends and the created_at and idx would start
+
+	if kind, pkb, d := getAddrTagElements(tagValue); len(pkb) == 32 {
+		// store value in the new special "a" tag index
+		k = make([]byte, 1+2+8+len(d)+4+4)
+		k[0] = indexTagAddrPrefix
+		binary.BigEndian.PutUint16(k[1:], kind)
+		copy(k[1+2:], pkb[0:8])
+		copy(k[1+2+8:], d)
+		offset = 1 + 2 + 8 + len(d)
+	} else if vb, _ := hex.DecodeString(tagValue); len(vb) == 32 {
+		// store value as bytes
+		k = make([]byte, 1+8+4+4)
+		k[0] = indexTag32Prefix
+		copy(k[1:], vb[0:8])
+		offset = 1 + 8
+	} else {
+		// store whatever as utf-8
+		k = make([]byte, 1+len(tagValue)+4+4)
+		k[0] = indexTagPrefix
+		copy(k[1:], tagValue)
+		offset = 1 + len(tagValue)
+	}
+
+	return k, offset
+}
+
+func (b *BadgerBackend) getIndexKeysForEvent(evt *nostr.Event, idx []byte) iter.Seq[[]byte] {
+	return func(yield func([]byte) bool) {
+		{
+			// ~ by id
+			idPrefix8, _ := hex.DecodeString(evt.ID[0 : 8*2])
+			k := make([]byte, 1+8+4)
+			k[0] = indexIdPrefix
+			copy(k[1:], idPrefix8)
+			copy(k[1+8:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+date
+			pubkeyPrefix8, _ := hex.DecodeString(evt.PubKey[0 : 8*2])
+			k := make([]byte, 1+8+4+4)
+			k[0] = indexPubkeyPrefix
+			copy(k[1:], pubkeyPrefix8)
+			binary.BigEndian.PutUint32(k[1+8:], uint32(evt.CreatedAt))
+			copy(k[1+8+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by kind+date
+			k := make([]byte, 1+2+4+4)
+			k[0] = indexKindPrefix
+			binary.BigEndian.PutUint16(k[1:], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[1+2:], uint32(evt.CreatedAt))
+			copy(k[1+2+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by pubkey+kind+date
+			pubkeyPrefix8, _ := hex.DecodeString(evt.PubKey[0 : 8*2])
+			k := make([]byte, 1+8+2+4+4)
+			k[0] = indexPubkeyKindPrefix
+			copy(k[1:], pubkeyPrefix8)
+			binary.BigEndian.PutUint16(k[1+8:], uint16(evt.Kind))
+			binary.BigEndian.PutUint32(k[1+8+2:], uint32(evt.CreatedAt))
+			copy(k[1+8+2+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		// ~ by tagvalue+date
+		customIndex := b.IndexLongerTag != nil
+		customSkip := b.SkipIndexingTag != nil
+
+		for i, tag := range evt.Tags {
+			if len(tag) < 2 || len(tag[0]) != 1 || len(tag[1]) == 0 || len(tag[1]) > 100 {
+				if !customIndex || !b.IndexLongerTag(evt, tag[0], tag[1]) {
+					// not indexable
+					continue
+				}
+			}
+
+			firstIndex := slices.IndexFunc(evt.Tags, func(t nostr.Tag) bool { return len(t) >= 2 && t[1] == tag[1] })
+			if firstIndex != i {
+				// duplicate
+				continue
+			}
+
+			if customSkip && b.SkipIndexingTag(evt, tag[0], tag[1]) {
+				// purposefully skipped
+				continue
+			}
+
+			// get key prefix (with full length) and offset where to write the last parts
+			k, offset := getTagIndexPrefix(tag[1])
+
+			// write the last parts (created_at and idx)
+			binary.BigEndian.PutUint32(k[offset:], uint32(evt.CreatedAt))
+			copy(k[offset+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+
+		{
+			// ~ by date only
+			k := make([]byte, 1+4+4)
+			k[0] = indexCreatedAtPrefix
+			binary.BigEndian.PutUint32(k[1:], uint32(evt.CreatedAt))
+			copy(k[1+4:], idx)
+			if !yield(k) {
+				return
+			}
+		}
+	}
+}
+
+func getAddrTagElements(tagValue string) (kind uint16, pkb []byte, d string) {
+	spl := strings.Split(tagValue, ":")
+	if len(spl) == 3 {
+		if pkb, _ := hex.DecodeString(spl[1]); len(pkb) == 32 {
+			if kind, err := strconv.ParseUint(spl[0], 10, 16); err == nil {
+				return uint16(kind), pkb, spl[2]
+			}
+		}
+	}
+	return 0, nil, ""
+}
+
+func filterMatchesTags(ef *nostr.Filter, event *nostr.Event) bool {
+	for f, v := range ef.Tags {
+		if v != nil && !event.Tags.ContainsAny(f, v) {
+			return false
+		}
+	}
+	return true
+}

eventstore/badger/lib.go

@@ -0,0 +1,100 @@
+package badger
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+const (
+	dbVersionKey          byte = 255
+	rawEventStorePrefix   byte = 0
+	indexCreatedAtPrefix  byte = 1
+	indexIdPrefix         byte = 2
+	indexKindPrefix       byte = 3
+	indexPubkeyPrefix     byte = 4
+	indexPubkeyKindPrefix byte = 5
+	indexTagPrefix        byte = 6
+	indexTag32Prefix      byte = 7
+	indexTagAddrPrefix    byte = 8
+)
+
+var _ eventstore.Store = (*BadgerBackend)(nil)
+
+type BadgerBackend struct {
+	Path                  string
+	MaxLimit              int
+	MaxLimitNegentropy    int
+	BadgerOptionsModifier func(badger.Options) badger.Options
+
+	// Experimental
+	SkipIndexingTag func(event *nostr.Event, tagName string, tagValue string) bool
+	// Experimental
+	IndexLongerTag func(event *nostr.Event, tagName string, tagValue string) bool
+
+	*badger.DB
+
+	serial atomic.Uint32
+}
+
+func (b *BadgerBackend) Init() error {
+	opts := badger.DefaultOptions(b.Path)
+	if b.BadgerOptionsModifier != nil {
+		opts = b.BadgerOptionsModifier(opts)
+	}
+
+	db, err := badger.Open(opts)
+	if err != nil {
+		return err
+	}
+	b.DB = db
+
+	if err := b.runMigrations(); err != nil {
+		return fmt.Errorf("error running migrations: %w", err)
+	}
+
+	if b.MaxLimit != 0 {
+		b.MaxLimitNegentropy = b.MaxLimit
+	} else {
+		b.MaxLimit = 1000
+		if b.MaxLimitNegentropy == 0 {
+			b.MaxLimitNegentropy = 16777216
+		}
+	}
+
+	if err := b.DB.View(func(txn *badger.Txn) error {
+		it := txn.NewIterator(badger.IteratorOptions{
+			Prefix:  []byte{0},
+			Reverse: true,
+		})
+		it.Seek([]byte{1})
+		if it.Valid() {
+			key := it.Item().Key()
+			idx := key[1:]
+			serial := binary.BigEndian.Uint32(idx)
+			b.serial.Store(serial)
+		}
+		it.Close()
+		return nil
+	}); err != nil {
+		return fmt.Errorf("error initializing serial: %w", err)
+	}
+
+	return nil
+}
+
+func (b *BadgerBackend) Close() {
+	b.DB.Close()
+}
+
+func (b *BadgerBackend) Serial() []byte {
+	next := b.serial.Add(1)
+	vb := make([]byte, 5)
+	vb[0] = rawEventStorePrefix
+	binary.BigEndian.PutUint32(vb[1:], next)
+	return vb
+}

eventstore/badger/migrations.go

@@ -0,0 +1,66 @@
+package badger
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/dgraph-io/badger/v4"
+)
+
+func (b *BadgerBackend) runMigrations() error {
+	return b.Update(func(txn *badger.Txn) error {
+		var version uint16
+
+		item, err := txn.Get([]byte{dbVersionKey})
+		if err == badger.ErrKeyNotFound {
+			version = 0
+		} else if err != nil {
+			return err
+		} else {
+			item.Value(func(val []byte) error {
+				version = binary.BigEndian.Uint16(val)
+				return nil
+			})
+		}
+
+		// do the migrations in increasing steps (there is no rollback)
+		//
+
+		// the 3 first migrations go to trash because on version 3 we need to export and import all the data anyway
+		if version < 3 {
+			// if there is any data in the relay we will stop and notify the user,
+			// otherwise we just set version to 3 and proceed
+			prefix := []byte{indexIdPrefix}
+			it := txn.NewIterator(badger.IteratorOptions{
+				PrefetchValues: true,
+				PrefetchSize:   100,
+				Prefix:         prefix,
+			})
+			defer it.Close()
+
+			hasAnyEntries := false
+			for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+				hasAnyEntries = true
+				break
+			}
+
+			if hasAnyEntries {
+				return fmt.Errorf("your database is at version %d, but in order to migrate up to version 3 you must manually export all the events and then import again: run an old version of this software, export the data, then delete the database files, run the new version, import the data back in.", version)
+			}
+
+			b.bumpVersion(txn, 3)
+		}
+
+		if version < 4 {
+			// ...
+		}
+
+		return nil
+	})
+}
+
+func (b *BadgerBackend) bumpVersion(txn *badger.Txn, version uint16) error {
+	buf := make([]byte, 2)
+	binary.BigEndian.PutUint16(buf, version)
+	return txn.Set([]byte{dbVersionKey}, buf)
+}

eventstore/badger/query.go

@@ -0,0 +1,432 @@
+package badger
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/internal"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+	"golang.org/x/exp/slices"
+)
+
+var batchFilled = errors.New("batch-filled")
+
+func (b *BadgerBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
+	ch := make(chan *nostr.Event)
+
+	if filter.Search != "" {
+		close(ch)
+		return ch, nil
+	}
+
+	// max number of events we'll return
+	maxLimit := b.MaxLimit
+	var limit int
+	if eventstore.IsNegentropySession(ctx) {
+		maxLimit = b.MaxLimitNegentropy
+		limit = maxLimit
+	} else {
+		limit = maxLimit / 4
+	}
+	if filter.Limit > 0 && filter.Limit <= maxLimit {
+		limit = filter.Limit
+	}
+	if tlimit := nostr.GetTheoreticalLimit(filter); tlimit == 0 {
+		close(ch)
+		return ch, nil
+	} else if tlimit > 0 {
+		limit = tlimit
+	}
+
+	// fmt.Println("limit", limit)
+
+	go b.View(func(txn *badger.Txn) error {
+		defer close(ch)
+
+		results, err := b.query(txn, filter, limit)
+		if err != nil {
+			return err
+		}
+
+		for _, evt := range results {
+			ch <- evt.Event
+		}
+
+		return nil
+	})
+
+	return ch, nil
+}
+
+func (b *BadgerBackend) query(txn *badger.Txn, filter nostr.Filter, limit int) ([]internal.IterEvent, error) {
+	queries, extraFilter, since, err := prepareQueries(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	iterators := make([]*badger.Iterator, len(queries))
+	exhausted := make([]bool, len(queries)) // indicates that a query won't be used anymore
+	results := make([][]internal.IterEvent, len(queries))
+	pulledPerQuery := make([]int, len(queries))
+
+	// these are kept updated so we never pull from the iterator that is at further distance
+	// (i.e. the one that has the oldest event among all)
+	// we will continue to pull from it as soon as some other iterator takes the position
+	oldest := internal.IterEvent{Q: -1}
+
+	secondPhase := false // after we have gathered enough events we will change the way we iterate
+	secondBatch := make([][]internal.IterEvent, 0, len(queries)+1)
+	secondPhaseParticipants := make([]int, 0, len(queries)+1)
+
+	// while merging results in the second phase we will alternate between these two lists
+	//   to avoid having to create new lists all the time
+	var secondPhaseResultsA []internal.IterEvent
+	var secondPhaseResultsB []internal.IterEvent
+	var secondPhaseResultsToggle bool // this is just a dummy thing we use to keep track of the alternating
+	var secondPhaseHasResultsPending bool
+
+	remainingUnexhausted := len(queries) // when all queries are exhausted we can finally end this thing
+	batchSizePerQuery := internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+	firstPhaseTotalPulled := 0
+
+	exhaust := func(q int) {
+		exhausted[q] = true
+		remainingUnexhausted--
+		if q == oldest.Q {
+			oldest = internal.IterEvent{Q: -1}
+		}
+	}
+
+	var firstPhaseResults []internal.IterEvent
+
+	for q := range queries {
+		iterators[q] = txn.NewIterator(badger.IteratorOptions{
+			Reverse:        true,
+			PrefetchValues: false, // we don't even have values, only keys
+			Prefix:         queries[q].prefix,
+		})
+		defer iterators[q].Close()
+		iterators[q].Seek(queries[q].startingPoint)
+		results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
+	}
+
+	// we will reuse this throughout the iteration
+	valIdx := make([]byte, 5)
+
+	// fmt.Println("queries", len(queries))
+
+	for c := 0; ; c++ {
+		batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
+
+		// fmt.Println("  iteration", c, "remaining", remainingUnexhausted, "batchsize", batchSizePerQuery)
+		// we will go through all the iterators in batches until we have pulled all the required results
+		for q, query := range queries {
+			if exhausted[q] {
+				continue
+			}
+			if oldest.Q == q && remainingUnexhausted > 1 {
+				continue
+			}
+			// fmt.Println("    query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
+
+			it := iterators[q]
+			pulledThisIteration := 0
+
+			for {
+				if !it.Valid() {
+					// fmt.Println("      reached end")
+					exhaust(q)
+					break
+				}
+
+				item := it.Item()
+				key := item.Key()
+
+				idxOffset := len(key) - 4 // this is where the idx actually starts
+
+				// "id" indexes don't contain a timestamp
+				if !query.skipTimestamp {
+					createdAt := binary.BigEndian.Uint32(key[idxOffset-4 : idxOffset])
+					if createdAt < since {
+						// fmt.Println("        reached since", createdAt, "<", since)
+						exhaust(q)
+						break
+					}
+				}
+
+				valIdx[0] = rawEventStorePrefix
+				copy(valIdx[1:], key[idxOffset:])
+
+				// fetch actual event
+				item, err := txn.Get(valIdx)
+				if err != nil {
+					if err == badger.ErrDiscardedTxn {
+						return nil, err
+					}
+					log.Printf("badger: failed to get %x based on prefix %x, index key %x from raw event store: %s\n",
+						valIdx, query.prefix, key, err)
+					return nil, err
+				}
+
+				if err := item.Value(func(val []byte) error {
+					// fmt.Println("      event", hex.EncodeToString(val[0:4]), "kind", binary.BigEndian.Uint16(val[132:134]), "author", hex.EncodeToString(val[32:36]), "ts", nostr.Timestamp(binary.BigEndian.Uint32(val[128:132])))
+
+					// check it against pubkeys without decoding the entire thing
+					if extraFilter != nil && extraFilter.Authors != nil &&
+						!slices.Contains(extraFilter.Authors, hex.EncodeToString(val[32:64])) {
+						// fmt.Println("        skipped (authors)")
+						return nil
+					}
+
+					// check it against kinds without decoding the entire thing
+					if extraFilter != nil && extraFilter.Kinds != nil &&
+						!slices.Contains(extraFilter.Kinds, int(binary.BigEndian.Uint16(val[132:134]))) {
+						// fmt.Println("        skipped (kinds)")
+						return nil
+					}
+
+					event := &nostr.Event{}
+					if err := bin.Unmarshal(val, event); err != nil {
+						log.Printf("badger: value read error (id %x): %s\n", val[0:32], err)
+						return err
+					}
+
+					// check if this matches the other filters that were not part of the index
+					if extraFilter != nil && !filterMatchesTags(extraFilter, event) {
+						// fmt.Println("        skipped (filter)", extraFilter, event)
+						return nil
+					}
+
+					// this event is good to be used
+					evt := internal.IterEvent{Event: event, Q: q}
+					//
+					//
+					if secondPhase {
+						// do the process described below at HIWAWVRTP.
+						// if we've reached here this means we've already passed the `since` check.
+						// now we have to eliminate the event currently at the `since` threshold.
+						nextThreshold := firstPhaseResults[len(firstPhaseResults)-2]
+						if oldest.Event == nil {
+							// fmt.Println("          b1")
+							// BRANCH WHEN WE DON'T HAVE THE OLDEST EVENT (BWWDHTOE)
+							// when we don't have the oldest set, we will keep the results
+							//   and not change the cutting point -- it's bad, but hopefully not that bad.
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+						} else if nextThreshold.CreatedAt > oldest.CreatedAt {
+							// fmt.Println("          b2", nextThreshold.CreatedAt, ">", oldest.CreatedAt)
+							// one of the events we have stored is the actual next threshold
+							// eliminate last, update since with oldest
+							firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+							since = uint32(oldest.CreatedAt)
+							// fmt.Println("            new since", since)
+							//  we null the oldest Event as we can't rely on it anymore
+							//   (we'll fall under BWWDHTOE above) until we have a new oldest set.
+							oldest = internal.IterEvent{Q: -1}
+							// anything we got that would be above this won't trigger an update to
+							//   the oldest anyway, because it will be discarded as being after the limit.
+							//
+							// finally
+							// add this to the results to be merged later
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+						} else if nextThreshold.CreatedAt < evt.CreatedAt {
+							// the next last event in the firstPhaseResults is the next threshold
+							// fmt.Println("          b3", nextThreshold.CreatedAt, "<", oldest.CreatedAt)
+							// eliminate last, update since with the antelast
+							firstPhaseResults = firstPhaseResults[0 : len(firstPhaseResults)-1]
+							since = uint32(nextThreshold.CreatedAt)
+							// fmt.Println("            new since", since)
+							// add this to the results to be merged later
+							results[q] = append(results[q], evt)
+							secondPhaseHasResultsPending = true
+							// update the oldest event
+							if evt.CreatedAt < oldest.CreatedAt {
+								oldest = evt
+							}
+						} else {
+							// fmt.Println("          b4")
+							// oops, _we_ are the next `since` threshold
+							firstPhaseResults[len(firstPhaseResults)-1] = evt
+							since = uint32(evt.CreatedAt)
+							// fmt.Println("            new since", since)
+							// do not add us to the results to be merged later
+							//   as we're already inhabiting the firstPhaseResults slice
+						}
+					} else {
+						results[q] = append(results[q], evt)
+						firstPhaseTotalPulled++
+
+						// update the oldest event
+						if oldest.Event == nil || evt.CreatedAt < oldest.CreatedAt {
+							oldest = evt
+						}
+					}
+
+					pulledPerQuery[q]++
+					pulledThisIteration++
+					if pulledThisIteration > batchSizePerQuery {
+						return batchFilled
+					}
+					if pulledPerQuery[q] >= limit {
+						exhaust(q)
+						return batchFilled
+					}
+
+					return nil
+				}); err == batchFilled {
+					// fmt.Println("      #")
+					it.Next()
+					break
+				} else if err != nil {
+					return nil, fmt.Errorf("iteration error: %w", err)
+				}
+
+				it.Next()
+			}
+		}
+
+		// we will do this check if we don't accumulated the requested number of events yet
+		// fmt.Println("oldest", oldest.Event, "from iter", oldest.Q)
+		if secondPhase && secondPhaseHasResultsPending && (oldest.Event == nil || remainingUnexhausted == 0) {
+			// fmt.Println("second phase aggregation!")
+			// when we are in the second phase we will aggressively aggregate results on every iteration
+			//
+			secondBatch = secondBatch[:0]
+			for s := 0; s < len(secondPhaseParticipants); s++ {
+				q := secondPhaseParticipants[s]
+
+				if len(results[q]) > 0 {
+					secondBatch = append(secondBatch, results[q])
+				}
+
+				if exhausted[q] {
+					secondPhaseParticipants = internal.SwapDelete(secondPhaseParticipants, s)
+					s--
+				}
+			}
+
+			// every time we get here we will alternate between these A and B lists
+			//   combining everything we have into a new partial results list.
+			// after we've done that we can again set the oldest.
+			// fmt.Println("  xxx", secondPhaseResultsToggle)
+			if secondPhaseResultsToggle {
+				secondBatch = append(secondBatch, secondPhaseResultsB)
+				secondPhaseResultsA = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsA)
+				oldest = secondPhaseResultsA[len(secondPhaseResultsA)-1]
+				// fmt.Println("  new aggregated a", len(secondPhaseResultsB))
+			} else {
+				secondBatch = append(secondBatch, secondPhaseResultsA)
+				secondPhaseResultsB = internal.MergeSortMultiple(secondBatch, limit, secondPhaseResultsB)
+				oldest = secondPhaseResultsB[len(secondPhaseResultsB)-1]
+				// fmt.Println("  new aggregated b", len(secondPhaseResultsB))
+			}
+			secondPhaseResultsToggle = !secondPhaseResultsToggle
+
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("  new since", since)
+
+			// reset the `results` list so we can keep using it
+			results = results[:len(queries)]
+			for _, q := range secondPhaseParticipants {
+				results[q] = results[q][:0]
+			}
+		} else if !secondPhase && firstPhaseTotalPulled >= limit && remainingUnexhausted > 0 {
+			// fmt.Println("have enough!", firstPhaseTotalPulled, "/", limit, "remaining", remainingUnexhausted)
+
+			// we will exclude this oldest number as it is not relevant anymore
+			// (we now want to keep track only of the oldest among the remaining iterators)
+			oldest = internal.IterEvent{Q: -1}
+
+			// HOW IT WORKS AFTER WE'VE REACHED THIS POINT (HIWAWVRTP)
+			// now we can combine the results we have and check what is our current oldest event.
+			// we also discard anything that is after the current cutting point (`limit`).
+			// so if we have [1,2,3], [10, 15, 20] and [7, 21, 49] but we only want 6 total
+			//   we can just keep [1,2,3,7,10,15] and discard [20, 21, 49],
+			//   and also adjust our `since` parameter to `15`, discarding anything we get after it
+			//   and immediately declaring that iterator exhausted.
+			// also every time we get result that is more recent than this updated `since` we can
+			//   keep it but also discard the previous since, moving the needle one back -- for example,
+			//   if we get an `8` we can keep it and move the `since` parameter to `10`, discarding `15`
+			//   in the process.
+			all := make([][]internal.IterEvent, len(results))
+			copy(all, results) // we have to use this otherwise mergeSortMultiple will scramble our results slice
+			firstPhaseResults = internal.MergeSortMultiple(all, limit, nil)
+			oldest = firstPhaseResults[limit-1]
+			since = uint32(oldest.CreatedAt)
+			// fmt.Println("new since", since)
+
+			for q := range queries {
+				if exhausted[q] {
+					continue
+				}
+
+				// we also automatically exhaust any of the iterators that have already passed the
+				// cutting point (`since`)
+				if results[q][len(results[q])-1].CreatedAt < oldest.CreatedAt {
+					exhausted[q] = true
+					remainingUnexhausted--
+					continue
+				}
+
+				// for all the remaining iterators,
+				// since we have merged all the events in this `firstPhaseResults` slice, we can empty the
+				//   current `results` slices and reuse them.
+				results[q] = results[q][:0]
+
+				// build this index of indexes with everybody who remains
+				secondPhaseParticipants = append(secondPhaseParticipants, q)
+			}
+
+			// we create these two lists and alternate between them so we don't have to create a
+			//   a new one every time
+			secondPhaseResultsA = make([]internal.IterEvent, 0, limit*2)
+			secondPhaseResultsB = make([]internal.IterEvent, 0, limit*2)
+
+			// from now on we won't run this block anymore
+			secondPhase = true
+		}
+
+		// fmt.Println("remaining", remainingUnexhausted)
+		if remainingUnexhausted == 0 {
+			break
+		}
+	}
+
+	// fmt.Println("is secondPhase?", secondPhase)
+
+	var combinedResults []internal.IterEvent
+
+	if secondPhase {
+		// fmt.Println("ending second phase")
+		// when we reach this point either secondPhaseResultsA or secondPhaseResultsB will be full of stuff,
+		//   the other will be empty
+		var secondPhaseResults []internal.IterEvent
+		// fmt.Println("xxx", secondPhaseResultsToggle, len(secondPhaseResultsA), len(secondPhaseResultsB))
+		if secondPhaseResultsToggle {
+			secondPhaseResults = secondPhaseResultsB
+			combinedResults = secondPhaseResultsA[0:limit] // reuse this
+			// fmt.Println("  using b", len(secondPhaseResultsA))
+		} else {
+			secondPhaseResults = secondPhaseResultsA
+			combinedResults = secondPhaseResultsB[0:limit] // reuse this
+			// fmt.Println("  using a", len(secondPhaseResultsA))
+		}
+
+		all := [][]internal.IterEvent{firstPhaseResults, secondPhaseResults}
+		combinedResults = internal.MergeSortMultiple(all, limit, combinedResults)
+		// fmt.Println("final combinedResults", len(combinedResults), cap(combinedResults), limit)
+	} else {
+		combinedResults = make([]internal.IterEvent, limit)
+		combinedResults = internal.MergeSortMultiple(results, limit, combinedResults)
+	}
+
+	return combinedResults, nil
+}

eventstore/badger/query_planner.go

@@ -0,0 +1,147 @@
+package badger
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type query struct {
+	i             int
+	prefix        []byte
+	startingPoint []byte
+	skipTimestamp bool
+}
+
+func prepareQueries(filter nostr.Filter) (
+	queries []query,
+	extraFilter *nostr.Filter,
+	since uint32,
+	err error,
+) {
+	// these things have to run for every result we return
+	defer func() {
+		if queries == nil {
+			return
+		}
+
+		var until uint32 = 4294967295
+		if filter.Until != nil {
+			if fu := uint32(*filter.Until); fu < until {
+				until = fu + 1
+			}
+		}
+
+		for i, q := range queries {
+			queries[i].startingPoint = binary.BigEndian.AppendUint32(q.prefix, uint32(until))
+		}
+
+		// this is where we'll end the iteration
+		if filter.Since != nil {
+			if fs := uint32(*filter.Since); fs > since {
+				since = fs
+			}
+		}
+	}()
+
+	var index byte
+
+	if len(filter.IDs) > 0 {
+		queries = make([]query, len(filter.IDs))
+		for i, idHex := range filter.IDs {
+			prefix := make([]byte, 1+8)
+			prefix[0] = indexIdPrefix
+			if len(idHex) != 64 {
+				return nil, nil, 0, fmt.Errorf("invalid id '%s'", idHex)
+			}
+			hex.Decode(prefix[1:], []byte(idHex[0:8*2]))
+			queries[i] = query{i: i, prefix: prefix, skipTimestamp: true}
+		}
+
+		return queries, extraFilter, since, nil
+	}
+
+	if len(filter.Tags) > 0 {
+		// we will select ONE tag to query with
+		tagKey, tagValues, goodness := internal.ChooseNarrowestTag(filter)
+
+		// we won't use a tag index for this as long as we have something else to match with
+		if goodness < 3 && (len(filter.Authors) > 0 || len(filter.Kinds) > 0) {
+			goto pubkeyMatching
+		}
+
+		queries = make([]query, len(tagValues))
+		for i, value := range tagValues {
+			// get key prefix (with full length) and offset where to write the created_at
+			k, offset := getTagIndexPrefix(value)
+			// remove the last parts part to get just the prefix we want here
+			prefix := k[0:offset]
+			queries[i] = query{i: i, prefix: prefix}
+			i++
+		}
+
+		extraFilter = &nostr.Filter{
+			Kinds:   filter.Kinds,
+			Authors: filter.Authors,
+			Tags:    internal.CopyMapWithoutKey(filter.Tags, tagKey),
+		}
+
+		return queries, extraFilter, since, nil
+	}
+
+pubkeyMatching:
+	if len(filter.Authors) > 0 {
+		if len(filter.Kinds) == 0 {
+			queries = make([]query, len(filter.Authors))
+			for i, pubkeyHex := range filter.Authors {
+				if len(pubkeyHex) != 64 {
+					return nil, nil, 0, fmt.Errorf("invalid pubkey '%s'", pubkeyHex)
+				}
+				prefix := make([]byte, 1+8)
+				prefix[0] = indexPubkeyPrefix
+				hex.Decode(prefix[1:], []byte(pubkeyHex[0:8*2]))
+				queries[i] = query{i: i, prefix: prefix}
+			}
+		} else {
+			queries = make([]query, len(filter.Authors)*len(filter.Kinds))
+			i := 0
+			for _, pubkeyHex := range filter.Authors {
+				for _, kind := range filter.Kinds {
+					if len(pubkeyHex) != 64 {
+						return nil, nil, 0, fmt.Errorf("invalid pubkey '%s'", pubkeyHex)
+					}
+
+					prefix := make([]byte, 1+8+2)
+					prefix[0] = indexPubkeyKindPrefix
+					hex.Decode(prefix[1:], []byte(pubkeyHex[0:8*2]))
+					binary.BigEndian.PutUint16(prefix[1+8:], uint16(kind))
+					queries[i] = query{i: i, prefix: prefix}
+					i++
+				}
+			}
+		}
+		extraFilter = &nostr.Filter{Tags: filter.Tags}
+	} else if len(filter.Kinds) > 0 {
+		index = indexKindPrefix
+		queries = make([]query, len(filter.Kinds))
+		for i, kind := range filter.Kinds {
+			prefix := make([]byte, 1+2)
+			prefix[0] = index
+			binary.BigEndian.PutUint16(prefix[1:], uint16(kind))
+			queries[i] = query{i: i, prefix: prefix}
+		}
+		extraFilter = &nostr.Filter{Tags: filter.Tags}
+	} else {
+		index = indexCreatedAtPrefix
+		queries = make([]query, 1)
+		prefix := make([]byte, 1)
+		prefix[0] = index
+		queries[0] = query{i: 0, prefix: prefix}
+		extraFilter = nil
+	}
+
+	return queries, extraFilter, since, nil
+}

eventstore/badger/replace.go

@@ -0,0 +1,49 @@
+package badger
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore/internal"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BadgerBackend) ReplaceEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.Update(func(txn *badger.Txn) error {
+		filter := nostr.Filter{Limit: 1, Kinds: []int{evt.Kind}, Authors: []string{evt.PubKey}}
+		if nostr.IsAddressableKind(evt.Kind) {
+			// when addressable, add the "d" tag to the filter
+			filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
+		}
+
+		// now we fetch the past events, whatever they are, delete them and then save the new
+		results, err := b.query(txn, filter, 10) // in theory limit could be just 1 and this should work
+		if err != nil {
+			return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+		}
+
+		shouldStore := true
+		for _, previous := range results {
+			if internal.IsOlder(previous.Event, evt) {
+				if _, err := b.delete(txn, previous.Event); err != nil {
+					return fmt.Errorf("failed to delete event %s for replacing: %w", previous.Event.ID, err)
+				}
+			} else {
+				// there is a newer event already stored, so we won't store this
+				shouldStore = false
+			}
+		}
+		if shouldStore {
+			return b.save(txn, evt)
+		}
+
+		return nil
+	})
+}

eventstore/badger/save.go

@@ -0,0 +1,59 @@
+package badger
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"math"
+
+	"github.com/dgraph-io/badger/v4"
+	"github.com/fiatjaf/eventstore"
+	bin "github.com/fiatjaf/eventstore/internal/binary"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *BadgerBackend) SaveEvent(ctx context.Context, evt *nostr.Event) error {
+	// sanity checking
+	if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
+		return fmt.Errorf("event with values out of expected boundaries")
+	}
+
+	return b.Update(func(txn *badger.Txn) error {
+		// query event by id to ensure we don't save duplicates
+		id, _ := hex.DecodeString(evt.ID)
+		prefix := make([]byte, 1+8)
+		prefix[0] = indexIdPrefix
+		copy(prefix[1:], id)
+		it := txn.NewIterator(badger.IteratorOptions{})
+		defer it.Close()
+		it.Seek(prefix)
+		if it.ValidForPrefix(prefix) {
+			// event exists
+			return eventstore.ErrDupEvent
+		}
+
+		return b.save(txn, evt)
+	})
+}
+
+func (b *BadgerBackend) save(txn *badger.Txn, evt *nostr.Event) error {
+	// encode to binary
+	bin, err := bin.Marshal(evt)
+	if err != nil {
+		return err
+	}
+
+	idx := b.Serial()
+	// raw event store
+	if err := txn.Set(idx, bin); err != nil {
+		return err
+	}
+
+	for k := range b.getIndexKeysForEvent(evt, idx[1:]) {
+		if err := txn.Set(k, nil); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

eventstore/badger/testdata/fuzz/FuzzQuery

@@ -0,0 +1 @@
+../../../internal/testdata/fuzz/FuzzQuery