coracle/nostrlib
2
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eventstore tests.

Browse Source
This commit is contained in:
fiatjaf
2025-04-18 11:27:22 -03:00
parent 32efaa7c58
commit 92c2de6294
32 changed files with 355 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eventstore/lmdb/count.go
+1 -1
View File
@@ -4,13 +4,13 @@ import (
"bytes"
"encoding/binary"
"encoding/hex"
"slices"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore/codec/betterbinary"
"fiatjaf.com/nostr/nip45"
"fiatjaf.com/nostr/nip45/hyperloglog"
"github.com/PowerDNS/lmdb-go/lmdb"
"golang.org/x/exp/slices"
)
func (b *LMDBBackend) CountEvents(filter nostr.Filter) (uint32, error) {
eventstore/lmdb/delete.go
+2 -2
View File
@@ -40,13 +40,13 @@ func (b *LMDBBackend) delete(txn *lmdb.Txn, id nostr.ID) error {
for k := range b.getIndexKeysForEvent(evt) {
err := txn.Del(k.dbi, k.key, idx)
if err != nil {
return fmt.Errorf("failed to delete index entry %s for %x: %w", b.keyName(k), evt.ID[0:8*2], err)
return fmt.Errorf("failed to delete index entry %s for %x: %w", b.keyName(k), evt.ID[0:8], err)
}
}
// delete the raw event
if err := txn.Del(b.rawEventStore, idx, nil); err != nil {
return fmt.Errorf("failed to delete raw event %x (idx %x): %w", evt.ID[0:8*2], idx, err)
return fmt.Errorf("failed to delete raw event %x (idx %x): %w", evt.ID[0:8], idx, err)
}
return nil
eventstore/lmdb/fuzz_test.go
+20 -29
View File
@@ -2,24 +2,19 @@ package lmdb
import (
"cmp"
"context"
"encoding/binary"
"encoding/hex"
"fmt"
"os"
"slices"
"testing"
"time"
"github.com/PowerDNS/lmdb-go/lmdb"
"fiatjaf.com/nostr/eventstore"
"fiatjaf.com/nostr"
"github.com/PowerDNS/lmdb-go/lmdb"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
func FuzzQuery(f *testing.F) {
ctx := context.Background()
f.Add(uint(200), uint(50), uint(13), uint(2), uint(2), uint(0), uint(1))
f.Fuzz(func(t *testing.T, total, limit, authors, timestampAuthorFactor, seedFactor, kinds, kindFactor uint) {
total++
@@ -51,41 +46,41 @@ func FuzzQuery(f *testing.F) {
// ~ start actual test
filter := nostr.Filter{
Authors: make([]string, authors),
Authors: make([]nostr.PubKey, authors),
Limit: int(limit),
}
maxKind := 1
var maxKind uint16 = 1
if kinds > 0 {
filter.Kinds = make([]int, kinds)
filter.Kinds = make([]uint16, kinds)
for i := range filter.Kinds {
filter.Kinds[i] = int(kindFactor) * i
filter.Kinds[i] = uint16(int(kindFactor) * i)
}
maxKind = filter.Kinds[len(filter.Kinds)-1]
}
for i := 0; i < int(authors); i++ {
sk := make([]byte, 32)
binary.BigEndian.PutUint32(sk, uint32(i%int(authors*seedFactor))+1)
pk, _ := nostr.GetPublicKey(hex.EncodeToString(sk))
var sk nostr.SecretKey
binary.BigEndian.PutUint32(sk[:], uint32(i%int(authors*seedFactor))+1)
pk := nostr.GetPublicKey(sk)
filter.Authors[i] = pk
}
expected := make([]*nostr.Event, 0, total)
expected := make([]nostr.Event, 0, total)
for i := 0; i < int(total); i++ {
skseed := uint32(i%int(authors*seedFactor)) + 1
sk := make([]byte, 32)
binary.BigEndian.PutUint32(sk, skseed)
sk := nostr.SecretKey{}
binary.BigEndian.PutUint32(sk[:], skseed)
evt := &nostr.Event{
evt := nostr.Event{
CreatedAt: nostr.Timestamp(skseed)*nostr.Timestamp(timestampAuthorFactor) + nostr.Timestamp(i),
Content: fmt.Sprintf("unbalanced %d", i),
Tags: nostr.Tags{},
Kind: i % maxKind,
Kind: uint16(i) % maxKind,
}
err := evt.Sign(hex.EncodeToString(sk))
err := evt.Sign(sk)
require.NoError(t, err)
err = db.SaveEvent(ctx, evt)
err = db.SaveEvent(evt)
require.NoError(t, err)
if filter.Matches(evt) {
@@ -93,25 +88,21 @@ func FuzzQuery(f *testing.F) {
}
}
slices.SortFunc(expected, nostr.CompareEventPtrReverse)
slices.SortFunc(expected, nostr.CompareEventReverse)
if len(expected) > int(limit) {
expected = expected[0:limit]
}
w := eventstore.RelayWrapper{Store: db}
start := time.Now()
res, err := w.QuerySync(ctx, filter)
res := slices.Collect(db.QueryEvents(filter))
end := time.Now()
require.NoError(t, err)
require.Equal(t, len(expected), len(res), "number of results is different than expected")
require.Less(t, end.Sub(start).Milliseconds(), int64(1500), "query took too long")
nresults := len(expected)
getTimestamps := func(events []*nostr.Event) []nostr.Timestamp {
getTimestamps := func(events []nostr.Event) []nostr.Timestamp {
res := make([]nostr.Timestamp, len(events))
for i, evt := range events {
res[i] = evt.CreatedAt
@@ -132,6 +123,6 @@ func FuzzQuery(f *testing.F) {
require.True(t, filter.Matches(evt), "event %s doesn't match filter %s", evt, filter)
}
require.True(t, slices.IsSortedFunc(res, func(a, b *nostr.Event) int { return cmp.Compare(b.CreatedAt, a.CreatedAt) }), "results are not sorted")
require.True(t, slices.IsSortedFunc(res, func(a, b nostr.Event) int { return cmp.Compare(b.CreatedAt, a.CreatedAt) }), "results are not sorted")
})
}
eventstore/lmdb/helpers.go
+3 -3
View File
@@ -6,12 +6,12 @@ import (
"encoding/hex"
"fmt"
"iter"
"slices"
"strconv"
"strings"
"fiatjaf.com/nostr"
"github.com/PowerDNS/lmdb-go/lmdb"
"golang.org/x/exp/slices"
)
// this iterator always goes backwards
@@ -64,7 +64,7 @@ func (b *LMDBBackend) getIndexKeysForEvent(evt nostr.Event) iter.Seq[key] {
{
// ~ by pubkey+date
k := make([]byte, 8+4)
hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
copy(k[0:8], evt.PubKey[0:8])
binary.BigEndian.PutUint32(k[8:8+4], uint32(evt.CreatedAt))
if !yield(key{dbi: b.indexPubkey, key: k[0 : 8+4]}) {
return
@@ -84,7 +84,7 @@ func (b *LMDBBackend) getIndexKeysForEvent(evt nostr.Event) iter.Seq[key] {
{
// ~ by pubkey+kind+date
k := make([]byte, 8+2+4)
hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
copy(k[0:8], evt.PubKey[0:8])
binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
if !yield(key{dbi: b.indexPubkeyKind, key: k[0 : 8+2+4]}) {
eventstore/lmdb/query.go
+2 -2
View File
@@ -99,7 +99,7 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
results[q] = make([]internal.IterEvent, 0, batchSizePerQuery*2)
}
// fmt.Println("queries", len(queries))
// fmt.Println("queries", filter, len(queries))
for c := 0; ; c++ {
batchSizePerQuery = internal.BatchSizePerNumberOfQueries(limit, remainingUnexhausted)
@@ -113,7 +113,7 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int) ([]in
if oldest.Q == q && remainingUnexhausted > 1 {
continue
}
// fmt.Println(" query", q, unsafe.Pointer(&results[q]), hex.EncodeToString(query.prefix), len(results[q]))
// fmt.Println(" query", q, unsafe.Pointer(&results[q]), b.dbiName(query.dbi), hex.EncodeToString(query.prefix), len(results[q]))
it := iterators[q]
pulledThisIteration := 0
eventstore/lmdb/query_planner.go
+6 -23
View File
@@ -53,14 +53,9 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
if filter.IDs != nil {
// when there are ids we ignore everything else
queries = make([]query, len(filter.IDs))
for i, idHex := range filter.IDs {
if len(idHex) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid id '%s'", idHex)
}
for i, id := range filter.IDs {
prefix := make([]byte, 8)
if _, err := hex.Decode(prefix[0:8], []byte(idHex[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid id '%s'", idHex)
}
copy(prefix[0:8], id[0:8])
queries[i] = query{i: i, dbi: b.indexId, prefix: prefix[0:8], keySize: 8, timestampSize: 0}
}
return queries, nil, nil, "", nil, 0, nil
@@ -161,29 +156,17 @@ pubkeyMatching:
if len(filter.Kinds) == 0 {
// will use pubkey index
queries = make([]query, len(filter.Authors))
for i, pubkeyHex := range filter.Authors {
if len(pubkeyHex) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
prefix := make([]byte, 8)
if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
queries[i] = query{i: i, dbi: b.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
for i, pk := range filter.Authors {
queries[i] = query{i: i, dbi: b.indexPubkey, prefix: pk[0:8], keySize: 8 + 4, timestampSize: 4}
}
} else {
// will use pubkeyKind index
queries = make([]query, len(filter.Authors)*len(filter.Kinds))
i := 0
for _, pubkeyHex := range filter.Authors {
for _, pk := range filter.Authors {
for _, kind := range filter.Kinds {
if len(pubkeyHex) != 64 {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
prefix := make([]byte, 8+2)
if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
}
copy(prefix[0:8], pk[0:8])
binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
queries[i] = query{i: i, dbi: b.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
i++
eventstore/lmdb/replace.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) error {
// sanity checking
if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
return fmt.Errorf("event with values out of expected boundaries")
return fmt.Errorf("event with values out of expected boundaries %d/%d", evt.CreatedAt, evt.Kind)
}
return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
eventstore/lmdb/save.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func (b *LMDBBackend) SaveEvent(evt nostr.Event) error {
// sanity checking
if evt.CreatedAt > math.MaxUint32 || evt.Kind > math.MaxUint16 {
return fmt.Errorf("event with values out of expected boundaries")
return fmt.Errorf("event with values out of expected boundaries %d/%d", evt.CreatedAt, evt.Kind)
}
return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {