eventstore tests.

eventstore/mmm/fuzz_test.go

@@ -1,8 +1,8 @@
 package mmm
 
 import (
-	"context"
 	"fmt"
+	"iter"
 	"math/rand/v2"
 	"os"
 	"slices"
@@ -47,11 +47,10 @@ func FuzzTest(f *testing.F) {
 		}
 
 		// create test events
-		ctx := context.Background()
-		sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
-		storedIds := make([]string, nevents)
-		nTags := make(map[string]int)
-		storedByLayer := make(map[string][]string)
+		sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
+		storedIds := make([]nostr.ID, nevents)
+		nTags := make(map[nostr.ID]int)
+		storedByLayer := make(map[string][]nostr.ID)
 
 		// create n events with random combinations of tags
 		for i := 0; i < int(nevents); i++ {
@@ -68,9 +67,9 @@ func FuzzTest(f *testing.F) {
 				}
 			}
 
-			evt := &nostr.Event{
+			evt := nostr.Event{
 				CreatedAt: nostr.Timestamp(i),
-				Kind:      i, // hack to query by serial id
+				Kind:      uint16(i), // hack to query by serial id
 				Tags:      tags,
 				Content:   fmt.Sprintf("test content %d", i),
 			}
@@ -78,23 +77,20 @@ func FuzzTest(f *testing.F) {
 
 			for _, layer := range mmm.layers {
 				if evt.Tags.FindWithValue("t", layer.name) != nil {
-					err := layer.SaveEvent(ctx, evt)
+					err := layer.SaveEvent(evt)
 					require.NoError(t, err)
 					storedByLayer[layer.name] = append(storedByLayer[layer.name], evt.ID)
 				}
 			}
 
-			storedIds = append(storedIds, evt.ID)
+			storedIds[i] = evt.ID
 			nTags[evt.ID] = len(evt.Tags)
 		}
 
 		// verify each layer has the correct events
 		for _, layer := range mmm.layers {
-			results, err := layer.QueryEvents(ctx, nostr.Filter{})
-			require.NoError(t, err)
-
 			count := 0
-			for evt := range results {
+			for evt := range layer.QueryEvents(nostr.Filter{}) {
 				require.True(t, evt.Tags.ContainsAny("t", []string{layer.name}))
 				count++
 			}
@@ -102,7 +98,7 @@ func FuzzTest(f *testing.F) {
 		}
 
 		// randomly select n events to delete from random layers
-		deleted := make(map[string][]*IndexingLayer)
+		deleted := make(map[nostr.ID][]*IndexingLayer)
 
 		for range ndeletes {
 			id := storedIds[rnd.Int()%len(storedIds)]
@@ -117,7 +113,7 @@ func FuzzTest(f *testing.F) {
 				require.Contains(t, layers, layer)
 
 				// delete now
-				layer.DeleteEvent(ctx, evt)
+				layer.DeleteEvent(evt.ID)
 				deleted[id] = append(deleted[id], layer)
 			} else {
 				// was never saved to this in the first place
@@ -152,16 +148,16 @@ func FuzzTest(f *testing.F) {
 				for _, layer := range mmm.layers {
 					// verify event still accessible from other layers
 					if slices.Contains(foundlayers, layer) {
-						ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
-						require.NoError(t, err)
-						fetched := <-ch
-						require.NotNil(t, fetched)
+						next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []uint16{evt.Kind}})) // hack
+						_, fetched := next()
+						require.True(t, fetched)
+						stop()
 					} else {
 						// and not accessible from this layer we just deleted
-						ch, err := layer.QueryEvents(ctx, nostr.Filter{Kinds: []int{evt.Kind}}) // hack
-						require.NoError(t, err)
-						fetched := <-ch
-						require.Nil(t, fetched)
+						next, stop := iter.Pull(layer.QueryEvents(nostr.Filter{Kinds: []uint16{evt.Kind}})) // hack
+						_, fetched := next()
+						require.True(t, fetched)
+						stop()
 					}
 				}
 			}
@@ -169,11 +165,8 @@ func FuzzTest(f *testing.F) {
 
 		// now delete a layer and events that only exist in that layer should vanish
 		layer := mmm.layers[rnd.Int()%len(mmm.layers)]
-		ch, err := layer.QueryEvents(ctx, nostr.Filter{})
-		require.NoError(t, err)
-
-		eventsThatShouldVanish := make([]string, 0, nevents/2)
-		for evt := range ch {
+		eventsThatShouldVanish := make([]nostr.ID, 0, nevents/2)
+		for evt := range layer.QueryEvents(nostr.Filter{}) {
 			if len(evt.Tags) == 1+len(deleted[evt.ID]) {
 				eventsThatShouldVanish = append(eventsThatShouldVanish, evt.ID)
 			}

eventstore/mmm/helpers.go

@@ -51,7 +51,7 @@ func (il *IndexingLayer) getIndexKeysForEvent(evt nostr.Event) iter.Seq[key] {
 		{
 			// ~ by pubkey+date
 			k := make([]byte, 8+4)
-			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			copy(k[0:8], evt.PubKey[0:8])
 			binary.BigEndian.PutUint32(k[8:8+4], uint32(evt.CreatedAt))
 			if !yield(key{dbi: il.indexPubkey, key: k[0 : 8+4]}) {
 				return
@@ -71,7 +71,7 @@ func (il *IndexingLayer) getIndexKeysForEvent(evt nostr.Event) iter.Seq[key] {
 		{
 			// ~ by pubkey+kind+date
 			k := make([]byte, 8+2+4)
-			hex.Decode(k[0:8], []byte(evt.PubKey[0:8*2]))
+			copy(k[0:8], evt.PubKey[0:8])
 			binary.BigEndian.PutUint16(k[8:8+2], uint16(evt.Kind))
 			binary.BigEndian.PutUint32(k[8+2:8+2+4], uint32(evt.CreatedAt))
 			if !yield(key{dbi: il.indexPubkeyKind, key: k[0 : 8+2+4]}) {

eventstore/mmm/mmmm_test.go

@@ -1,386 +0,0 @@
-package mmm
-
-import (
-	"context"
-	"encoding/binary"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/PowerDNS/lmdb-go/lmdb"
-	"fiatjaf.com/nostr"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMultiLayerIndexing(t *testing.T) {
-	// Create a temporary directory for the test
-	tmpDir := "/tmp/eventstore-mmm-test"
-	os.RemoveAll(tmpDir)
-
-	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
-
-	// initialize MMM with three layers:
-	// 1. odd timestamps layer
-	// 2. even timestamps layer
-	// 3. all events layer
-	mmm := &MultiMmapManager{
-		Dir:    tmpDir,
-		Logger: &logger,
-	}
-
-	err := mmm.Init()
-	require.NoError(t, err)
-	defer mmm.Close()
-
-	// create layers
-	err = mmm.EnsureLayer("odd", &IndexingLayer{
-		MaxLimit: 100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
-			return evt.CreatedAt%2 == 1
-		},
-	})
-	require.NoError(t, err)
-	err = mmm.EnsureLayer("even", &IndexingLayer{
-		MaxLimit: 100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
-			return evt.CreatedAt%2 == 0
-		},
-	})
-	require.NoError(t, err)
-	err = mmm.EnsureLayer("all", &IndexingLayer{
-		MaxLimit: 100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool {
-			return true
-		},
-	})
-	require.NoError(t, err)
-
-	// create test events
-	ctx := context.Background()
-	baseTime := nostr.Timestamp(0)
-	sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
-	events := make([]*nostr.Event, 10)
-	for i := 0; i < 10; i++ {
-		evt := &nostr.Event{
-			CreatedAt: baseTime + nostr.Timestamp(i),
-			Kind:      1,
-			Tags:      nostr.Tags{},
-			Content:   "test content",
-		}
-		evt.Sign(sk)
-		events[i] = evt
-		stored, err := mmm.StoreGlobal(ctx, evt)
-		require.NoError(t, err)
-		require.True(t, stored)
-	}
-
-	{
-		// query odd layer
-		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-
-		oddCount := 0
-		for evt := range oddResults {
-			require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(1))
-			oddCount++
-		}
-		require.Equal(t, 5, oddCount)
-	}
-
-	{
-		// query even layer
-		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-
-		evenCount := 0
-		for evt := range evenResults {
-			require.Equal(t, evt.CreatedAt%2, nostr.Timestamp(0))
-			evenCount++
-		}
-		require.Equal(t, 5, evenCount)
-	}
-
-	{
-		// query all layer
-		allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-
-		allCount := 0
-		for range allResults {
-			allCount++
-		}
-		require.Equal(t, 10, allCount)
-	}
-
-	// delete some events
-	err = mmm.layers[0].DeleteEvent(ctx, events[1]) // odd timestamp
-	require.NoError(t, err)
-	err = mmm.layers[1].DeleteEvent(ctx, events[2]) // even timestamp
-
-	// verify deletions
-	{
-		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-		oddCount := 0
-		for range oddResults {
-			oddCount++
-		}
-		require.Equal(t, 4, oddCount)
-	}
-
-	{
-		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-		evenCount := 0
-		for range evenResults {
-			evenCount++
-		}
-		require.Equal(t, 4, evenCount)
-	}
-
-	{
-		allResults, err := mmm.layers[2].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-		allCount := 0
-		for range allResults {
-			allCount++
-		}
-		require.Equal(t, 10, allCount)
-	}
-
-	// save events directly to layers regardless of timestamp
-	{
-		oddEvent := &nostr.Event{
-			CreatedAt: baseTime + 100, // even timestamp
-			Kind:      1,
-			Content:   "forced odd",
-		}
-		oddEvent.Sign(sk)
-		err = mmm.layers[0].SaveEvent(ctx, oddEvent) // save even timestamp to odd layer
-		require.NoError(t, err)
-
-		// it is added to the odd il
-		oddResults, err := mmm.layers[0].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-		oddCount := 0
-		for range oddResults {
-			oddCount++
-		}
-		require.Equal(t, 5, oddCount)
-
-		// it doesn't affect the event il
-		evenResults, err := mmm.layers[1].QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{1},
-		})
-		require.NoError(t, err)
-		evenCount := 0
-		for range evenResults {
-			evenCount++
-		}
-		require.Equal(t, 4, evenCount)
-	}
-
-	// test replaceable events
-	for _, layer := range mmm.layers {
-		replaceable := &nostr.Event{
-			CreatedAt: baseTime + 0,
-			Kind:      0,
-			Content:   fmt.Sprintf("first"),
-		}
-		replaceable.Sign(sk)
-		err := layer.ReplaceEvent(ctx, replaceable)
-		require.NoError(t, err)
-	}
-
-	// replace events alternating between layers
-	for i := range mmm.layers {
-		content := fmt.Sprintf("last %d", i)
-
-		newEvt := &nostr.Event{
-			CreatedAt: baseTime + 1000,
-			Kind:      0,
-			Content:   content,
-		}
-		newEvt.Sign(sk)
-
-		layer := mmm.layers[i]
-		err = layer.ReplaceEvent(ctx, newEvt)
-		require.NoError(t, err)
-
-		// verify replacement in the layer that did it
-		results, err := layer.QueryEvents(ctx, nostr.Filter{
-			Kinds: []int{0},
-		})
-		require.NoError(t, err)
-
-		count := 0
-		for evt := range results {
-			require.Equal(t, content, evt.Content)
-			count++
-		}
-		require.Equal(t, 1, count)
-
-		// verify other layers still have the old version
-		for j := 0; j < 3; j++ {
-			if mmm.layers[j] == layer {
-				continue
-			}
-			results, err := mmm.layers[j].QueryEvents(ctx, nostr.Filter{
-				Kinds: []int{0},
-			})
-			require.NoError(t, err)
-
-			count := 0
-			for evt := range results {
-				if i < j {
-					require.Equal(t, "first", evt.Content)
-				} else {
-					require.Equal(t, evt.Content, fmt.Sprintf("last %d", j))
-				}
-				count++
-			}
-
-			require.Equal(t, 1, count, "%d/%d", i, j)
-		}
-	}
-}
-
-func TestLayerReferenceTracking(t *testing.T) {
-	// Create a temporary directory for the test
-	tmpDir, err := os.MkdirTemp("", "mmm-test-*")
-	require.NoError(t, err)
-	defer os.RemoveAll(tmpDir)
-
-	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr})
-
-	// initialize MMM with three layers
-	mmm := &MultiMmapManager{
-		Dir:    tmpDir,
-		Logger: &logger,
-	}
-
-	err = mmm.Init()
-	require.NoError(t, err)
-	defer mmm.Close()
-
-	// create three layers
-	err = mmm.EnsureLayer("layer1", &IndexingLayer{
-		MaxLimit:    100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
-	})
-	require.NoError(t, err)
-	err = mmm.EnsureLayer("layer2", &IndexingLayer{
-		MaxLimit:    100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
-	})
-	require.NoError(t, err)
-	err = mmm.EnsureLayer("layer3", &IndexingLayer{
-		MaxLimit:    100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
-	})
-	require.NoError(t, err)
-	err = mmm.EnsureLayer("layer4", &IndexingLayer{
-		MaxLimit:    100,
-		ShouldIndex: func(ctx context.Context, evt *nostr.Event) bool { return true },
-	})
-	require.NoError(t, err)
-
-	// create test events
-	ctx := context.Background()
-	sk := "945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb"
-	evt1 := &nostr.Event{
-		CreatedAt: 1000,
-		Kind:      1,
-		Tags:      nostr.Tags{},
-		Content:   "event 1",
-	}
-	evt1.Sign(sk)
-
-	evt2 := &nostr.Event{
-		CreatedAt: 2000,
-		Kind:      1,
-		Tags:      nostr.Tags{},
-		Content:   "event 2",
-	}
-	evt2.Sign(sk)
-
-	// save evt1 to layer1
-	err = mmm.layers[0].SaveEvent(ctx, evt1)
-	require.NoError(t, err)
-
-	// save evt1 to layer2
-	err = mmm.layers[1].SaveEvent(ctx, evt1)
-	require.NoError(t, err)
-
-	// save evt1 to layer4
-	err = mmm.layers[0].SaveEvent(ctx, evt1)
-	require.NoError(t, err)
-
-	// delete evt1 from layer1
-	err = mmm.layers[0].DeleteEvent(ctx, evt1)
-	require.NoError(t, err)
-
-	// save evt2 to layer3
-	err = mmm.layers[2].SaveEvent(ctx, evt2)
-	require.NoError(t, err)
-
-	// save evt2 to layer4
-	err = mmm.layers[3].SaveEvent(ctx, evt2)
-	require.NoError(t, err)
-
-	// save evt2 to layer3 again
-	err = mmm.layers[2].SaveEvent(ctx, evt2)
-	require.NoError(t, err)
-
-	// delete evt1 from layer4
-	err = mmm.layers[3].DeleteEvent(ctx, evt1)
-	require.NoError(t, err)
-
-	// verify the state of the indexId database
-	err = mmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
-		cursor, err := txn.OpenCursor(mmm.indexId)
-		if err != nil {
-			return err
-		}
-		defer cursor.Close()
-
-		count := 0
-		for k, v, err := cursor.Get(nil, nil, lmdb.First); err == nil; k, v, err = cursor.Get(nil, nil, lmdb.Next) {
-			count++
-			if hex.EncodeToString(k) == evt1.ID[:16] {
-				// evt1 should only reference layer2
-				require.Equal(t, 14, len(v), "evt1 should have one layer reference")
-				layerRef := binary.BigEndian.Uint16(v[12:14])
-				require.Equal(t, mmm.layers[1].id, layerRef, "evt1 should reference layer2")
-			} else if hex.EncodeToString(k) == evt2.ID[:16] {
-				// evt2 should references to layer3 and layer4
-				require.Equal(t, 16, len(v), "evt2 should have two layer references")
-				layer3Ref := binary.BigEndian.Uint16(v[12:14])
-				require.Equal(t, mmm.layers[2].id, layer3Ref, "evt2 should reference layer3")
-				layer4Ref := binary.BigEndian.Uint16(v[14:16])
-				require.Equal(t, mmm.layers[3].id, layer4Ref, "evt2 should reference layer4")
-			} else {
-				t.Errorf("unexpected event in indexId: %x", k)
-			}
-		}
-		require.Equal(t, 2, count, "should have exactly two events in indexId")
-		return nil
-	})
-	require.NoError(t, err)
-}

eventstore/mmm/query_planner.go

@@ -145,29 +145,19 @@ pubkeyMatching:
 		if len(filter.Kinds) == 0 {
 			// will use pubkey index
 			queries = make([]query, len(filter.Authors))
-			for i, pubkeyHex := range filter.Authors {
-				if len(pubkeyHex) != 64 {
-					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
-				}
+			for i, pk := range filter.Authors {
 				prefix := make([]byte, 8)
-				if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
-					return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
-				}
+				copy(prefix[0:8], pk[0:8])
 				queries[i] = query{i: i, dbi: il.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
 			}
 		} else {
 			// will use pubkeyKind index
 			queries = make([]query, len(filter.Authors)*len(filter.Kinds))
 			i := 0
-			for _, pubkeyHex := range filter.Authors {
+			for _, pk := range filter.Authors {
 				for _, kind := range filter.Kinds {
-					if len(pubkeyHex) != 64 {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
-					}
 					prefix := make([]byte, 8+2)
-					if _, err := hex.Decode(prefix[0:8], []byte(pubkeyHex[0:8*2])); err != nil {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid author '%s'", pubkeyHex)
-					}
+					copy(prefix[0:8], pk[0:8])
 					binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
 					queries[i] = query{i: i, dbi: il.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
 					i++