coracle/nostrlib
2
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eventstore/mmm: defrag.

Browse Source
This commit is contained in:
fiatjaf
2026-06-15 14:23:54 -03:00
parent 0616b30ab3
commit 0f8843afac
8 changed files with 471 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eventstore/mmm/freeranges_test.go
+283
View File
@@ -1,8 +1,11 @@
package mmm
import (
"cmp"
"fmt"
"math/rand/v2"
"os"
"slices"
"strings"
"testing"
@@ -125,6 +128,100 @@ func FuzzFreeRanges(f *testing.F) {
})
}
func TestDefragment(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "mmm-defrag-test-*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
mmmm := &MultiMmapManager{Dir: tmpDir}
err = mmmm.Init()
require.NoError(t, err)
defer mmmm.Close()
il, err := mmmm.EnsureLayer("a")
require.NoError(t, err)
defer il.Close()
sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
const nevents = 30
var stored [nevents]nostr.Event
for i := range nevents {
evt := nostr.Event{
CreatedAt: nostr.Timestamp(i),
Kind: nostr.KindTextNote,
Tags: nostr.Tags{},
Content: fmt.Sprintf("============= event %d ============= "+strings.Repeat("+", 23), i),
}
evt.Sign(sk)
err := il.SaveEvent(evt)
require.NoError(t, err)
stored[i] = evt
}
toDelete := []int{0, 5, 10, 15, 20}
var remaining []nostr.Event
for i, evt := range stored {
if slices.Contains(toDelete, i) {
err := il.DeleteEvent(evt.ID)
require.NoError(t, err)
} else {
remaining = append(remaining, evt)
}
}
require.Len(t, toDelete, len(mmmm.freeRangesAll))
err = mmmm.Defragment(2)
require.NoError(t, err)
require.Len(t, mmmm.freeRangesAll, 3)
require.Len(t, remaining, nevents-len(toDelete))
// all remaining events still accessible with correct content via GetByID
for _, evt := range remaining {
gotEvt, layers := mmmm.GetByID(evt.ID)
require.NotNil(t, gotEvt, "event %s should exist after defrag", evt.ID)
require.NotEmpty(t, layers, "event %s should have layers after defrag", evt.ID)
require.Equal(t, evt.Content, gotEvt.Content, "event %s content should match after defrag", evt.ID)
// also accessible via a query
require.Equal(t, il, layers[0])
}
evts := slices.Collect(il.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{nostr.KindTextNote}}, 100))
require.Len(t, evts, nevents-len(toDelete))
// free range invariants hold after defrag
verifyFreeRangesInvariants(t, mmmm)
// no overlapping positions after defrag
mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
cursor, err := txn.OpenCursor(mmmm.indexId)
require.NoError(t, err)
defer cursor.Close()
var allPositions []position
for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
pos := positionFromBytes(val[0:12])
allPositions = append(allPositions, pos)
}
slices.SortFunc(allPositions, func(a, b position) int {
return cmp.Compare(a.start, b.start)
})
var lastEnd uint64
for _, pos := range allPositions {
if pos.start < lastEnd {
t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
}
lastEnd = pos.start + uint64(pos.size)
}
return nil
})
}
func countUsableFreeRanges(t *testing.T, mmmm *MultiMmapManager) (count int, space int) {
for _, fr := range mmmm.freeRangesAll {
if fr.size >= LARGE_FREERANGE {
@@ -176,3 +273,189 @@ func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
return nil
})
}
func FuzzDefragment(f *testing.F) {
f.Add(0)
f.Fuzz(func(t *testing.T, seed int) {
tmpDir, err := os.MkdirTemp("", "mmm-defrag-fuzz-*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
logger := zerolog.Nop()
rnd := rand.New(rand.NewPCG(uint64(seed), 0))
mmmm := &MultiMmapManager{
Dir: tmpDir,
Logger: &logger,
}
err = mmmm.Init()
require.NoError(t, err)
defer mmmm.Close()
layerNames := []string{"a", "b", "c"}
var layers []*IndexingLayer
for _, name := range layerNames {
il, err := mmmm.EnsureLayer(name)
require.NoError(t, err)
defer il.Close()
layers = append(layers, il)
}
type indexedEvent struct {
evt nostr.Event
tag string
}
layerEvents := make([][]indexedEvent, len(layers))
sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
pk := sk.Public()
totalEvents := rnd.IntN(500)
tChoices := []string{"foo", "bar", "banana"}
var written int
for written < totalEvents {
n := rnd.IntN(50) + 1
if n > totalEvents-written {
n = totalEvents - written
}
for i := 0; i < n; i++ {
sizeParam := rnd.IntN(2000)
content := strings.Repeat("z", sizeParam)
chosenTag := tChoices[rnd.IntN(3)]
evt := nostr.Event{
CreatedAt: nostr.Timestamp(rnd.Uint32()),
Kind: nostr.KindTextNote,
Content: content,
Tags: nostr.Tags{{"t", chosenTag}},
}
evt.Sign(sk)
nLayers := rnd.IntN(len(layers)) + 1
perm := rnd.Perm(len(layers))
for pi := 0; pi < nLayers; pi++ {
li := perm[pi]
err := layers[li].SaveEvent(evt)
require.NoError(t, err)
layerEvents[li] = append(layerEvents[li], indexedEvent{evt, chosenTag})
}
written++
}
if n > 0 {
totalRemaining := 0
for _, levts := range layerEvents {
totalRemaining += len(levts)
}
if totalRemaining > 0 {
m := rnd.IntN(n)
if m > totalRemaining {
m = totalRemaining
}
for i := 0; i < m; i++ {
var nonEmpty []int
for li, levts := range layerEvents {
if len(levts) > 0 {
nonEmpty = append(nonEmpty, li)
}
}
if len(nonEmpty) == 0 {
break
}
li := nonEmpty[rnd.IntN(len(nonEmpty))]
idx := rnd.IntN(len(layerEvents[li]))
evtInfo := layerEvents[li][idx]
err := layers[li].DeleteEvent(evtInfo.evt.ID)
require.NoError(t, err)
layerEvents[li] = append(layerEvents[li][:idx], layerEvents[li][idx+1:]...)
}
}
}
if n > 0 {
o := rnd.IntN(n)
for i := 0; i < o; i++ {
if len(mmmm.freeRangesAll) > 1 {
param := rnd.IntN(len(mmmm.freeRangesAll))
err := mmmm.Defragment(param)
require.NoError(t, err)
}
}
}
}
// query each layer
for li, il := range layers {
levts := layerEvents[li]
// query by author
evts := slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}}, 10000))
require.Equal(t, len(levts), len(evts))
// query by author and kind
evts = slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}, Kinds: []nostr.Kind{nostr.KindTextNote}}, 10000))
require.Equal(t, len(levts), len(evts))
// query by "t" tag
for _, tagVal := range tChoices {
expected := 0
for _, ie := range levts {
if ie.tag == tagVal {
expected++
}
}
evts = slices.Collect(il.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"t": []string{tagVal}}}, 10000))
require.Equal(t, expected, len(evts))
}
// query with no parameters
allEvts := slices.Collect(il.QueryEvents(nostr.Filter{}, 10000))
require.Equal(t, len(levts), len(allEvts))
}
// build union of all events across all layers
allEventSet := make(map[string]nostr.Event)
for _, levts := range layerEvents {
for _, ie := range levts {
allEventSet[ie.evt.ID.String()] = ie.evt
}
}
// all events still accessible via GetByID
for _, evt := range allEventSet {
gotEvt, eventLayers := mmmm.GetByID(evt.ID)
require.NotNil(t, gotEvt)
require.NotEmpty(t, eventLayers)
require.Equal(t, evt.Content, gotEvt.Content)
}
verifyFreeRangesInvariants(t, mmmm)
mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
cursor, err := txn.OpenCursor(mmmm.indexId)
require.NoError(t, err)
defer cursor.Close()
var allPositions []position
for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
pos := positionFromBytes(val[0:12])
allPositions = append(allPositions, pos)
}
slices.SortFunc(allPositions, func(a, b position) int {
return cmp.Compare(a.start, b.start)
})
var lastEnd uint64
for _, pos := range allPositions {
if pos.start < lastEnd {
t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
}
lastEnd = pos.start + uint64(pos.size)
}
return nil
})
})
}