coracle/nostrlib
2
1
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eventstore: fuzz testing found us a bug!

Browse Source
This commit is contained in:
fiatjaf
2025-08-05 22:56:55 -03:00
parent 47ca205e9e
commit 030cad168d
13 changed files with 1203 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eventstore/boltdb/helpers.go
+15 -13
View File
@@ -113,14 +113,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item)
func (its iterators) quickselect(k int) uint32 {
func (its iterators) quickselect(k int) {
if len(its) == 0 || k >= len(its) {
return 0
return
}
left, right := 0, len(its)-1
for {
// insertion sort for small ranges
if right-left <= 20 {
@@ -129,7 +127,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j]
}
}
return its[0].last
return
}
// median-of-three to choose pivot
@@ -165,14 +163,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr
if k == pivotIndex {
// now that stuff is selected we get the highest "last"
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
return
}
if k < pivotIndex {
@@ -183,6 +174,17 @@ func (its iterators) quickselect(k int) uint32 {
}
}
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct {
bucket []byte
fullkey []byte
eventstore/boltdb/query.go
+3 -1
View File
@@ -110,7 +110,9 @@ func (b *BoltBackend) query(txn *bbolt.Tx, filter nostr.Filter, limit int, yield
// after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators)))
k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold
for i := range iterators {
eventstore/boltdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d
Vendored
+8
View File
@@ -0,0 +1,8 @@
go test fuzz v1
uint(200)
uint(15)
uint(11)
uint(49)
uint(2)
uint(91)
uint(1)
eventstore/boltdb/utils_test.go
-67
View File
@@ -1,67 +0,0 @@
package boltdb
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestQuickselect(t *testing.T) {
{
its := iterators{
{last: 781},
{last: 900},
{last: 1},
{last: 81},
{last: 325},
{last: 781},
{last: 562},
{last: 81},
{last: 444},
}
its.quickselect(3)
require.ElementsMatch(t,
[]uint32{its[0].last, its[1].last, its[2].last},
[]uint32{900, 781, 781},
)
}
{
its := iterators{
{last: 781},
{last: 781},
{last: 900},
{last: 1},
{last: 87},
{last: 315},
{last: 789},
{last: 500},
{last: 812},
{last: 306},
{last: 612},
{last: 444},
{last: 59},
{last: 441},
{last: 901},
{last: 901},
{last: 2},
{last: 81},
{last: 325},
{last: 781},
{last: 562},
{last: 81},
{last: 326},
{last: 662},
{last: 444},
{last: 81},
{last: 444},
}
its.quickselect(6)
require.ElementsMatch(t,
[]uint32{its[0].last, its[1].last, its[2].last, its[3].last, its[4].last, its[5].last},
[]uint32{901, 900, 901, 781, 812, 789},
)
}
}
eventstore/internal/helpers.go
+6 -3
View File
@@ -83,9 +83,12 @@ func BatchSizePerNumberOfQueries(totalFilterLimit int, numberOfQueries int) int
return totalFilterLimit
}
return int(
math.Ceil(
math.Pow(float64(totalFilterLimit), 0.80) / math.Pow(float64(numberOfQueries), 0.71),
return max(
4,
int(
math.Ceil(
math.Pow(float64(totalFilterLimit), 0.80)/math.Pow(float64(numberOfQueries), 0.71),
),
),
)
}
eventstore/lmdb/helpers.go
+15 -13
View File
@@ -98,14 +98,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item)
func (its iterators) quickselect(k int) uint32 {
func (its iterators) quickselect(k int) {
if len(its) == 0 || k >= len(its) {
return 0
return
}
left, right := 0, len(its)-1
for {
// insertion sort for small ranges
if right-left <= 20 {
@@ -114,7 +112,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j]
}
}
return its[0].last
return
}
// median-of-three to choose pivot
@@ -150,14 +148,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr
if k == pivotIndex {
// now that stuff is selected we get the highest "last"
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
return
}
if k < pivotIndex {
@@ -168,6 +159,17 @@ func (its iterators) quickselect(k int) uint32 {
}
}
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct {
dbi lmdb.DBI
key []byte
eventstore/lmdb/query.go
+3 -1
View File
@@ -114,7 +114,9 @@ func (b *LMDBBackend) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yield
// after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators)))
k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold
for i := range iterators {
eventstore/lmdb/testdata/fuzz/FuzzQuery/bca5f7631b59619d
Vendored
+8
View File
@@ -0,0 +1,8 @@
go test fuzz v1
uint(200)
uint(15)
uint(11)
uint(49)
uint(2)
uint(91)
uint(1)
eventstore/lmdb/utils_test.go
+1126
View File
File diff suppressed because it is too large Load Diff
eventstore/mmm/helpers.go
+15 -13
View File
@@ -96,14 +96,12 @@ type iterators []*iterator
// i.e. [1, 700, 25, 312, 44, 28] with k=3 becomes something like [700, 312, 44, 1, 25, 28]
// in this case it's hardcoded to use the 'last' field of the iterator
// copied from https://github.com/chrislee87/go-quickselect
// this is modified to also return the highest 'last' (because it's not guaranteed it will be the first item)
func (its iterators) quickselect(k int) uint32 {
func (its iterators) quickselect(k int) {
if len(its) == 0 || k >= len(its) {
return 0
return
}
left, right := 0, len(its)-1
for {
// insertion sort for small ranges
if right-left <= 20 {
@@ -112,7 +110,7 @@ func (its iterators) quickselect(k int) uint32 {
its[j], its[j-1] = its[j-1], its[j]
}
}
return its[0].last
return
}
// median-of-three to choose pivot
@@ -148,14 +146,7 @@ func (its iterators) quickselect(k int) uint32 {
pivotIndex = rr
if k == pivotIndex {
// now that stuff is selected we get the highest "last"
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
return
}
if k < pivotIndex {
@@ -166,6 +157,17 @@ func (its iterators) quickselect(k int) uint32 {
}
}
// return the highest 'last' value among the first k items in its
func (its iterators) threshold(k int) uint32 {
highest := its[0].last
for i := 1; i < k; i++ {
if its[i].last > highest {
highest = its[i].last
}
}
return highest
}
type key struct {
dbi lmdb.DBI
key []byte
eventstore/mmm/query.go
+3 -1
View File
@@ -139,7 +139,9 @@ func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yi
// after pulling from all iterators once we now find out what iterators are
// the ones we should keep pulling from next (i.e. which one's last emitted timestamp is the highest)
threshold := iterators.quickselect(min(numberOfIteratorsToPullOnEachRound, len(iterators)))
k := min(numberOfIteratorsToPullOnEachRound, len(iterators))
iterators.quickselect(k)
threshold := iterators.threshold(k)
// so we can emit all the events higher than the threshold
for i := range iterators {