bring in khatru and eventstore.
This commit is contained in:
fiatjaf
@@ -0,0 +1 @@
|
||||
decode-binary
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/fiatjaf/eventstore/internal/binary"
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
func main() {
|
||||
b, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to read from stdin: %s\n", err)
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
b = bytes.TrimSpace(b)
|
||||
|
||||
if bytes.HasPrefix(b, []byte("0x")) {
|
||||
fromHex := make([]byte, (len(b)-2)/2)
|
||||
_, err := hex.Decode(fromHex, b[2:])
|
||||
if err == nil {
|
||||
b = fromHex
|
||||
}
|
||||
}
|
||||
|
||||
var evt nostr.Event
|
||||
err = binary.Unmarshal(b, &evt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to decode: %s\n", err)
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
fmt.Println(evt.String())
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package binary
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
// Deprecated -- the encoding used here is not very elegant, we'll have a better binary format later.
|
||||
func Unmarshal(data []byte, evt *nostr.Event) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("failed to decode binary for event %s from %s at %d: %v", evt.ID, evt.PubKey, evt.CreatedAt, r)
|
||||
}
|
||||
}()
|
||||
|
||||
evt.ID = hex.EncodeToString(data[0:32])
|
||||
evt.PubKey = hex.EncodeToString(data[32:64])
|
||||
evt.Sig = hex.EncodeToString(data[64:128])
|
||||
evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
|
||||
evt.Kind = int(binary.BigEndian.Uint16(data[132:134]))
|
||||
contentLength := int(binary.BigEndian.Uint16(data[134:136]))
|
||||
evt.Content = string(data[136 : 136+contentLength])
|
||||
|
||||
curr := 136 + contentLength
|
||||
|
||||
nTags := binary.BigEndian.Uint16(data[curr : curr+2])
|
||||
curr++
|
||||
evt.Tags = make(nostr.Tags, nTags)
|
||||
|
||||
for t := range evt.Tags {
|
||||
curr++
|
||||
nItems := int(data[curr])
|
||||
tag := make(nostr.Tag, nItems)
|
||||
for i := range tag {
|
||||
curr = curr + 1
|
||||
itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
|
||||
itemStart := curr + 2
|
||||
item := string(data[itemStart : itemStart+itemSize])
|
||||
tag[i] = item
|
||||
curr = itemStart + itemSize
|
||||
}
|
||||
evt.Tags[t] = tag
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Deprecated -- the encoding used here is not very elegant, we'll have a better binary format later.
|
||||
func Marshal(evt *nostr.Event) ([]byte, error) {
|
||||
content := []byte(evt.Content)
|
||||
buf := make([]byte, 32+32+64+4+2+2+len(content)+65536+len(evt.Tags)*40 /* blergh */)
|
||||
|
||||
hex.Decode(buf[0:32], []byte(evt.ID))
|
||||
hex.Decode(buf[32:64], []byte(evt.PubKey))
|
||||
hex.Decode(buf[64:128], []byte(evt.Sig))
|
||||
|
||||
if evt.CreatedAt > MaxCreatedAt {
|
||||
return nil, fmt.Errorf("created_at is too big: %d", evt.CreatedAt)
|
||||
}
|
||||
binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
|
||||
|
||||
if evt.Kind > MaxKind {
|
||||
return nil, fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, MaxKind)
|
||||
}
|
||||
binary.BigEndian.PutUint16(buf[132:134], uint16(evt.Kind))
|
||||
|
||||
if contentLength := len(content); contentLength > MaxContentSize {
|
||||
return nil, fmt.Errorf("content is too large: %d, max is %d", contentLength, MaxContentSize)
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(buf[134:136], uint16(contentLength))
|
||||
}
|
||||
copy(buf[136:], content)
|
||||
|
||||
if tagCount := len(evt.Tags); tagCount > MaxTagCount {
|
||||
return nil, fmt.Errorf("can't encode too many tags: %d, max is %d", tagCount, MaxTagCount)
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(buf[136+len(content):136+len(content)+2], uint16(tagCount))
|
||||
}
|
||||
|
||||
buf = buf[0 : 136+len(content)+2]
|
||||
|
||||
for _, tag := range evt.Tags {
|
||||
if itemCount := len(tag); itemCount > MaxTagItemCount {
|
||||
return nil, fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, MaxTagItemCount)
|
||||
} else {
|
||||
buf = append(buf, uint8(itemCount))
|
||||
}
|
||||
for _, item := range tag {
|
||||
itemb := []byte(item)
|
||||
itemSize := len(itemb)
|
||||
if itemSize > MaxTagItemSize {
|
||||
return nil, fmt.Errorf("tag item is too large: %d, max is %d", itemSize, MaxTagItemSize)
|
||||
}
|
||||
buf = binary.BigEndian.AppendUint16(buf, uint16(itemSize))
|
||||
buf = append(buf, itemb...)
|
||||
buf = append(buf, 0)
|
||||
}
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package binary
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxKind = math.MaxUint16
|
||||
MaxCreatedAt = math.MaxUint32
|
||||
MaxContentSize = math.MaxUint16
|
||||
MaxTagCount = math.MaxUint16
|
||||
MaxTagItemCount = math.MaxUint8
|
||||
MaxTagItemSize = math.MaxUint16
|
||||
)
|
||||
|
||||
func EventEligibleForBinaryEncoding(event *nostr.Event) bool {
|
||||
if len(event.Content) > MaxContentSize || event.Kind > MaxKind || event.CreatedAt > MaxCreatedAt || len(event.Tags) > MaxTagCount {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, tag := range event.Tags {
|
||||
if len(tag) > MaxTagItemCount {
|
||||
return false
|
||||
}
|
||||
for _, item := range tag {
|
||||
if len(item) > MaxTagItemSize {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package checks
|
||||
|
||||
import (
|
||||
"github.com/fiatjaf/eventstore"
|
||||
"github.com/fiatjaf/eventstore/badger"
|
||||
"github.com/fiatjaf/eventstore/bluge"
|
||||
"github.com/fiatjaf/eventstore/edgedb"
|
||||
"github.com/fiatjaf/eventstore/lmdb"
|
||||
"github.com/fiatjaf/eventstore/mongo"
|
||||
"github.com/fiatjaf/eventstore/mysql"
|
||||
"github.com/fiatjaf/eventstore/postgresql"
|
||||
"github.com/fiatjaf/eventstore/sqlite3"
|
||||
"github.com/fiatjaf/eventstore/strfry"
|
||||
)
|
||||
|
||||
// compile-time checks to ensure all backends implement Store
|
||||
var (
|
||||
_ eventstore.Store = (*badger.BadgerBackend)(nil)
|
||||
_ eventstore.Store = (*lmdb.LMDBBackend)(nil)
|
||||
_ eventstore.Store = (*edgedb.EdgeDBBackend)(nil)
|
||||
_ eventstore.Store = (*postgresql.PostgresBackend)(nil)
|
||||
_ eventstore.Store = (*mongo.MongoDBBackend)(nil)
|
||||
_ eventstore.Store = (*sqlite3.SQLite3Backend)(nil)
|
||||
_ eventstore.Store = (*strfry.StrfryBackend)(nil)
|
||||
_ eventstore.Store = (*bluge.BlugeBackend)(nil)
|
||||
_ eventstore.Store = (*mysql.MySQLBackend)(nil)
|
||||
)
|
||||
@@ -0,0 +1,183 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mergesortedslices "fiatjaf.com/lib/merge-sorted-slices"
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
func IsOlder(previous, next *nostr.Event) bool {
|
||||
return previous.CreatedAt < next.CreatedAt ||
|
||||
(previous.CreatedAt == next.CreatedAt && previous.ID > next.ID)
|
||||
}
|
||||
|
||||
func ChooseNarrowestTag(filter nostr.Filter) (key string, values []string, goodness int) {
|
||||
var tagKey string
|
||||
var tagValues []string
|
||||
for key, values := range filter.Tags {
|
||||
switch key {
|
||||
case "e", "E", "q":
|
||||
// 'e' and 'q' are the narrowest possible, so if we have that we will use it and that's it
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
goodness = 9
|
||||
break
|
||||
case "a", "A", "i", "I", "g", "r":
|
||||
// these are second-best as they refer to relatively static things
|
||||
goodness = 8
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
case "d":
|
||||
// this is as good as long as we have an "authors"
|
||||
if len(filter.Authors) != 0 && goodness < 7 {
|
||||
goodness = 7
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
} else if goodness < 4 {
|
||||
goodness = 4
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
}
|
||||
case "h", "t", "l", "k", "K":
|
||||
// these things denote "categories", so they are a little more broad
|
||||
if goodness < 6 {
|
||||
goodness = 6
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
}
|
||||
case "p":
|
||||
// this is broad and useless for a pure tag search, but we will still prefer it over others
|
||||
// for secondary filtering
|
||||
if goodness < 2 {
|
||||
goodness = 2
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
}
|
||||
default:
|
||||
// all the other tags are probably too broad and useless
|
||||
if goodness == 0 {
|
||||
tagKey = key
|
||||
tagValues = values
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tagKey, tagValues, goodness
|
||||
}
|
||||
|
||||
func CopyMapWithoutKey[K comparable, V any](originalMap map[K]V, key K) map[K]V {
|
||||
newMap := make(map[K]V, len(originalMap)-1)
|
||||
for k, v := range originalMap {
|
||||
if k != key {
|
||||
newMap[k] = v
|
||||
}
|
||||
}
|
||||
return newMap
|
||||
}
|
||||
|
||||
type IterEvent struct {
|
||||
*nostr.Event
|
||||
Q int
|
||||
}
|
||||
|
||||
// MergeSortMultipleBatches takes the results of multiple iterators, which are already sorted,
|
||||
// and merges them into a single big sorted slice
|
||||
func MergeSortMultiple(batches [][]IterEvent, limit int, dst []IterEvent) []IterEvent {
|
||||
// clear up empty lists here while simultaneously computing the total count.
|
||||
// this helps because if there are a bunch of empty lists then this pre-clean
|
||||
// step will get us in the faster 'merge' branch otherwise we would go to the other.
|
||||
// we would have to do the cleaning anyway inside it.
|
||||
// and even if we still go on the other we save one iteration by already computing the
|
||||
// total count.
|
||||
total := 0
|
||||
for i := len(batches) - 1; i >= 0; i-- {
|
||||
if len(batches[i]) == 0 {
|
||||
batches = SwapDelete(batches, i)
|
||||
} else {
|
||||
total += len(batches[i])
|
||||
}
|
||||
}
|
||||
|
||||
if limit == -1 {
|
||||
limit = total
|
||||
}
|
||||
|
||||
// this amazing equation will ensure that if one of the two sides goes very small (like 1 or 2)
|
||||
// the other can go very high (like 500) and we're still in the 'merge' branch.
|
||||
// if values go somewhere in the middle then they may match the 'merge' branch (batches=20,limit=70)
|
||||
// or not (batches=25, limit=60)
|
||||
if math.Log(float64(len(batches)*2))+math.Log(float64(limit)) < 8 {
|
||||
if dst == nil {
|
||||
dst = make([]IterEvent, limit)
|
||||
} else if cap(dst) < limit {
|
||||
dst = slices.Grow(dst, limit-len(dst))
|
||||
}
|
||||
dst = dst[0:limit]
|
||||
return mergesortedslices.MergeFuncNoEmptyListsIntoSlice(dst, batches, compareIterEvent)
|
||||
} else {
|
||||
if dst == nil {
|
||||
dst = make([]IterEvent, total)
|
||||
} else if cap(dst) < total {
|
||||
dst = slices.Grow(dst, total-len(dst))
|
||||
}
|
||||
dst = dst[0:total]
|
||||
|
||||
// use quicksort in a dumb way that will still be fast because it's cheated
|
||||
lastIndex := 0
|
||||
for _, batch := range batches {
|
||||
copy(dst[lastIndex:], batch)
|
||||
lastIndex += len(batch)
|
||||
}
|
||||
|
||||
slices.SortFunc(dst, compareIterEvent)
|
||||
|
||||
for i, j := 0, total-1; i < j; i, j = i+1, j-1 {
|
||||
dst[i], dst[j] = dst[j], dst[i]
|
||||
}
|
||||
|
||||
if limit < len(dst) {
|
||||
return dst[0:limit]
|
||||
}
|
||||
return dst
|
||||
}
|
||||
}
|
||||
|
||||
// BatchSizePerNumberOfQueries tries to make an educated guess for the batch size given the total filter limit and
|
||||
// the number of abstract queries we'll be conducting at the same time
|
||||
func BatchSizePerNumberOfQueries(totalFilterLimit int, numberOfQueries int) int {
|
||||
if numberOfQueries == 1 || totalFilterLimit*numberOfQueries < 50 {
|
||||
return totalFilterLimit
|
||||
}
|
||||
|
||||
return int(
|
||||
math.Ceil(
|
||||
math.Pow(float64(totalFilterLimit), 0.80) / math.Pow(float64(numberOfQueries), 0.71),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func SwapDelete[A any](arr []A, i int) []A {
|
||||
arr[i] = arr[len(arr)-1]
|
||||
return arr[:len(arr)-1]
|
||||
}
|
||||
|
||||
func compareIterEvent(a, b IterEvent) int {
|
||||
if a.Event == nil {
|
||||
if b.Event == nil {
|
||||
return 0
|
||||
} else {
|
||||
return -1
|
||||
}
|
||||
} else if b.Event == nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
if a.CreatedAt == b.CreatedAt {
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
}
|
||||
return cmp.Compare(a.CreatedAt, b.CreatedAt)
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(256)
|
||||
uint(31)
|
||||
uint(260)
|
||||
uint(2)
|
||||
uint(69)
|
||||
uint(385)
|
||||
uint(1)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(267)
|
||||
uint(50)
|
||||
uint(355)
|
||||
uint(2)
|
||||
uint(69)
|
||||
uint(213)
|
||||
uint(1)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(280)
|
||||
uint(0)
|
||||
uint(13)
|
||||
uint(2)
|
||||
uint(2)
|
||||
uint(0)
|
||||
uint(0)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(259)
|
||||
uint(126)
|
||||
uint(5)
|
||||
uint(23)
|
||||
uint(0)
|
||||
uint(0)
|
||||
uint(92)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(201)
|
||||
uint(50)
|
||||
uint(13)
|
||||
uint(97)
|
||||
uint(0)
|
||||
uint(0)
|
||||
uint(77)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(164)
|
||||
uint(50)
|
||||
uint(13)
|
||||
uint(1)
|
||||
uint(2)
|
||||
uint(13)
|
||||
uint(0)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(200)
|
||||
uint(50)
|
||||
uint(13)
|
||||
uint(8)
|
||||
uint(2)
|
||||
uint(0)
|
||||
uint(1)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(200)
|
||||
uint(117)
|
||||
uint(13)
|
||||
uint(2)
|
||||
uint(2)
|
||||
uint(0)
|
||||
uint(1)
|
||||
@@ -0,0 +1,8 @@
|
||||
go test fuzz v1
|
||||
uint(200)
|
||||
uint(50)
|
||||
uint(13)
|
||||
uint(2)
|
||||
uint(2)
|
||||
uint(0)
|
||||
uint(0)
|
||||
Reference in New Issue
Block a user