Add signevent to nip86

fixes nostr:nevent1qvzqqqqqqypzqtrucc4xjl4r57px2g0nl560pje8x6fuhe0fxy8n23ylgd3z5hxuqythwumn8ghj7un9d3shjtnswf5k6ctv9ehx2ap0qy88wumn8ghj7mn0wvhxcmmv9uqzpctrpfqmk9nqzdlvemhgrskz6yyer4j93gudlrlf7jn2qafys9pzyw9u5s

connection.go

@@ -1,158 +0,0 @@
-package nostr
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/textproto"
-	"sync/atomic"
-	"time"
-
-	ws "github.com/coder/websocket"
-)
-
-var ErrDisconnected = errors.New("<disconnected>")
-
-type writeRequest struct {
-	msg    []byte
-	answer chan error
-}
-
-func (r *Relay) newConnection(ctx context.Context, httpClient *http.Client) error {
-	debugLogf("{%s} connecting!\n", r.URL)
-
-	dialCtx := ctx
-	if _, ok := dialCtx.Deadline(); !ok {
-		// if no timeout is set, force it to 7 seconds
-		dialCtx, _ = context.WithTimeoutCause(ctx, 7*time.Second, errors.New("connection took too long"))
-	}
-
-	dialOpts := &ws.DialOptions{
-		HTTPHeader: http.Header{
-			textproto.CanonicalMIMEHeaderKey("User-Agent"): {"fiatjaf.com/nostr"},
-		},
-		CompressionMode: ws.CompressionContextTakeover,
-		HTTPClient:      httpClient,
-	}
-	for k, v := range r.requestHeader {
-		dialOpts.HTTPHeader[k] = v
-	}
-
-	c, _, err := ws.Dial(dialCtx, r.URL, dialOpts)
-	if err != nil {
-		return err
-	}
-	c.SetReadLimit(2 << 24) // 33MB
-
-	// this will tell if the connection is closed
-
-	// ping every 29 seconds
-	ticker := time.NewTicker(29 * time.Second)
-
-	// main websocket loop
-	readQueue := make(chan string)
-
-	r.conn = c
-	r.writeQueue = make(chan writeRequest)
-	r.closed = &atomic.Bool{}
-	r.closedNotify = make(chan struct{})
-
-	go func() {
-		pingAttempt := 0
-
-		for {
-			select {
-			case <-ctx.Done():
-				r.closeConnection(ws.StatusNormalClosure, "")
-				debugLogf("{%s} closing!, context done: '%s'\n", r.URL, context.Cause(ctx))
-				return
-			case <-r.closedNotify:
-				return
-			case <-ticker.C:
-				debugLogf("{%s} pinging\n", r.URL)
-				ctx, cancel := context.WithTimeoutCause(ctx, time.Millisecond*800, errors.New("ping took too long"))
-				err := c.Ping(ctx)
-				cancel()
-
-				if err != nil {
-					pingAttempt++
-					debugLogf("{%s} error writing ping (attempt %d): %v", r.URL, pingAttempt, err)
-
-					if pingAttempt >= 3 {
-						debugLogf("{%s} error writing ping after multiple attempts; closing websocket", r.URL)
-						err = r.Close() // this should trigger a context cancelation
-						if err != nil {
-							debugLogf("{%s} failed to close relay: %v", r.URL, err)
-						}
-					}
-
-					continue
-				}
-
-				// ping was OK
-				debugLogf("{%s} ping OK", r.URL)
-				pingAttempt = 0
-			case wr := <-r.writeQueue:
-				debugLogf("{%s} sending '%v'\n", r.URL, string(wr.msg))
-				ctx, cancel := context.WithTimeoutCause(ctx, time.Second*10, errors.New("write took too long"))
-				err := c.Write(ctx, ws.MessageText, wr.msg)
-				cancel()
-				if err != nil {
-					debugLogf("{%s} closing!, write failed: '%s'\n", r.URL, err)
-					r.closeConnection(ws.StatusAbnormalClosure, "write failed")
-					if wr.answer != nil {
-						wr.answer <- err
-					}
-					return
-				}
-				if wr.answer != nil {
-					close(wr.answer)
-				}
-			case msg := <-readQueue:
-				debugLogf("{%s} received %v\n", r.URL, msg)
-				r.handleMessage(msg)
-			}
-		}
-	}()
-
-	// read loop -- loops back to the main loop
-	go func() {
-		buf := new(bytes.Buffer)
-
-		for {
-			buf.Reset()
-
-			_, reader, err := c.Reader(ctx)
-			if err != nil {
-				debugLogf("{%s} closing!, reader failure: '%s'\n", r.URL, err)
-				r.closeConnection(ws.StatusAbnormalClosure, "failed to get reader")
-				return
-			}
-			if _, err := io.Copy(buf, reader); err != nil {
-				debugLogf("{%s} closing!, read failure: '%s'\n", r.URL, err)
-				r.closeConnection(ws.StatusAbnormalClosure, "failed to read")
-				return
-			}
-
-			readQueue <- string(buf.Bytes())
-		}
-	}()
-
-	return nil
-}
-
-func (r *Relay) closeConnection(code ws.StatusCode, reason string) {
-	wasClosed := r.closed.Swap(true)
-	if !wasClosed {
-		r.conn.Close(code, reason)
-		r.connectionContextCancel(fmt.Errorf("doClose(): %s", reason))
-		r.closeMutex.Lock()
-		close(r.closedNotify)
-		close(r.writeQueue)
-		r.conn = nil
-		r.closeMutex.Unlock()
-	}
-}

eose_test.go

@@ -17,7 +17,7 @@ func TestEOSEMadness(t *testing.T) {
 	}, SubscriptionOptions{})
 	assert.NoError(t, err)
 
-	timeout := time.After(3 * time.Second)
+	timeout := time.After(2 * time.Second)
 	n := 0
 	e := 0
 

event.go

@@ -2,7 +2,9 @@ package nostr
 
 import (
 	"crypto/sha256"
+	"hash"
 	"strconv"
+	"unsafe"
 
 	"github.com/mailru/easyjson"
 	"github.com/templexxx/xhex"
@@ -26,10 +28,17 @@ func (evt Event) String() string {
 
 // GetID serializes and returns the event ID as a string.
 func (evt Event) GetID() ID {
-	return sha256.Sum256(evt.Serialize())
+	var id ID
+	evt.serializedHash(&id)
+	return id
 }
 
-// CheckID checks if the implied ID matches the given ID more efficiently.
+// SetID calculates and sets the id to the event in a single operation.
+func (evt *Event) SetID() {
+	evt.serializedHash(&evt.ID)
+}
+
+// CheckID checks if the implied ID matches the currently assigned ID.
 func (evt Event) CheckID() bool {
 	return evt.GetID() == evt.ID
 }
@@ -38,17 +47,56 @@ func (evt Event) CheckID() bool {
 func (evt Event) Serialize() []byte {
 	// the serialization process is just putting everything into a JSON array
 	// so the order is kept. See NIP-01
-	dst := make([]byte, 4+64, 100+len(evt.Content)+len(evt.Tags)*80)
+	dst := make([]byte, 0, 100+len(evt.Content)+len(evt.Tags)*80)
+	return evt.appendSerialized(dst)
+}
 
-	// the header portion is easy to serialize
+var escTable [256]bool
-	// [0,"pubkey",created_at,kind,[
+
-	copy(dst, `[0,"`)
+// pre-built escape sequences; index by the offending byte.
-	xhex.Encode(dst[4:4+64], evt.PubKey[:]) // there will always be such capacity
+var escSeq [256][2]byte
+
+// pre-built []byte slices for hash.Write calls (no per-call allocation).
+var escSlice [256][]byte
+
+var (
+	jsonQuote           = []byte{'"'}
+	serializedStart     = []byte(`[0,"`)
+	serializedPubkeyEnd = []byte(`",`)
+	serializedTagsEnd   = []byte("],")
+	serializedTagStart  = []byte{'['}
+	serializedTagEnd    = []byte{']'}
+	serializedComma     = []byte{','}
+	serializedEnd       = []byte{']'}
+)
+
+func init() {
+	for _, b := range []byte{'"', '\\', '\n', '\r', '\t'} {
+		escTable[b] = true
+	}
+
+	escSeq['"'] = [2]byte{'\\', '"'}
+	escSeq['\\'] = [2]byte{'\\', '\\'}
+	escSeq['\n'] = [2]byte{'\\', 'n'}
+	escSeq['\r'] = [2]byte{'\\', 'r'}
+	escSeq['\t'] = [2]byte{'\\', 't'}
+	for b, seq := range escSeq {
+		if escTable[b] {
+			escSlice[b] = seq[:]
+		}
+	}
+}
+
+func (evt Event) appendSerialized(dst []byte) []byte {
+	start := len(dst)
+	dst = append(dst, `[0,"`...)
+	dst = append(dst, make([]byte, 64)...)
+	xhex.Encode(dst[start+4:start+4+64], evt.PubKey[:])
 	dst = append(dst, `",`...)
-	dst = append(dst, strconv.FormatInt(int64(evt.CreatedAt), 10)...)
+	dst = strconv.AppendInt(dst, int64(evt.CreatedAt), 10)
-	dst = append(dst, `,`...)
+	dst = append(dst, ',')
-	dst = append(dst, strconv.FormatUint(uint64(evt.Kind), 10)...)
+	dst = strconv.AppendUint(dst, uint64(evt.Kind), 10)
-	dst = append(dst, `,`...)
+	dst = append(dst, ',')
 
 	// tags
 	dst = append(dst, '[')
@@ -62,15 +110,167 @@ func (evt Event) Serialize() []byte {
 			if i > 0 {
 				dst = append(dst, ',')
 			}
-			dst = escapeString(dst, s)
+			dst = appendJSONString(dst, s)
 		}
 		dst = append(dst, ']')
 	}
 	dst = append(dst, "],"...)
 
 	// content needs to be escaped in general as it is user generated.
-	dst = escapeString(dst, evt.Content)
+	dst = appendJSONString(dst, evt.Content)
 	dst = append(dst, ']')
 
 	return dst
 }
+
+func (evt Event) serializedHash(dst *ID) {
+	h := sha256.New()
+	h.Write(serializedStart)
+
+	var pubkeyHex [64]byte
+	xhex.Encode(pubkeyHex[:], evt.PubKey[:])
+	h.Write(pubkeyHex[:])
+	h.Write(serializedPubkeyEnd)
+
+	var numBuf [20]byte
+	b := strconv.AppendInt(numBuf[:0], int64(evt.CreatedAt), 10)
+	h.Write(b)
+	h.Write(serializedComma)
+	b = strconv.AppendUint(numBuf[:0], uint64(evt.Kind), 10)
+	h.Write(b)
+	h.Write(serializedComma)
+
+	h.Write(serializedTagStart)
+	for i, tag := range evt.Tags {
+		if i > 0 {
+			h.Write(serializedComma)
+		}
+		h.Write(serializedTagStart)
+		for j, s := range tag {
+			if j > 0 {
+				h.Write(serializedComma)
+			}
+			writeJSONString(h, s)
+		}
+		h.Write(serializedTagEnd)
+	}
+	h.Write(serializedTagsEnd)
+
+	writeJSONString(h, evt.Content)
+	h.Write(serializedEnd)
+
+	h.Sum((*dst)[:0])
+}
+
+// ── SWAR helper ──────────────────────────────────────────────────────────────
+
+// hasSpecial returns non-zero if any byte in w is one of: \t 0x09, \n 0x0A,
+// " 0x22, \ 0x5C. Uses the classic "hasvalue" bit-trick — no branches, no
+// memory, pure ALU. Works regardless of endianness because we only care
+// whether a match exists, not where.
+//
+//go:nosplit
+func hasSpecial(w uint64) bool {
+	match := func(w, v uint64) uint64 {
+		x := w ^ (0x0101010101010101 * v)
+		return (x - 0x0101010101010101) & ^x & 0x8080808080808080
+	}
+	return match(w, 0x09)|match(w, 0x0A)|match(w, 0x0D)|match(w, 0x22)|match(w, 0x5C) != 0
+}
+
+func appendJSONString(dst []byte, s string) []byte {
+	dst = append(dst, '"')
+
+	n := len(s)
+	if n == 0 {
+		return append(dst, '"')
+	}
+
+	base := uintptr(unsafe.Pointer(unsafe.StringData(s)))
+	start, i := 0, 0
+
+	// consume 8 bytes at a time;
+	// if the whole word is clean, advance without touching dst at all;
+	// but when a word is dirty, fall back to the byte loop only for that 8-byte window
+	for i+8 <= n {
+		w := *(*uint64)(unsafe.Pointer(base + uintptr(i)))
+		if hasSpecial(w) {
+			for end := i + 8; i < end; i++ {
+				if escTable[s[i]] {
+					// append everything since the start or the last time we did this up to here
+					dst = append(dst, s[start:i]...)
+
+					// append this special sequence
+					seq := escSeq[s[i]]
+					dst = append(dst, seq[0], seq[1])
+
+					// set this as a checkpoint
+					start = i + 1
+				}
+			}
+		} else {
+			i += 8
+		}
+	}
+
+	// scalar tail for the remaining <8 bytes (same logic used for the hasSpecial branch above)
+	for ; i < n; i++ {
+		if escTable[s[i]] {
+			dst = append(dst, s[start:i]...)
+			seq := escSeq[s[i]]
+			dst = append(dst, seq[0], seq[1])
+			start = i + 1
+		}
+	}
+
+	// add the remaining chunk (in a string without any specials this will add everything at once)
+	dst = append(dst, s[start:]...)
+
+	return append(dst, '"')
+}
+
+func writeJSONString(h hash.Hash, s string) {
+	h.Write(jsonQuote)
+
+	n := len(s)
+	if n == 0 {
+		h.Write(jsonQuote)
+		return
+	}
+
+	base := uintptr(unsafe.Pointer(unsafe.StringData(s)))
+	start, i := 0, 0
+
+	for i+8 <= n {
+		w := *(*uint64)(unsafe.Pointer(base + uintptr(i)))
+		// apply same logic as of appendJSONString()
+		if hasSpecial(w) {
+			for end := i + 8; i < end; i++ {
+				if escTable[s[i]] {
+					if i > start {
+						h.Write(unsafe.Slice(unsafe.StringData(s[start:i]), i-start))
+					}
+					h.Write(escSlice[s[i]])
+					start = i + 1
+				}
+			}
+		} else {
+			i += 8
+		}
+	}
+
+	for ; i < n; i++ {
+		if escTable[s[i]] {
+			if i > start {
+				h.Write(unsafe.Slice(unsafe.StringData(s[start:i]), i-start))
+			}
+			h.Write(escSlice[s[i]])
+			start = i + 1
+		}
+	}
+
+	if start < n {
+		h.Write(unsafe.Slice(unsafe.StringData(s[start:]), len(s)-start))
+	}
+	h.Write(jsonQuote)
+}

event_easyjson.go

@@ -74,6 +74,8 @@ func easyjsonDecodeEvent(in *jlexer.Lexer, out *Event) {
 			if len(b) == 128 {
 				xhex.Decode(out.Sig[:], b)
 			}
+		default:
+			in.SkipRecursive()
 		}
 		in.WantComma()
 	}

event_test.go

@@ -1,8 +1,12 @@
 package nostr
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
+	"io"
 	"math/rand/v2"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -102,23 +106,49 @@ func TestIDCheck(t *testing.T) {
 	}
 }
 
-func BenchmarkIDCheck(b *testing.B) {
+func BenchmarkEventVerifySignatureJSONL(b *testing.B) {
-	evt := Event{
+	events := loadBenchmarkEvents(b)
-		CreatedAt: Timestamp(rand.Int64N(9999999)),
+	b.ReportAllocs()
-		Content:   fmt.Sprintf("hello"),
+	b.ResetTimer()
-		Tags:      Tags{},
-	}
-	evt.Sign(Generate())
 
-	b.Run("naïve", func(b *testing.B) {
+	for i := 0; i < b.N; i++ {
-		for b.Loop() {
+		for _, evt := range events {
-			_ = evt.GetID() == evt.ID
+			if !evt.VerifySignature() {
+				b.Fatal("signature verification failed")
+			}
+		}
+	}
 }
-	})
 
-	b.Run("big brain", func(b *testing.B) {
+func loadBenchmarkEvents(b *testing.B) []Event {
-		for b.Loop() {
+	b.Helper()
-			_ = evt.CheckID()
+
+	f, err := os.Open("testdata/events.jsonl")
+	require.NoError(b, err)
+	b.Cleanup(func() { _ = f.Close() })
+
+	r := bufio.NewReader(f)
+	events := make([]Event, 0, 1024)
+
+	for {
+		line, err := r.ReadBytes('\n')
+		if err != nil && err != io.EOF {
+			require.NoError(b, err)
 		}
-	})
+
+		line = bytes.TrimSpace(line)
+		if len(line) != 0 {
+			var evt Event
+			require.NoError(b, json.Unmarshal(line, &evt))
+			require.True(b, evt.VerifySignature(), "fixture contains invalid signature")
+			events = append(events, evt)
+		}
+
+		if err == io.EOF {
+			break
+		}
+	}
+
+	require.NotEmpty(b, events)
+	return events
 }

eventstore/README.md

@@ -29,8 +29,6 @@ type Store interface {
 }
 ```
 
-[![Go Reference](https://pkg.go.dev/badge/fiatjaf.com/nostr/eventstore.svg)](https://pkg.go.dev/fiatjaf.com/nostr/eventstore) [![Run Tests](https://fiatjaf.com/nostr/eventstore/actions/workflows/test.yml/badge.svg)](https://fiatjaf.com/nostr/eventstore/actions/workflows/test.yml)
-
 ## Available Implementations
 
 - **bleve**: Full-text search and indexing using the Bleve search library

eventstore/bleve/bleve_test.go

@@ -2,11 +2,15 @@ package bleve
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore/lmdb"
+	"fiatjaf.com/nostr/eventstore/slicestore"
+	"github.com/pemistahl/lingua-go"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBleveFlow(t *testing.T) {
@@ -20,8 +24,11 @@ func TestBleveFlow(t *testing.T) {
 	bl := BleveBackend{
 		Path:          "/tmp/blevetest-bleve",
 		RawEventStore: bb,
+		Languages:     []lingua.Language{lingua.English},
 	}
-	bl.Init()
+	err := bl.Init()
+	require.NoError(t, err, "init")
+
 	defer bl.Close()
 
 	willDelete := make([]nostr.Event, 0, 3)
@@ -71,3 +78,184 @@ func TestBleveFlow(t *testing.T) {
 		assert.Equal(t, 1, n)
 	}
 }
+
+func TestSearch(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "test_search_pyramid")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	db := &slicestore.SliceStore{}
+	db.Init()
+
+	index := &BleveBackend{
+		Path:          filepath.Join(tempDir, "test_index"),
+		RawEventStore: db,
+		Languages: []lingua.Language{
+			lingua.English,
+			lingua.Portuguese,
+			lingua.Italian,
+		},
+	}
+	err = index.Init()
+	require.NoError(t, err)
+	defer index.Close()
+
+	pirateEvents := []nostr.Event{
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			CreatedAt: nostr.Timestamp(1609459200),
+			Kind:      1,
+			Content:   "Ahoy mateys! I've discovered a treasure chest filled with gold doubloons and silver pieces buried beneath the old palm tree on Skull Island. The secret map shows an X marks the spot where the legendary pirate Blackbeard hid his most valuable plunder. The chest contains rubies, emeralds, and ancient coins from sunken Spanish galleons. https://www.youtube.com/watch?v=enTAromEeHo&t=88s",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000002"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			CreatedAt: nostr.Timestamp(1609545600),
+			Kind:      1111,
+			Content:   "The treasure map I found reveals the location of Captain Morgan's lost gold mine deep in the Caribbean waters. Following the ancient compass directions leads to a hidden cave filled with golden artifacts, jeweled swords, and the crown jewels of forgotten kingdoms. The secret passage is guarded by mysterious symbols only known to the brotherhood of the sea. https://www.youtube.com/watch?v=yBtyNIqZios",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000003"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000003"),
+			CreatedAt: nostr.Timestamp(1609632000),
+			Kind:      1,
+			Content:   "Legends speak of the Emerald City of the Lost Pirates, a mythical place where streets are paved with gold and buildings adorned with precious gems. The secret entrance can only be found during a full moon when the tides reveal a hidden path across the coral reefs. Ancient scrolls tell of guardians protecting treasure vaults containing the world's most valuable gems.",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000004"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000004"),
+			CreatedAt: nostr.Timestamp(1609545601),
+			Kind:      1111,
+			Content:   "Bom dia seus piratas melequentos, onde está esse bendito tesouro? nostr:nprofile1qqsv6jemsnaq925ddfqjhwm3du3k0zk7dnj2ksk2k4hcfkf80mzf56spz9mhxue69uhkzcnpvdshgefwvdhk6tmjzyj",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000005"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000005"),
+			CreatedAt: nostr.Timestamp(1609545602),
+			Kind:      30023,
+			Content:   "I pirati dei Caraibi del XVII e XVIII secolo sono diventati leggendari per la loro ricerca di tesori. Questi avventurieri del mare saccheggiavano navi cariche d'oro, argento e pietre preziose provenienti dalle colonie spagnole del Nuovo Mondo.\n\nSecondo la leggenda, molti pirati seppellivano i loro tesori su isole remote, creando mappe segrete con la famosa \"X\" che segnava il punto. Capitani famosi come Barbanera, Capitan Kidd e Henry Morgan sono entrati nell'immaginario collettivo come custodi di ricchezze nascoste.\n\nAnche se la maggior parte dei tesori dei pirati sono probabilmente solo miti, alcuni sono stati davvero ritrovati. Il fascino di questi bottini nascosti continua ad ispirare storie, film e cacciatori di tesori ancora oggi.",
+			Tags:      nil,
+		},
+	}
+
+	for _, event := range pirateEvents {
+		err := db.SaveEvent(event)
+		require.NoError(t, err)
+		err = index.SaveEvent(event)
+		require.NoError(t, err)
+	}
+
+	testCases := []struct {
+		name     string
+		filter   nostr.Filter
+		expected int
+	}{
+		{
+			name: "search for 'gold'",
+			filter: nostr.Filter{
+				Search: "gold",
+			},
+			expected: 3, // all events mention gold
+		},
+		{
+			name: "search for 'treasure'",
+			filter: nostr.Filter{
+				Search: "treasure",
+			},
+			expected: 3, // all events mention treasure
+		},
+		{
+			name: "search for 'emerald' together with 'astronomical'",
+			filter: nostr.Filter{
+				Search: "astronomical emeralds",
+			},
+			expected: 0, // no events mention emeralds together with astronomical
+		},
+		{
+			name: "search for 'secret map'",
+			filter: nostr.Filter{
+				Search: "\"secret map\"",
+			},
+			expected: 1, // only one event mentions secret map
+		},
+		{
+			name: "search with kind filter",
+			filter: nostr.Filter{
+				Search: "gold",
+				Kinds:  []nostr.Kind{1},
+			},
+			expected: 2, // only two events are kind 1
+		},
+		{
+			name: "search in portuguese",
+			filter: nostr.Filter{
+				Search: "melequento",
+			},
+			expected: 1,
+		},
+		{
+			name: "search with exact match",
+			filter: nostr.Filter{
+				Search: "\"the secret entrance can only be found during a full moon\"",
+			},
+			expected: 1,
+		},
+		{
+			name: "search with OR across languages",
+			filter: nostr.Filter{
+				Search: "melequento OR matey",
+			},
+			expected: 2,
+		},
+		{
+			name: "search with exact reference found in the text",
+			filter: nostr.Filter{
+				Search: "tesouro nostr:nprofile1qqsv6jemsnaq925ddfqjhwm3du3k0zk7dnj2ksk2k4hcfkf80mzf56spzpmhxue69uhkyctwv9hxztnrdaksmfp5mw", // this is the same pubkey from above, but it's a different nprofile
+			},
+			expected: 1,
+		},
+		{
+			name: "search for URL",
+			filter: nostr.Filter{
+				Search: "https://www.youtube.com/watch?v=yBtyNIqZios treasure",
+			},
+			expected: 1,
+		},
+		{
+			name: "search for host/domain of URL",
+			filter: nostr.Filter{
+				Search: "www.youtube.com",
+			},
+			expected: 2,
+		},
+		{
+			name: "mentioning the author should include their notes in the result",
+			filter: nostr.Filter{
+				Search: " nostr:npub1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqshp52w2",
+			},
+			expected: 2,
+		},
+		{
+			name: "mentioning the author should include their notes in the result",
+			filter: nostr.Filter{
+				Search: "found gold? nostr:npub1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqshp52w2",
+			},
+			expected: 1,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var count int
+			for range index.QueryEvents(tc.filter, 10) {
+				count++
+			}
+			require.Equal(t, tc.expected, count)
+		})
+	}
+}

eventstore/bleve/delete.go

@@ -1,9 +0,0 @@
-package bleve
-
-import (
-	"fiatjaf.com/nostr"
-)
-
-func (b *BleveBackend) DeleteEvent(id nostr.ID) error {
-	return b.index.Delete(id.Hex())
-}

eventstore/bleve/helpers.go

@@ -1,9 +0,0 @@
-package bleve
-
-const (
-	idField        = "i"
-	contentField   = "c"
-	kindField      = "k"
-	createdAtField = "a"
-	pubkeyField    = "p"
-)

eventstore/bleve/lexer.go

@@ -0,0 +1,104 @@
+package bleve
+
+import (
+	"strings"
+	"unicode"
+)
+
+// lexer tokenizes the input string
+type Lexer struct {
+	input string
+	pos   int
+
+	peekedQueue []Token
+}
+
+func NewLexer(input string) *Lexer {
+	return &Lexer{input: input, pos: 0}
+}
+
+func (l *Lexer) peek() rune {
+	if l.pos >= len(l.input) {
+		return 0
+	}
+	return rune(l.input[l.pos])
+}
+
+func (l *Lexer) advance() rune {
+	if l.pos >= len(l.input) {
+		return 0
+	}
+	ch := rune(l.input[l.pos])
+	l.pos++
+	return ch
+}
+
+func (l *Lexer) skipWhitespace() {
+	for l.peek() != 0 && unicode.IsSpace(l.peek()) {
+		l.advance()
+	}
+}
+
+func (l *Lexer) readWord() string {
+	start := l.pos
+
+	// read regular word (alphanumeric, hyphens, underscores)
+	for l.peek() != 0 && !unicode.IsSpace(l.peek()) &&
+		l.peek() != '(' && l.peek() != ')' && l.peek() != '"' {
+		l.advance()
+	}
+
+	return l.input[start:l.pos]
+}
+
+func (l *Lexer) PeekToken() Token {
+	next := l.NextToken()
+	l.peekedQueue = append(l.peekedQueue, next)
+	return next
+}
+
+func (l *Lexer) ReturnToken(tok Token) {
+	l.peekedQueue = append(l.peekedQueue, tok)
+}
+
+func (l *Lexer) NextToken() (tok Token) {
+	if len(l.peekedQueue) > 0 {
+		next := l.peekedQueue[len(l.peekedQueue)-1]
+		l.peekedQueue = l.peekedQueue[0 : len(l.peekedQueue)-1]
+		return next
+	}
+
+	l.skipWhitespace()
+
+	if l.pos >= len(l.input) {
+		return Token{Type: TokenEOF}
+	}
+
+	ch := l.peek()
+
+	switch ch {
+	case '(':
+		l.advance()
+		return Token{Type: TokenLParen, Value: "("}
+	case ')':
+		l.advance()
+		return Token{Type: TokenRParen, Value: ")"}
+	case '"':
+		l.advance()
+		return Token{Type: TokenQuote, Value: "\""}
+	default:
+		word := l.readWord()
+		upperWord := strings.ToUpper(word)
+
+		switch upperWord {
+		case "OR", "||":
+			return Token{Type: TokenOR, Value: word}
+		case "AND", "&&":
+			return Token{Type: TokenAND, Value: word}
+		case "NOT", "!":
+			return Token{Type: TokenNOT, Value: word}
+		default:
+			return Token{Type: TokenWord, Value: word}
+		}
+	}
+}

eventstore/bleve/lib.go

@@ -1,34 +1,101 @@
 package bleve
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"iter"
+	"slices"
+	"strconv"
+	"strings"
 	"sync"
+	"time"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore"
+	"fiatjaf.com/nostr/nip27"
+	"fiatjaf.com/nostr/nip73"
+	"fiatjaf.com/nostr/sdk"
 	bleve "github.com/blevesearch/bleve/v2"
+	_ "github.com/blevesearch/bleve/v2/analysis/analyzer/simple"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ar"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/da"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/de"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/en"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/es"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fa"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/fr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/gl"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hi"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hr"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/hu"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/in"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/it"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/nl"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/no"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/pl"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/pt"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ro"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/sv"
+	_ "github.com/blevesearch/bleve/v2/analysis/lang/tr"
 	bleveMapping "github.com/blevesearch/bleve/v2/mapping"
+	bleveQuery "github.com/blevesearch/bleve/v2/search/query"
+	"github.com/pemistahl/lingua-go"
 )
 
-var _ eventstore.Store = (*BleveBackend)(nil)
+const (
+	labelContentField    = "c"
+	labelKindField       = "k"
+	labelCreatedAtField  = "a"
+	labelAuthorField     = "p"
+	labelReferencesField = "r"
+	labelExtrasField     = "x"
+)
+
+var SupportedLanguages = []lingua.Language{
+	// each of these translates to a specific bleve analyzer
+	// except for japanese-korean-chinese that all use the same "cjk" analyzer
+	lingua.Arabic,
+	lingua.Chinese,
+	lingua.Croatian,
+	lingua.Danish,
+	lingua.Dutch,
+	lingua.English,
+	lingua.Finnish,
+	lingua.French,
+	lingua.German,
+	lingua.Hindi,
+	lingua.Hungarian,
+	lingua.Italian,
+	lingua.Japanese,
+	lingua.Korean,
+	lingua.Persian,
+	lingua.Polish,
+	lingua.Portuguese,
+	lingua.Romanian,
+	lingua.Russian,
+	lingua.Spanish,
+	lingua.Swedish,
+	lingua.Turkish,
+}
 
 type BleveBackend struct {
 	sync.Mutex
-	// Path is where the index will be saved
 	Path          string
-
-	// RawEventStore is where we'll fetch the raw events from
-	// bleve will only store ids, so the actual events must be somewhere else
 	RawEventStore eventstore.Store
+	ReadOnly      bool
+	OpenTimeout   time.Duration
+
+	IndexableKinds []nostr.Kind
+
+	Languages     []lingua.Language
+	languageCodes []string
 
 	index    bleve.Index
-}
+	detector lingua.LanguageDetector
-
-func (b *BleveBackend) Close() {
-	if b.index != nil {
-		b.index.Close()
-	}
 }
 
 func (b *BleveBackend) Init() error {
@@ -38,12 +105,94 @@ func (b *BleveBackend) Init() error {
 	if b.RawEventStore == nil {
 		return fmt.Errorf("missing RawEventStore")
 	}
+	if len(b.Languages) == 0 {
+		return fmt.Errorf("missing Languages")
+	}
+	if len(b.IndexableKinds) == 0 {
+		b.IndexableKinds = []nostr.Kind{0, 1, 6, 11, 16, 20, 21, 22, 24, 1111, 9802, 30023, 30818}
+	}
 
-	// try to open existing index
+	validLanguages := make([]lingua.Language, 0, len(b.Languages))
-	index, err := bleve.Open(b.Path)
+	b.languageCodes = make([]string, 0, len(b.Languages))
+	for _, lang := range b.Languages {
+		var code string
+
+		switch lang {
+		case lingua.Chinese, lingua.Korean, lingua.Japanese:
+			code = "cjk"
+		default:
+			code = strings.ToLower(lang.IsoCode639_1().String())
+		}
+
+		if slices.Contains(b.languageCodes, code) {
+			continue
+		}
+
+		validLanguages = append(validLanguages, lang)
+		b.languageCodes = append(b.languageCodes, code)
+	}
+	b.Languages = validLanguages
+
+	opts := map[string]any{
+		"read_only": b.ReadOnly,
+	}
+	if b.OpenTimeout != 0 {
+		opts["bolt_timeout"] = b.OpenTimeout.String()
+	}
+
+	index, err := bleve.OpenUsing(b.Path, opts)
 	if err == bleve.ErrorIndexPathDoesNotExist {
-		// create new index with default mapping
 		mapping := bleveMapping.NewIndexMapping()
+		mapping.DefaultMapping.Dynamic = false
+		doc := bleveMapping.NewDocumentStaticMapping()
+
+		for _, code := range b.languageCodes {
+			contentField := bleveMapping.NewTextFieldMapping()
+			contentField.Analyzer = code
+			contentField.Store = false
+			contentField.IncludeTermVectors = false
+			contentField.DocValues = false
+			contentField.IncludeInAll = false
+			doc.AddFieldMappingsAt(labelContentField+"_"+code, contentField)
+		}
+
+		extrasField := bleveMapping.NewTextFieldMapping()
+		extrasField.Analyzer = "simple"
+		extrasField.Store = false
+		extrasField.IncludeTermVectors = false
+		extrasField.DocValues = false
+		extrasField.IncludeInAll = false
+		doc.AddFieldMappingsAt(labelExtrasField, extrasField)
+
+		referencesField := bleveMapping.NewKeywordFieldMapping()
+		referencesField.DocValues = false
+		referencesField.Store = false
+		referencesField.IncludeTermVectors = false
+		referencesField.IncludeInAll = false
+		doc.AddFieldMappingsAt(labelReferencesField, referencesField)
+
+		authorField := bleveMapping.NewKeywordFieldMapping()
+		authorField.DocValues = false
+		authorField.Store = false
+		authorField.IncludeTermVectors = false
+		doc.AddFieldMappingsAt(labelAuthorField, authorField)
+
+		kindField := bleveMapping.NewKeywordFieldMapping()
+		kindField.DocValues = false
+		kindField.Store = false
+		kindField.IncludeTermVectors = false
+		kindField.IncludeInAll = false
+		doc.AddFieldMappingsAt(labelKindField, kindField)
+
+		timestampField := bleveMapping.NewDateTimeFieldMapping()
+		timestampField.DocValues = false
+		timestampField.Store = false
+		timestampField.IncludeTermVectors = false
+		timestampField.IncludeInAll = false
+		doc.AddFieldMappingsAt(labelCreatedAtField, timestampField)
+
+		mapping.AddDocumentMapping("_default", doc)
+
 		index, err = bleve.New(b.Path, mapping)
 		if err != nil {
 			return fmt.Errorf("error creating index: %w", err)
@@ -53,6 +202,126 @@ func (b *BleveBackend) Init() error {
 	}
 
 	b.index = index
+
+	if len(b.Languages) >= 2 {
+		b.detector = lingua.NewLanguageDetectorBuilder().
+			FromLanguages(b.Languages...).
+			Build()
+	}
+
+	return nil
+}
+
+func (b *BleveBackend) Close() {
+	if b != nil && b.index != nil {
+		b.index.Close()
+	}
+}
+
+func (b *BleveBackend) SaveEvent(event nostr.Event) error {
+	if slices.Contains(b.IndexableKinds, event.Kind) {
+		return b.indexEvent(event)
+	}
+	return nil
+}
+
+func (b *BleveBackend) DeleteEvent(id nostr.ID) error {
+	if b != nil && b.index != nil {
+		return b.index.Delete(id.Hex())
+	}
+	return nil
+}
+
+func (b *BleveBackend) indexEvent(evt nostr.Event) error {
+	docID := evt.ID
+
+	var references []string
+	var extras string
+
+	switch evt.Kind {
+	case 6, 16:
+		var innerEvt nostr.Event
+		if err := json.Unmarshal([]byte(evt.Content), &innerEvt); err != nil || !innerEvt.VerifySignature() {
+			return nil
+		}
+		evt = innerEvt
+	case 0:
+		var pm sdk.ProfileMetadata
+		if err := json.Unmarshal([]byte(evt.Content), &pm); err == nil {
+			evt.Content = pm.Name + "\n" + pm.DisplayName + "\n" + pm.About
+			references = append(references, pm.NIP05)
+		}
+	}
+
+	for _, tag := range evt.Tags {
+		if len(tag) < 2 {
+			continue
+		}
+		switch tag[0] {
+		case "comment", "name", "title", "about", "description":
+			evt.Content += "\n\n" + tag[1]
+		case "e":
+			if ptr, err := nostr.EventPointerFromTag(tag); err == nil {
+				references = append(references, ptr.AsTagReference())
+			}
+		case "a":
+			if ptr, err := nostr.EntityPointerFromTag(tag); err == nil {
+				references = append(references, ptr.AsTagReference())
+			}
+		case "r":
+			references = append(references, tag[1])
+		}
+	}
+
+	doc := map[string]any{
+		labelKindField:      strconv.Itoa(int(evt.Kind)),
+		labelAuthorField:    evt.PubKey.Hex()[56:],
+		labelCreatedAtField: evt.CreatedAt.Time(),
+	}
+
+	content := strings.Builder{}
+	content.Grow(len(evt.Content))
+
+	for block := range nip27.Parse(evt.Content) {
+		if block.Pointer == nil {
+			content.WriteString(strings.TrimSpace(block.Text))
+		} else {
+			references = append(references, block.Pointer.AsTagReference())
+			if ep, ok := block.Pointer.(nip73.ExternalPointer); ok {
+				extras += ep.Thing + " "
+			}
+		}
+	}
+
+	indexableContent := content.String()
+
+	var lang lingua.Language
+	if len(b.Languages) == 1 {
+		lang = b.Languages[0]
+	} else {
+		var ok bool
+		lang, ok = b.detector.DetectLanguageOf(indexableContent)
+		if !ok {
+			lang = lingua.English
+		}
+	}
+
+	var analyzerLangCode string
+	switch lang {
+	case lingua.Japanese, lingua.Chinese, lingua.Korean:
+		analyzerLangCode = "cjk"
+	default:
+		analyzerLangCode = strings.ToLower(lang.IsoCode639_1().String())
+	}
+	doc[labelContentField+"_"+analyzerLangCode] = indexableContent
+
+	doc[labelReferencesField] = references
+	doc[labelExtrasField] = extras
+
+	if err := b.index.Index(docID.Hex(), doc); err != nil {
+		return fmt.Errorf("failed to index '%s' document: %w", docID.Hex(), err)
+	}
+
 	return nil
 }
 
@@ -64,3 +333,154 @@ func (b *BleveBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 
 	return 0, errors.New("not supported")
 }
+
+func (b *BleveBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
+	return func(yield func(nostr.Event) bool) {
+		if tlimit := filter.GetTheoreticalLimit(); tlimit == 0 {
+			return
+		} else if tlimit < maxLimit {
+			maxLimit = tlimit
+		}
+
+		filter.Search = strings.TrimSpace(filter.Search)
+		if len(filter.Search) < 2 {
+			return
+		}
+
+		and := make([]bleveQuery.Query, 0, 3)
+
+		searchC := strings.Builder{}
+		searchC.Grow(len(filter.Search))
+
+		for block := range nip27.Parse(filter.Search) {
+			if block.Pointer != nil {
+				genericRef := bleve.NewTermQuery(block.Pointer.AsTagReference())
+				genericRef.SetField(labelReferencesField)
+				genericRef.SetBoost(2)
+
+				var ref bleveQuery.Query = genericRef
+				if profile, ok := block.Pointer.(nostr.ProfilePointer); ok {
+					authorQuery := bleve.NewTermQuery(profile.PublicKey.Hex()[56:])
+					authorQuery.SetField(labelAuthorField)
+					authorQuery.SetBoost(2)
+					orRef := bleve.NewDisjunctionQuery()
+					orRef.AddQuery(genericRef)
+					orRef.AddQuery(authorQuery)
+					ref = orRef
+				} else if addr, ok := block.Pointer.(nostr.EntityPointer); ok {
+					authorQuery := bleve.NewTermQuery(addr.PublicKey.Hex()[56:])
+					authorQuery.SetField(labelAuthorField)
+					authorQuery.SetBoost(2)
+					orRef := bleve.NewDisjunctionQuery()
+					orRef.AddQuery(genericRef)
+					orRef.AddQuery(authorQuery)
+					ref = orRef
+				}
+				and = append(and, ref)
+			} else {
+				searchC.WriteString(strings.TrimSpace(block.Text))
+			}
+		}
+
+		searchContent := searchC.String()
+
+		var exactMatches []string
+		if len(searchContent) > 0 {
+			contentQueries := make([]bleveQuery.Query, 0, len(b.Languages)+1)
+
+			searchQ, exactMatches_, err := parse(searchContent, labelContentField+"_"+b.languageCodes[0])
+			if err != nil {
+				for _, code := range b.languageCodes {
+					match := bleve.NewMatchQuery(searchContent)
+					match.SetField(labelContentField + "_" + code)
+					contentQueries = append(contentQueries, match)
+				}
+			} else {
+				contentQueries = append(contentQueries, searchQ)
+				for _, code := range b.languageCodes[1:] {
+					searchQ, _, _ := parse(searchContent, labelContentField+"_"+code)
+					contentQueries = append(contentQueries, searchQ)
+				}
+			}
+			exactMatches = exactMatches_
+
+			extrasQ := bleve.NewMatchQuery(searchContent)
+			extrasQ.SetField(labelExtrasField)
+			contentQueries = append(contentQueries, extrasQ)
+
+			and = append(and, bleveQuery.NewDisjunctionQuery(contentQueries))
+		}
+
+		if len(filter.Kinds) > 0 {
+			eitherKind := bleve.NewDisjunctionQuery()
+			for _, kind := range filter.Kinds {
+				kindQ := bleve.NewTermQuery(strconv.Itoa(int(kind)))
+				kindQ.SetField(labelKindField)
+				eitherKind.AddQuery(kindQ)
+			}
+			and = append(and, eitherKind)
+		}
+
+		if len(filter.Authors) > 0 {
+			eitherPubkey := bleve.NewDisjunctionQuery()
+			for _, pubkey := range filter.Authors {
+				pubkeyQ := bleve.NewTermQuery(pubkey.Hex()[56:])
+				pubkeyQ.SetField(labelAuthorField)
+				eitherPubkey.AddQuery(pubkeyQ)
+			}
+			and = append(and, eitherPubkey)
+		}
+
+		if filter.Since != 0 || filter.Until != 0 {
+			var min time.Time
+			if filter.Since != 0 {
+				min = filter.Since.Time()
+			}
+			var max time.Time
+			if filter.Until != 0 {
+				max = filter.Until.Time()
+			} else {
+				max = time.Now()
+			}
+			dateRangeQ := bleve.NewDateRangeQuery(min, max)
+			dateRangeQ.SetField(labelCreatedAtField)
+			and = append(and, dateRangeQ)
+		}
+
+		q := bleveQuery.NewConjunctionQuery(and)
+		req := bleve.NewSearchRequest(q)
+		req.Size = maxLimit
+		req.From = 0
+		req.Explain = true
+
+		result, err := b.index.Search(req)
+		if err != nil {
+			return
+		}
+
+	resultHit:
+		for _, hit := range result.Hits {
+			id, err := nostr.IDFromHex(hit.ID)
+			if err != nil {
+				continue
+			}
+			for evt := range b.RawEventStore.QueryEvents(nostr.Filter{IDs: []nostr.ID{id}}, 1) {
+				for _, exactMatch := range exactMatches {
+					if !strings.Contains(strings.ToLower(evt.Content), exactMatch) {
+						continue resultHit
+					}
+				}
+
+				for f, v := range filter.Tags {
+					if !evt.Tags.ContainsAny(f, v) {
+						continue resultHit
+					}
+				}
+
+				if !yield(evt) {
+					return
+				}
+			}
+		}
+	}
+}

eventstore/bleve/query.go

@@ -1,94 +0,0 @@
-package bleve
-
-import (
-	"iter"
-	"strconv"
-
-	"fiatjaf.com/nostr"
-	bleve "github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/v2/search/query"
-)
-
-func (b *BleveBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
-	return func(yield func(nostr.Event) bool) {
-		if tlimit := filter.GetTheoreticalLimit(); tlimit == 0 {
-			return
-		} else if tlimit < maxLimit {
-			maxLimit = tlimit
-		}
-
-		if len(filter.Search) < 2 {
-			return
-		}
-
-		searchQ := bleve.NewMatchQuery(filter.Search)
-		searchQ.SetField(contentField)
-		var q query.Query = searchQ
-
-		conjQueries := []query.Query{searchQ}
-
-		if len(filter.Kinds) > 0 {
-			eitherKind := bleve.NewDisjunctionQuery()
-			for _, kind := range filter.Kinds {
-				kindQ := bleve.NewTermQuery(strconv.Itoa(int(kind)))
-				kindQ.SetField(kindField)
-				eitherKind.AddQuery(kindQ)
-			}
-			conjQueries = append(conjQueries, eitherKind)
-		}
-
-		if len(filter.Authors) > 0 {
-			eitherPubkey := bleve.NewDisjunctionQuery()
-			for _, pubkey := range filter.Authors {
-				if len(pubkey) != 64 {
-					continue
-				}
-				pubkeyQ := bleve.NewTermQuery(pubkey.Hex()[56:])
-				pubkeyQ.SetField(pubkeyField)
-				eitherPubkey.AddQuery(pubkeyQ)
-			}
-			conjQueries = append(conjQueries, eitherPubkey)
-		}
-
-		if filter.Since != 0 || filter.Until != 0 {
-			var min *float64
-			if filter.Since != 0 {
-				minVal := float64(filter.Since)
-				min = &minVal
-			}
-			var max *float64
-			if filter.Until != 0 {
-				maxVal := float64(filter.Until)
-				max = &maxVal
-			}
-			dateRangeQ := bleve.NewNumericRangeInclusiveQuery(min, max, nil, nil)
-			dateRangeQ.SetField(createdAtField)
-			conjQueries = append(conjQueries, dateRangeQ)
-		}
-
-		if len(conjQueries) > 1 {
-			q = bleve.NewConjunctionQuery(conjQueries...)
-		}
-
-		req := bleve.NewSearchRequest(q)
-		req.Size = maxLimit
-		req.From = 0
-
-		result, err := b.index.Search(req)
-		if err != nil {
-			return
-		}
-
-		for _, hit := range result.Hits {
-			id, err := nostr.IDFromHex(hit.ID)
-			if err != nil {
-				continue
-			}
-			for evt := range b.RawEventStore.QueryEvents(nostr.Filter{IDs: []nostr.ID{id}}, 1) {
-				if !yield(evt) {
-					return
-				}
-			}
-		}
-	}
-}

eventstore/bleve/query_parser.go

@@ -0,0 +1,209 @@
+package bleve
+
+import (
+	"strings"
+
+	bleve "github.com/blevesearch/bleve/v2"
+	bleveQuery "github.com/blevesearch/bleve/v2/search/query"
+)
+
+// token types
+type TokenType int
+
+const (
+	TokenWord TokenType = iota
+	TokenOR
+	TokenAND
+	TokenNOT
+	TokenLParen
+	TokenRParen
+	TokenQuote
+	TokenEOF
+)
+
+type Token struct {
+	Type  TokenType
+	Value string
+}
+
+type Parser struct {
+	lexer *Lexer
+	field string
+}
+
+func parse(input string, field string) (bleveQuery.Query, []string, error) {
+	lexer := NewLexer(input)
+	p := &Parser{
+		lexer: lexer,
+	}
+
+	var exactMatches []string
+	var reusableCurrentMatch strings.Builder
+	var currentExactMatch *strings.Builder
+	var currentWords []string
+	var negated bool
+	var parents []bleveQuery.Query
+	var parentOps []TokenType       // tracks if parent should be AND or OR
+	var lastOp TokenType = TokenAND // track last operator for parentheses
+
+	curr := bleve.NewBooleanQuery()
+
+	for {
+		token := p.lexer.NextToken()
+
+		if token.Type == TokenEOF {
+			if len(currentWords) > 0 {
+				match := bleve.NewMatchQuery(strings.Join(currentWords, " "))
+				match.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+				match.SetField(field)
+				if negated {
+					curr.AddMustNot(match)
+				} else {
+					curr.AddMust(match)
+				}
+			}
+			break
+		}
+
+		if token.Type == TokenQuote {
+			if currentExactMatch == nil {
+				currentExactMatch = &reusableCurrentMatch
+			} else {
+				exactMatches = append(exactMatches, currentExactMatch.String())
+				currentExactMatch.Reset()
+				reusableCurrentMatch = *currentExactMatch
+				currentExactMatch = nil
+			}
+			continue
+		}
+
+		if currentExactMatch != nil {
+			if currentExactMatch.Len() > 0 {
+				currentExactMatch.WriteByte(' ')
+			}
+			currentExactMatch.WriteString(strings.ToLower(token.Value))
+			currentWords = append(currentWords, token.Value)
+			continue
+		}
+
+		if token.Type == TokenWord {
+			currentWords = append(currentWords, token.Value)
+			continue
+		} else if len(currentWords) > 0 {
+			match := bleve.NewMatchQuery(strings.Join(currentWords, " "))
+			match.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+			match.SetField(field)
+			if negated {
+				curr.AddMustNot(match)
+			} else {
+				curr.AddMust(match)
+			}
+			currentWords = currentWords[:0]
+			negated = false
+		}
+
+		switch token.Type {
+		case TokenLParen:
+			// push current query to parents stack with the last operator
+			parents = append(parents, curr)
+			parentOps = append(parentOps, lastOp)
+			// reset lastOp to default for inner parentheses
+			lastOp = TokenAND
+			// start new boolean query for parentheses content
+			curr = bleve.NewBooleanQuery()
+			continue
+		case TokenRParen:
+			// finalize any remaining words
+			if len(currentWords) > 0 {
+				match := bleve.NewMatchQuery(strings.Join(currentWords, " "))
+				match.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+				match.SetField(field)
+				if negated {
+					curr.AddMustNot(match)
+				} else {
+					curr.AddMust(match)
+				}
+				currentWords = currentWords[:0]
+				negated = false
+			}
+
+			// pop parent and merge with current
+			if len(parents) > 0 {
+				parent := parents[len(parents)-1]
+				op := parentOps[len(parentOps)-1]
+
+				// create a new boolean query to combine parent and current
+				var combined bleveQuery.Query
+				switch op {
+				case TokenOR:
+					or := bleve.NewDisjunctionQuery()
+					or.AddQuery(parent)
+					or.AddQuery(curr)
+					combined = or
+				case TokenAND:
+					and := bleve.NewConjunctionQuery()
+					and.AddQuery(parent)
+					and.AddQuery(curr)
+					combined = and
+				}
+
+				curr = bleve.NewBooleanQuery()
+				curr.AddMust(combined)
+				parents = parents[:len(parents)-1]
+				parentOps = parentOps[:len(parentOps)-1]
+			}
+			continue
+		}
+
+		next := p.lexer.NextToken()
+		following := p.lexer.PeekToken()
+		if next.Type == TokenNOT {
+			negated = true
+		}
+
+		switch token.Type {
+		case TokenOR:
+			if next.Type != TokenLParen && !(next.Type == TokenNOT && following.Type == TokenLParen) {
+				// if this is not followed by a "(" or "NOT (" consider the follow next word as the only parameter
+				other := bleve.NewMatchQuery(next.Value)
+				other.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+				other.SetField(field)
+				or := bleve.NewDisjunctionQuery()
+				or.AddQuery(curr)
+				or.AddQuery(other)
+				curr = bleve.NewBooleanQuery()
+				curr.AddMust(or)
+			} else {
+				lastOp = TokenOR
+			}
+		case TokenAND:
+			if next.Type != TokenLParen && !(next.Type == TokenNOT && following.Type == TokenLParen) {
+				// if this is not followed by a "(" consider the follow next word as the only parameter
+				other := bleve.NewMatchQuery(next.Value)
+				other.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+				other.SetField(field)
+				and := bleve.NewConjunctionQuery()
+				and.AddQuery(curr)
+				and.AddQuery(other)
+				curr = bleve.NewBooleanQuery()
+				curr.AddMust(and)
+			} else {
+				lastOp = TokenAND
+			}
+		case TokenNOT:
+			if next.Type != TokenLParen {
+				// if this is not followed by a "(" or "NOT (" consider the follow next word as the only parameter
+				other := bleve.NewMatchQuery(next.Value)
+				other.SetOperator(bleveQuery.MatchQueryOperatorAnd)
+				other.SetField(field)
+				curr.AddMustNot(other)
+			} else {
+				negated = true
+			}
+		default:
+			p.lexer.ReturnToken(next)
+		}
+	}
+
+	return curr, exactMatches, nil
+}

eventstore/bleve/query_parser_test.go

@@ -0,0 +1,57 @@
+package bleve
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseQuery(t *testing.T) {
+	mapping := bleve.NewIndexMapping()
+	mapping.DefaultAnalyzer = "en"
+	index, err := bleve.NewMemOnly(mapping)
+	require.NoError(t, err)
+
+	docs := []map[string]interface{}{
+		{"id": "1", "phrase": "I like fruit especially banana and strawberry"},
+		{"id": "2", "phrase": "I like fruit like apples and oranges"},
+		{"id": "3", "phrase": "I like vegetables but not fruit"},
+		{"id": "4", "phrase": "Banana bread is delicious"},
+		{"id": "5", "phrase": "Strawberry jam and banana smoothie"},
+	}
+
+	for _, doc := range docs {
+		err := index.Index(doc["id"].(string), doc)
+		require.NoError(t, err)
+	}
+
+	testQueries := []struct {
+		query        string
+		expected     int
+		exactMatches []string
+	}{
+		{"fruit", 3, nil},
+		{"banana (NOT delicious)", 2, nil},
+		{"banana (NOT delicious) bread", 0, nil},
+		{"smoothie OR apples", 2, nil},
+		{"smoothie OR apples (NOT fruit)", 1, nil},
+		{"\"I like\"", 3, []string{"i like"}},
+		{"banana \"I like fruit\" strawberries", 1, []string{"i like fruit"}},
+		{"\"I like fruit\" (strawberry OR apple)", 2, []string{"i like fruit"}},
+	}
+
+	for _, test := range testQueries {
+		query, exactMatches, err := parse(test.query, "phrase")
+		require.NoError(t, err)
+
+		require.Equal(t, test.exactMatches, exactMatches)
+
+		search := bleve.NewSearchRequest(query)
+		results, err := index.Search(search)
+		require.NoError(t, err)
+
+		require.Equal(t, test.expected, int(results.Total),
+			"query '%s' expected %d results, got %d", test.query, test.expected, results.Total)
+	}
+}

eventstore/bleve/replace.go

@@ -1,37 +0,0 @@
-package bleve
-
-import (
-	"fmt"
-
-	"fiatjaf.com/nostr"
-	"fiatjaf.com/nostr/eventstore"
-)
-
-func (b *BleveBackend) ReplaceEvent(evt nostr.Event) error {
-	b.Lock()
-	defer b.Unlock()
-
-	filter := nostr.Filter{Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}}
-	if evt.Kind.IsAddressable() {
-		filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
-	}
-
-	shouldStore := true
-	for previous := range b.QueryEvents(filter, 1) {
-		if nostr.IsOlder(previous, evt) {
-			if err := b.DeleteEvent(previous.ID); err != nil {
-				return fmt.Errorf("failed to delete event for replacing: %w", err)
-			}
-		} else {
-			shouldStore = false
-		}
-	}
-
-	if shouldStore {
-		if err := b.SaveEvent(evt); err != nil && err != eventstore.ErrDupEvent {
-			return fmt.Errorf("failed to save: %w", err)
-		}
-	}
-
-	return nil
-}

eventstore/bleve/save.go

@@ -1,23 +0,0 @@
-package bleve
-
-import (
-	"fmt"
-	"strconv"
-
-	"fiatjaf.com/nostr"
-)
-
-func (b *BleveBackend) SaveEvent(evt nostr.Event) error {
-	doc := map[string]interface{}{
-		contentField:   evt.Content,
-		kindField:      strconv.Itoa(int(evt.Kind)),
-		pubkeyField:    evt.PubKey.Hex()[56:],
-		createdAtField: float64(evt.CreatedAt),
-	}
-
-	if err := b.index.Index(evt.ID.Hex(), doc); err != nil {
-		return fmt.Errorf("failed to index '%s' document: %w", evt.ID, err)
-	}
-
-	return nil
-}

eventstore/boltdb/lib.go

@@ -28,6 +28,8 @@ type BoltBackend struct {
 	MapSize int64
 	DB      *bbolt.DB
 
+	ReadOnly bool
+
 	EnableHLLCacheFor func(kind nostr.Kind) (useCache bool, skipSavingActualEvent bool)
 }
 
@@ -36,6 +38,7 @@ func (b *BoltBackend) Init() error {
 		Timeout:         2 * time.Second,
 		PreLoadFreelist: true,
 		FreelistType:    bbolt.FreelistMapType,
+		ReadOnly:        b.ReadOnly,
 	})
 	if err != nil {
 		return err

eventstore/boltdb/replace.go

@@ -8,8 +8,8 @@ import (
 	"go.etcd.io/bbolt"
 )
 
-func (b *BoltBackend) ReplaceEvent(evt nostr.Event) error {
+func (b *BoltBackend) ReplaceEvent(evt nostr.Event) (deleted []nostr.Event, err error) {
-	return b.DB.Update(func(txn *bbolt.Tx) error {
+	err = b.DB.Update(func(txn *bbolt.Tx) error {
 		rawBucket := txn.Bucket(rawEventStore)
 
 		// check if we already have this id
@@ -25,12 +25,12 @@ func (b *BoltBackend) ReplaceEvent(evt nostr.Event) error {
 		}
 
 		// now we fetch the past events, whatever they are, delete them and then save the new
-		var err error
+		var qerr error
 		var results iter.Seq[nostr.Event] = func(yield func(nostr.Event) bool) {
-			err = b.query(txn, filter, 10 /* in theory limit could be just 1 and this should work */, yield)
+			qerr = b.query(txn, filter, 10 /* in theory limit could be just 1 and this should work */, yield)
 		}
-		if err != nil {
+		if qerr != nil {
-			return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+			return fmt.Errorf("failed to query past events with %s: %w", filter, qerr)
 		}
 
 		shouldStore := true
@@ -39,6 +39,7 @@ func (b *BoltBackend) ReplaceEvent(evt nostr.Event) error {
 				if err := b.delete(txn, previous.ID); err != nil {
 					return fmt.Errorf("failed to delete event %s for replacing: %w", previous.ID, err)
 				}
+				deleted = append(deleted, previous)
 			} else {
 				// there is a newer event already stored, so we won't store this
 				shouldStore = false
@@ -50,4 +51,5 @@ func (b *BoltBackend) ReplaceEvent(evt nostr.Event) error {
 
 		return nil
 	})
+	return deleted, err
 }

eventstore/cmd/eventstore/main.go

@@ -19,7 +19,7 @@ import (
 
 var (
 	db  eventstore.Store
-	end func()
+	end = func() {}
 )
 
 var app = &cli.Command{

eventstore/codec/betterbinary/codec.go

@@ -40,12 +40,12 @@ func Marshal(evt nostr.Event, buf []byte) error {
 	buf[0] = 0
 
 	if evt.Kind > MaxKind {
-		return fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, MaxKind)
+		return fmt.Errorf("kind is too big: %d, max is %d", evt.Kind, uint16(MaxKind))
 	}
 	binary.LittleEndian.PutUint16(buf[1:3], uint16(evt.Kind))
 
 	if evt.CreatedAt > MaxCreatedAt {
-		return fmt.Errorf("created_at is too big: %d, max is %d", evt.CreatedAt, MaxCreatedAt)
+		return fmt.Errorf("created_at is too big: %d, max is %d", evt.CreatedAt, uint32(MaxCreatedAt))
 	}
 	binary.LittleEndian.PutUint32(buf[3:7], uint32(evt.CreatedAt))
 
@@ -58,7 +58,7 @@ func Marshal(evt nostr.Event, buf []byte) error {
 
 	ntags := len(evt.Tags)
 	if ntags > MaxTagCount {
-		return fmt.Errorf("can't encode too many tags: %d, max is %d", ntags, MaxTagCount)
+		return fmt.Errorf("can't encode too many tags: %d, max is %d", ntags, uint16(MaxTagCount))
 	}
 	binary.LittleEndian.PutUint16(buf[137:139], uint16(ntags))
 
@@ -68,7 +68,7 @@ func Marshal(evt nostr.Event, buf []byte) error {
 
 		itemCount := len(tag)
 		if itemCount > MaxTagItemCount {
-			return fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, MaxTagItemCount)
+			return fmt.Errorf("can't encode a tag with so many items: %d, max is %d", itemCount, uint8(MaxTagItemCount))
 		}
 		buf[tagBase+tagOffset] = uint8(itemCount)
 
@@ -76,7 +76,7 @@ func Marshal(evt nostr.Event, buf []byte) error {
 		for _, item := range tag {
 			itemSize := len(item)
 			if itemSize > MaxTagItemSize {
-				return fmt.Errorf("tag item is too large: %d, max is %d", itemSize, MaxTagItemSize)
+				return fmt.Errorf("tag item is too large: %d, max is %d", itemSize, uint16(MaxTagItemSize))
 			}
 
 			binary.LittleEndian.PutUint16(buf[tagBase+tagOffset+itemOffset:], uint16(itemSize))
@@ -91,7 +91,7 @@ func Marshal(evt nostr.Event, buf []byte) error {
 
 	// content
 	if contentLength := len(evt.Content); contentLength > MaxContentSize {
-		return fmt.Errorf("content is too large: %d, max is %d", contentLength, MaxContentSize)
+		return fmt.Errorf("content is too large: %d, max is %d", contentLength, uint16(MaxContentSize))
 	} else {
 		binary.LittleEndian.PutUint16(buf[tagBase+tagsSectionLength:], uint16(contentLength))
 	}

eventstore/internal/checks/interface.go

@@ -2,7 +2,6 @@ package checks
 
 import (
 	"fiatjaf.com/nostr/eventstore"
-	"fiatjaf.com/nostr/eventstore/bleve"
 	"fiatjaf.com/nostr/eventstore/boltdb"
 	"fiatjaf.com/nostr/eventstore/lmdb"
 	"fiatjaf.com/nostr/eventstore/mmm"
@@ -13,5 +12,4 @@ var (
 	_ eventstore.Store = (*lmdb.LMDBBackend)(nil)
 	_ eventstore.Store = (*mmm.IndexingLayer)(nil)
 	_ eventstore.Store = (*boltdb.BoltBackend)(nil)
-	_ eventstore.Store = (*bleve.BleveBackend)(nil)
 )

eventstore/lmdb/count.go

@@ -36,19 +36,16 @@ func (b *LMDBBackend) CountEvents(filter nostr.Filter) (uint32, error) {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
 				if it.exhausted ||
 					it.err != nil ||
-					len(it.key) != q.keySize ||
+					len(it.key) != len(q.prefix)+4 ||
 					!bytes.HasPrefix(it.key, q.prefix) {
 					// either iteration has errored or we reached the end of this prefix
 					break // stop this cursor and move to the next one
 				}
 
-				// "id" indexes don't contain a timestamp
-				if q.dbi != b.indexId {
 				createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
 				if createdAt < since {
 					break
 				}
-				}
 
 				if extraAuthors == nil && extraKinds == nil && extraTagValues == nil {
 					count++
@@ -129,19 +126,16 @@ func (b *LMDBBackend) CountEventsHLL(filter nostr.Filter, offset int) (uint32, *
 			for {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
 				if it.err != nil ||
-					len(it.key) != q.keySize ||
+					len(it.key) != len(q.prefix)+4 ||
 					!bytes.HasPrefix(it.key, q.prefix) {
 					// either iteration has errored or we reached the end of this prefix
 					break // stop this cursor and move to the next one
 				}
 
-				// "id" indexes don't contain a timestamp
-				if q.dbi != b.indexId {
 				createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
 				if createdAt < since {
 					break
 				}
-				}
 
 				// fetch actual event (we need it regardless because we need the pubkey for the hll)
 				bin, err := txn.Get(b.rawEventStore, it.valIdx)

eventstore/lmdb/helpers.go

@@ -45,7 +45,7 @@ func (it *iterator) pull(n int, since uint32) {
 			return
 		}
 
-		if len(it.key) != query.keySize || !bytes.HasPrefix(it.key, query.prefix) {
+		if len(it.key) != len(query.prefix)+4 || !bytes.HasPrefix(it.key, query.prefix) {
 			// we reached the end of this prefix
 			it.exhausted = true
 			return

eventstore/lmdb/lib.go

@@ -4,7 +4,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"os"
-	"sync/atomic"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore"
@@ -34,8 +33,6 @@ type LMDBBackend struct {
 
 	hllCache          lmdb.DBI
 	EnableHLLCacheFor func(kind nostr.Kind) (useCache bool, skipSavingActualEvent bool)
-
-	lastId atomic.Uint32
 }
 
 func (b *LMDBBackend) Init() error {
@@ -112,7 +109,7 @@ func (b *LMDBBackend) initialize() error {
 		env.SetMapSize(b.MapSize)
 	}
 
-	if err := env.Open(b.Path, lmdb.NoTLS|lmdb.WriteMap|b.extraFlags, 0644); err != nil {
+	if err := env.Open(b.Path, lmdb.NoTLS|b.extraFlags, 0644); err != nil {
 		return err
 	}
 	b.lmdbEnv = env
@@ -186,28 +183,5 @@ func (b *LMDBBackend) initialize() error {
 		return err
 	}
 
-	// get lastId
-	if err := b.lmdbEnv.View(func(txn *lmdb.Txn) error {
-		txn.RawRead = true
-		cursor, err := txn.OpenCursor(b.rawEventStore)
-		if err != nil {
-			return err
-		}
-		defer cursor.Close()
-		k, _, err := cursor.Get(nil, nil, lmdb.Last)
-		if lmdb.IsNotFound(err) {
-			// nothing found, so we're at zero
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-		b.lastId.Store(binary.BigEndian.Uint32(k))
-
-		return nil
-	}); err != nil {
-		return err
-	}
-
 	return b.migrate()
 }

eventstore/lmdb/query.go

@@ -54,7 +54,6 @@ func (b *LMDBBackend) queryByIds(txn *lmdb.Txn, ids []nostr.ID, yield func(nostr
 			continue
 		}
 
-		txn.Get(b.rawEventStore, idx)
 		bin, err := txn.Get(b.rawEventStore, idx)
 		if err != nil {
 			continue

eventstore/lmdb/query_planner.go

@@ -14,7 +14,6 @@ type query struct {
 	i             int
 	dbi           lmdb.DBI
 	prefix        []byte
-	keySize       int
 	startingPoint []byte
 }
 
@@ -40,10 +39,10 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
 			}
 		}
 		for i, q := range queries {
-			sp := make([]byte, len(q.prefix))
+			sp := make([]byte, len(q.prefix)+4)
-			sp = sp[0:len(q.prefix)]
+			copy(sp[0:len(q.prefix)], q.prefix)
-			copy(sp, q.prefix)
+			binary.BigEndian.PutUint32(sp[len(q.prefix):], uint32(until))
-			queries[i].startingPoint = binary.BigEndian.AppendUint32(sp, uint32(until))
+			queries[i].startingPoint = sp
 		}
 	}()
 
@@ -64,10 +63,9 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
 		}
 
 		// only "p" tag has a goodness of 2, so
-		if goodness == 2 {
+		if goodness == 2 && filter.Kinds != nil {
 			// this means we got a "p" tag, so we will use the ptag-kind index
 			i := 0
-			if filter.Kinds != nil {
 			queries = make([]query, len(tagValues)*len(filter.Kinds))
 			for _, value := range tagValues {
 				if len(value) != 64 {
@@ -80,25 +78,14 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
 						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
 					}
 					binary.BigEndian.PutUint16(k[8:8+2], uint16(kind))
-						queries[i] = query{i: i, dbi: b.indexPTagKind, prefix: k[0 : 8+2], keySize: 8 + 2 + 4}
+					queries[i] = query{
+						i:      i,
+						dbi:    b.indexPTagKind,
+						prefix: k[0 : 8+2],
+					}
 					i++
 				}
 			}
-			} else {
-				// even if there are no kinds, in that case we will just return any kind and not care
-				queries = make([]query, len(tagValues))
-				for i, value := range tagValues {
-					if len(value) != 64 {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
-					}
-
-					k := make([]byte, 8)
-					if err := xhex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
-					}
-					queries[i] = query{i: i, dbi: b.indexPTagKind, prefix: k[0:8], keySize: 8 + 2 + 4}
-				}
-			}
 		} else {
 			// otherwise we will use a plain tag index
 			queries = make([]query, len(tagValues))
@@ -107,7 +94,11 @@ func (b *LMDBBackend) prepareQueries(filter nostr.Filter) (
 				dbi, k, offset := b.getTagIndexPrefix(tagKey, value)
 				// remove the last parts part to get just the prefix we want here
 				prefix := k[0:offset]
-				queries[i] = query{i: i, dbi: dbi, prefix: prefix, keySize: len(prefix) + 4}
+				queries[i] = query{
+					i:      i,
+					dbi:    dbi,
+					prefix: prefix,
+				}
 			}
 
 			// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
@@ -142,7 +133,11 @@ pubkeyMatching:
 			// will use pubkey index
 			queries = make([]query, len(filter.Authors))
 			for i, pk := range filter.Authors {
-				queries[i] = query{i: i, dbi: b.indexPubkey, prefix: pk[0:8], keySize: 8 + 4}
+				queries[i] = query{
+					i:      i,
+					dbi:    b.indexPubkey,
+					prefix: pk[0:8],
+				}
 			}
 		} else {
 			// will use pubkeyKind index
@@ -153,7 +148,11 @@ pubkeyMatching:
 					prefix := make([]byte, 8+2)
 					copy(prefix[0:8], pk[0:8])
 					binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
-					queries[i] = query{i: i, dbi: b.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4}
+					queries[i] = query{
+						i:      i,
+						dbi:    b.indexPubkeyKind,
+						prefix: prefix[0 : 8+2],
+					}
 					i++
 				}
 			}
@@ -170,7 +169,11 @@ pubkeyMatching:
 		for i, kind := range filter.Kinds {
 			prefix := make([]byte, 2)
 			binary.BigEndian.PutUint16(prefix[0:2], uint16(kind))
-			queries[i] = query{i: i, dbi: b.indexKind, prefix: prefix[0:2], keySize: 2 + 4}
+			queries[i] = query{
+				i:      i,
+				dbi:    b.indexKind,
+				prefix: prefix[0:2],
+			}
 		}
 
 		// potentially with an extra useless tag filtering
@@ -181,6 +184,10 @@ pubkeyMatching:
 	// if we got here our query will have nothing to filter with
 	queries = make([]query, 1)
 	prefix := make([]byte, 0)
-	queries[0] = query{i: 0, dbi: b.indexCreatedAt, prefix: prefix, keySize: 0 + 4}
+	queries[0] = query{
+		i:      0,
+		dbi:    b.indexCreatedAt,
+		prefix: prefix,
+	}
 	return queries, nil, nil, "", nil, since, nil
 }

eventstore/lmdb/replace.go

@@ -2,14 +2,13 @@ package lmdb
 
 import (
 	"fmt"
-	"iter"
 
 	"fiatjaf.com/nostr"
 	"github.com/PowerDNS/lmdb-go/lmdb"
 )
 
-func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) error {
+func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) (deleted []nostr.Event, err error) {
-	return b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
+	err = b.lmdbEnv.Update(func(txn *lmdb.Txn) error {
 		// check if we already have this id
 		_, existsErr := txn.Get(b.indexId, evt.ID[0:8])
 		if existsErr == nil {
@@ -26,24 +25,21 @@ func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) error {
 		}
 
 		// now we fetch the past events, whatever they are, delete them and then save the new
-		var err error
-		var results iter.Seq[nostr.Event] = func(yield func(nostr.Event) bool) {
-			err = b.query(txn, filter, 10 /* in theory limit could be just 1 and this should work */, yield)
-		}
-		if err != nil {
-			return fmt.Errorf("failed to query past events with %s: %w", filter, err)
-		}
-
 		shouldStore := true
-		for previous := range results {
+		if qerr := b.query(txn, filter, 10 /* could be just 1 */, func(previous nostr.Event) bool {
 			if nostr.IsOlder(previous, evt) {
-				if err := b.delete(txn, previous.ID); err != nil {
+				if qerr := b.delete(txn, previous.ID); qerr != nil {
-					return fmt.Errorf("failed to delete event %s for replacing: %w", previous.ID, err)
+					qerr = fmt.Errorf("failed to delete event %s for replacing: %w", previous.ID, qerr)
+					return false
 				}
+				deleted = append(deleted, previous)
 			} else {
 				// there is a newer event already stored, so we won't store this
 				shouldStore = false
 			}
+			return true
+		}); qerr != nil {
+			return fmt.Errorf("failed to query past events with %s: %w", filter, qerr)
 		}
 		if shouldStore {
 			return b.save(txn, evt)
@@ -51,4 +47,6 @@ func (b *LMDBBackend) ReplaceEvent(evt nostr.Event) error {
 
 		return nil
 	})
+
+	return deleted, err
 }

eventstore/mmm/count.go

@@ -33,19 +33,16 @@ func (il *IndexingLayer) CountEvents(filter nostr.Filter) (uint32, error) {
 				// we already have a k and a v and an err from the cursor setup, so check and use these
 				if it.exhausted ||
 					it.err != nil ||
-					len(it.key) != q.keySize ||
+					len(it.key) != len(q.prefix)+4 ||
 					!bytes.HasPrefix(it.key, q.prefix) {
 					// either iteration has errored or we reached the end of this prefix
 					break // stop this cursor and move to the next one
 				}
 
-				// "id" indexes don't contain a timestamp
-				if q.timestampSize == 4 {
 				createdAt := binary.BigEndian.Uint32(it.key[len(it.key)-4:])
 				if createdAt < since {
 					break
 				}
-				}
 
 				if extraAuthors == nil && extraKinds == nil && extraTagValues == nil {
 					count++

eventstore/mmm/fix.go

@@ -116,8 +116,7 @@ func (b *MultiMmapManager) Rescan() error {
 			}
 		}
 
-		b.freeRanges, err = b.gatherFreeRanges(mmmtxn)
+		if err := b.gatherFreeRanges(mmmtxn); err != nil {
-		if err != nil {
 			return err
 		}
 

eventstore/mmm/freeranges.go

@@ -2,16 +2,36 @@ package mmm
 
 import (
 	"cmp"
+	"encoding/binary"
 	"fmt"
+	"iter"
+	"runtime"
 	"slices"
+	"syscall"
+	"unsafe"
 
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore/codec/betterbinary"
 	"github.com/PowerDNS/lmdb-go/lmdb"
 )
 
-func (b *MultiMmapManager) gatherFreeRanges(txn *lmdb.Txn) (positions, error) {
+const LARGE_FREERANGE = 142
+
+// AllFreeRanges returns an iterator of (start_pos, size) of all free ranges, in positional order.
+func (b *MultiMmapManager) AllFreeRanges() iter.Seq2[uint64, uint32] {
+	return func(yield func(uint64, uint32) bool) {
+		for _, pos := range b.freeRangesAll {
+			if !yield(pos.start, pos.size) {
+				return
+			}
+		}
+	}
+}
+
+func (b *MultiMmapManager) gatherFreeRanges(txn *lmdb.Txn) error {
 	cursor, err := txn.OpenCursor(b.indexId)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open cursor on indexId: %w", err)
+		return fmt.Errorf("failed to open cursor on indexId: %w", err)
 	}
 	defer cursor.Close()
 
@@ -28,31 +48,35 @@ func (b *MultiMmapManager) gatherFreeRanges(txn *lmdb.Txn) (positions, error) {
 	usedPositions = append(usedPositions, position{start: b.mmapfEnd, size: 0})
 
 	// calculate free ranges as gaps between used positions
-	freeRanges := make(positions, 0, len(usedPositions)/2)
+	b.freeRangesAll = make(positions, 0, len(usedPositions))
+	b.freeRangesLarge = make([]position, 0, len(usedPositions)/10)
 	var currentStart uint64 = 0
 	for _, used := range usedPositions {
 		if used.start > currentStart {
 			// gap from currentStart to pos.start
 			freeSize := used.start - currentStart
 			if freeSize > 0 {
-				freeRanges = append(freeRanges, position{
+				fr := position{
 					start: currentStart,
 					size:  uint32(freeSize),
-				})
+				}
+				b.freeRangesAll = append(b.freeRangesAll, fr)
+				if fr.isLarge() {
+					b.freeRangesLarge = append(b.freeRangesLarge, fr)
+				}
 			}
 		}
 		currentStart = used.start + uint64(used.size)
 	}
 
-	return freeRanges, nil
+	return nil
 }
 
 func (b *MultiMmapManager) mergeNewFreeRange(newFreeRange position) {
 	// use binary search to find the insertion point for the new pos
-	idx, exists := slices.BinarySearchFunc(b.freeRanges, newFreeRange.start, func(item position, target uint64) int {
+	idx, exists := slices.BinarySearchFunc(b.freeRangesAll, newFreeRange.start, func(item position, target uint64) int {
 		return cmp.Compare(item.start, target)
 	})
-
 	if exists {
 		panic(fmt.Errorf("can't add free range that already exists: %s", newFreeRange))
 	}
@@ -62,7 +86,7 @@ func (b *MultiMmapManager) mergeNewFreeRange(newFreeRange position) {
 
 	// check the range immediately before
 	if idx > 0 {
-		before := b.freeRanges[idx-1]
+		before := b.freeRangesAll[idx-1]
 		if before.start+uint64(before.size) == newFreeRange.start {
 			deleteStart = idx - 1
 			deleting++
@@ -72,8 +96,8 @@ func (b *MultiMmapManager) mergeNewFreeRange(newFreeRange position) {
 	}
 
 	// check the range immediately after
-	if idx < len(b.freeRanges) {
+	if idx < len(b.freeRangesAll) {
-		after := b.freeRanges[idx]
+		after := b.freeRangesAll[idx]
 		if newFreeRange.start+uint64(newFreeRange.size) == after.start {
 			if deleteStart == -1 {
 				deleteStart = idx
@@ -87,13 +111,241 @@ func (b *MultiMmapManager) mergeNewFreeRange(newFreeRange position) {
 	switch deleting {
 	case 0:
 		// if we are not deleting anything we must insert the new free range
-		b.freeRanges = slices.Insert(b.freeRanges, idx, newFreeRange)
+		b.freeRangesAll = slices.Insert(b.freeRangesAll, idx, newFreeRange)
+
+		// if it's large add it to the list of large free ranges
+		if newFreeRange.isLarge() {
+			b.freeRangesLarge = append(b.freeRangesLarge, newFreeRange)
+		}
 	case 1:
+		deleted := b.freeRangesAll[deleteStart]
+
 		// if we're deleting a single range, don't delete it, modify it in-place instead.
-		b.freeRanges[deleteStart] = newFreeRange
+		b.freeRangesAll[deleteStart] = newFreeRange
+
+		// if the list we're modifying is in the list of large ranges modify it there too
+		if deleted.isLarge() {
+			for i, large := range b.freeRangesLarge {
+				if large.start == deleted.start {
+					b.freeRangesLarge[i] = newFreeRange
+					break
+				}
+			}
+		} else if newFreeRange.isLarge() {
+			// otherwise: if after modification it's big enough we should add it to list of large ranges
+			b.freeRangesLarge = append(b.freeRangesLarge, newFreeRange)
+		}
 	case 2:
-		// now if we're deleting two ranges, delete just one instead and modify the other in place
+		// now if we're deleting two ranges, delete the second instead and modify the first in place
-		b.freeRanges[deleteStart] = newFreeRange
+		first := b.freeRangesAll[deleteStart]
-		b.freeRanges = slices.Delete(b.freeRanges, deleteStart+1, deleteStart+1+1)
+		second := b.freeRangesAll[deleteStart+1]
+
+		b.freeRangesAll = slices.Delete(b.freeRangesAll, deleteStart+1, deleteStart+1+1)
+		b.freeRangesAll[deleteStart] = newFreeRange
+
+		// if the second was in the list of large lists delete it from there too
+		if second.isLarge() {
+			for i, large := range b.freeRangesLarge {
+				if large.start == second.start {
+					b.freeRangesLarge[i] = b.freeRangesLarge[len(b.freeRangesLarge)-1]
+					b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
+					break
 				}
 			}
+		}
+
+		// if the list we're modifying (the first) is already in the list of large ranges modify it there too
+		if first.isLarge() {
+			for i, large := range b.freeRangesLarge {
+				if large.start == first.start {
+					b.freeRangesLarge[i] = newFreeRange
+					break
+				}
+			}
+		} else if newFreeRange.isLarge() {
+			// otherwise if after modification has become big enough we should add it to list of large ranges
+			b.freeRangesLarge = append(b.freeRangesLarge, newFreeRange)
+		}
+	}
+}
+
+func (b *MultiMmapManager) Defragment(n int) error {
+	for range min(n, len(b.freeRangesAll)-1) {
+		if err := b.DefragmentOne(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Defragment a single free range
+func (b *MultiMmapManager) DefragmentOne() error {
+	if b.ReadOnly {
+		return ReadOnly
+	}
+
+	b.writeMutex.Lock()
+	defer b.writeMutex.Unlock()
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	if len(b.freeRangesAll) < 2 {
+		return nil
+	}
+
+	mmmtxn, err := b.lmdbEnv.BeginTxn(nil, 0)
+	if err != nil {
+		return fmt.Errorf("failed to begin mmm transaction: %w", err)
+	}
+	defer mmmtxn.Abort()
+
+	type layerTxn struct {
+		il  *IndexingLayer
+		txn *lmdb.Txn
+	}
+	layerTxns := make(map[uint16]*layerTxn)
+	defer func() {
+		for _, lt := range layerTxns {
+			lt.txn.Abort()
+		}
+	}()
+
+	// will put stuff into the first free range
+	fr := b.freeRangesAll[0]
+
+	// where the free range ends, the events start (any number of them)
+	eventsStart := fr.start + uint64(fr.size)
+	eventsEnd := b.freeRangesAll[1].start // and they end when the next free range starts
+
+	fmt.Println("# defrag", fr, eventsStart, eventsEnd)
+
+	c := uint64(0) // this tracks our relative position inside the events section
+	for (eventsStart + c) < eventsEnd {
+		var evt nostr.Event
+		if err := betterbinary.Unmarshal(b.mmapf[(eventsStart+c):eventsEnd], &evt); err != nil {
+			id := betterbinary.GetID(b.mmapf[(eventsStart + c):eventsEnd])
+			return fmt.Errorf("failed to read event (%x) from mmap: %w", id[:], err)
+		}
+
+		// now that we have an event we'll update its pos on the id index and on every layer:
+		oldVal, err := mmmtxn.Get(b.indexId, evt.ID[0:8])
+		if err != nil {
+			return fmt.Errorf("failed to read val (%x) from index: %w", evt.ID[:], err)
+		}
+
+		// current position
+		pos := positionFromBytes(oldVal[0:12])
+
+		// new position (from the beginning of the free range before + relative position)
+		fmt.Println("  moving event", evt.ID, "from", pos)
+		pos.start = fr.start + uint64(c)
+
+		// update this cursor
+		c += uint64(pos.size)
+		fmt.Println("  to", pos, "...", c, "layers:", oldVal[12:])
+
+		// prepare and save id index
+		newVal := make([]byte, len(oldVal))
+		writeBytesFromPosition(newVal, pos)
+		copy(newVal[12:], oldVal[12:])
+		if err := mmmtxn.Put(b.indexId, evt.ID[0:8], newVal, 0); err != nil {
+			return fmt.Errorf("failed to write new pos to id index: %w", err)
+		}
+
+		for s := 12; s < len(oldVal); s += 2 {
+			layer := binary.BigEndian.Uint16(oldVal[s : s+2])
+			lt, ok := layerTxns[layer]
+			if !ok {
+				il := b.layers.ByID(layer)
+				if il == nil {
+					fmt.Println(b.layers)
+					panic(fmt.Errorf("missing layer %d", layer))
+				}
+				txn, err := il.lmdbEnv.BeginTxn(nil, 0)
+				if err != nil {
+					return fmt.Errorf("failed to begin layer txn for layer %d: %w", il.id, err)
+				}
+				txn.RawRead = true
+				lt = &layerTxn{il: il, txn: txn}
+				layerTxns[il.id] = lt
+			}
+
+			fmt.Println("    layer", lt.il.id)
+
+			for k := range lt.il.getIndexKeysForEvent(evt) {
+				fmt.Println("      index", k.dbi, k.key)
+				if err := lt.txn.Del(k.dbi, k.key, oldVal[0:12]); err != nil {
+					return fmt.Errorf("failed to delete old index entry for %x: %w", evt.ID[:], err)
+				}
+				if err := lt.txn.Put(k.dbi, k.key, newVal[0:12], 0); err != nil {
+					return fmt.Errorf("failed to insert new index entry for %x: %w", evt.ID[:], err)
+				}
+			}
+		}
+	}
+
+	// now that we have updated all the pointers, just copy all the bytes between the two free ranges
+	copy(b.mmapf[fr.start:], b.mmapf[fr.start+uint64(fr.size):eventsEnd])
+
+	// delete this free range if it's one of the big ones
+	if fr.isLarge() {
+		for l, lfr := range b.freeRangesLarge {
+			if lfr.start == fr.start {
+				fmt.Println("  deleting large fr", l, lfr)
+				b.freeRangesLarge[l] = b.freeRangesLarge[len(b.freeRangesLarge)-1]
+				b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
+				break
+			}
+		}
+	}
+
+	// now we have some space left at the end of this events section that is a free range
+	remainingSpaceStart := fr.start + c
+
+	// it must be merged with the next free range
+	updated := position{
+		start: remainingSpaceStart,
+		size:  b.freeRangesAll[1].size + uint32(eventsEnd) - uint32(remainingSpaceStart),
+	}
+	nextWasLarge := b.freeRangesAll[1].isLarge()
+	fmt.Println("  updating next", updated)
+	b.freeRangesAll[1] = updated
+
+	if nextWasLarge {
+		for l, lfr := range b.freeRangesLarge {
+			if lfr.start == eventsEnd {
+				fmt.Println("it is large:", l, lfr, "(now", updated, ")")
+				b.freeRangesLarge[l] = updated
+				break
+			}
+		}
+	} else if updated.isLarge() {
+		// if it wasn't large but now is, add it to the list of large free ranges
+		fmt.Println("  a new large fr was created", updated)
+		b.freeRangesLarge = append(b.freeRangesLarge, updated)
+	}
+
+	// msync
+	_, _, errno := syscall.Syscall(syscall.SYS_MSYNC,
+		uintptr(unsafe.Pointer(&b.mmapf[0])), uintptr(len(b.mmapf)), syscall.MS_SYNC)
+	if errno != 0 {
+		return fmt.Errorf("msync failed: %w", syscall.Errno(errno))
+	}
+
+	// commit transactions
+	if err := mmmtxn.Commit(); err != nil {
+		return fmt.Errorf("failed to commit mmm transaction: %w", err)
+	}
+	for lid, lt := range layerTxns {
+		if err := lt.txn.Commit(); err != nil {
+			return fmt.Errorf("failed to commit layer %d transaction: %w", lid, err)
+		}
+	}
+
+	// delete the first free range
+	b.freeRangesAll = slices.Delete(b.freeRangesAll, 0, 1)
+
+	return nil
+}

eventstore/mmm/freeranges_test.go

@@ -1,8 +1,11 @@
 package mmm
 
 import (
+	"cmp"
+	"fmt"
 	"math/rand/v2"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 
@@ -45,7 +48,7 @@ func FuzzFreeRanges(f *testing.F) {
 
 		total := 0
 		for {
-			freeBefore, spaceBefore := countUsableFreeRanges(mmmm)
+			freeBefore, spaceBefore := countUsableFreeRanges(t, mmmm)
 
 			hasAdded := false
 			for i := range rnd.IntN(40) {
@@ -69,7 +72,7 @@ func FuzzFreeRanges(f *testing.F) {
 				total++
 			}
 
-			freeAfter, spaceAfter := countUsableFreeRanges(mmmm)
+			freeAfter, spaceAfter := countUsableFreeRanges(t, mmmm)
 			if hasAdded && freeBefore > 0 {
 				require.Lessf(t, spaceAfter, spaceBefore, "must use some of the existing free ranges when inserting new events (before: %d, after: %d)", freeBefore, freeAfter)
 			}
@@ -86,9 +89,35 @@ func FuzzFreeRanges(f *testing.F) {
 				}
 			}
 
+			verifyFreeRangesInvariants(t, mmmm)
+
+			// add more events
+			for i := range rnd.IntN(40) {
+				content := "1"
+				if i > 0 {
+					content = strings.Repeat("z", rnd.IntN(1000))
+				}
+
+				evt := nostr.Event{
+					CreatedAt: nostr.Timestamp(rnd.Uint32()),
+					Kind:      1,
+					Content:   content,
+					Tags:      nostr.Tags{},
+				}
+				evt.Sign(sk)
+				err := il.SaveEvent(evt)
+				require.NoError(t, err)
+
+				total++
+			}
+
+			verifyFreeRangesInvariants(t, mmmm)
+
 			mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
-				expectedFreeRanges, _ := mmmm.gatherFreeRanges(txn)
+				before := mmmm.freeRangesAll
-				require.Equalf(t, expectedFreeRanges, mmmm.freeRanges, "expected %s, got %s", expectedFreeRanges, mmmm.freeRanges)
+				err := mmmm.gatherFreeRanges(txn)
+				require.NoError(t, err)
+				require.Equalf(t, mmmm.freeRangesAll, before, "expected %s, got %s", before, mmmm.freeRangesAll)
 				return nil
 			})
 
@@ -99,12 +128,353 @@ func FuzzFreeRanges(f *testing.F) {
 	})
 }
 
-func countUsableFreeRanges(mmmm *MultiMmapManager) (count int, space int) {
+func TestDefragment(t *testing.T) {
-	for _, fr := range mmmm.freeRanges {
+	tmpDir, err := os.MkdirTemp("", "mmm-defrag-test-*")
-		if fr.size >= 142 {
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	mmmm := &MultiMmapManager{Dir: tmpDir}
+	err = mmmm.Init()
+	require.NoError(t, err)
+	defer mmmm.Close()
+
+	il, err := mmmm.EnsureLayer("a")
+	require.NoError(t, err)
+	defer il.Close()
+
+	sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
+
+	const nevents = 30
+	var stored [nevents]nostr.Event
+	for i := range nevents {
+		evt := nostr.Event{
+			CreatedAt: nostr.Timestamp(i),
+			Kind:      nostr.KindTextNote,
+			Tags:      nostr.Tags{},
+			Content:   fmt.Sprintf("============= event %d ============= "+strings.Repeat("+", 23), i),
+		}
+		evt.Sign(sk)
+		err := il.SaveEvent(evt)
+		require.NoError(t, err)
+		stored[i] = evt
+	}
+
+	toDelete := []int{0, 5, 10, 15, 20}
+	var remaining []nostr.Event
+	for i, evt := range stored {
+		if slices.Contains(toDelete, i) {
+			err := il.DeleteEvent(evt.ID)
+			require.NoError(t, err)
+		} else {
+			remaining = append(remaining, evt)
+		}
+	}
+
+	require.Len(t, toDelete, len(mmmm.freeRangesAll))
+
+	err = mmmm.Defragment(2)
+	require.NoError(t, err)
+
+	require.Len(t, mmmm.freeRangesAll, 3)
+	require.Len(t, remaining, nevents-len(toDelete))
+
+	// all remaining events still accessible with correct content via GetByID
+	for _, evt := range remaining {
+		gotEvt, layers := mmmm.GetByID(evt.ID)
+		require.NotNil(t, gotEvt, "event %s should exist after defrag", evt.ID)
+		require.NotEmpty(t, layers, "event %s should have layers after defrag", evt.ID)
+		require.Equal(t, evt.Content, gotEvt.Content, "event %s content should match after defrag", evt.ID)
+
+		// also accessible via a query
+		require.Equal(t, il, layers[0])
+	}
+
+	evts := slices.Collect(il.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{nostr.KindTextNote}}, 100))
+	require.Len(t, evts, nevents-len(toDelete))
+
+	// free range invariants hold after defrag
+	verifyFreeRangesInvariants(t, mmmm)
+
+	// no overlapping positions after defrag
+	mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		cursor, err := txn.OpenCursor(mmmm.indexId)
+		require.NoError(t, err)
+		defer cursor.Close()
+
+		var allPositions []position
+		for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
+			pos := positionFromBytes(val[0:12])
+			allPositions = append(allPositions, pos)
+		}
+
+		slices.SortFunc(allPositions, func(a, b position) int {
+			return cmp.Compare(a.start, b.start)
+		})
+
+		var lastEnd uint64
+		for _, pos := range allPositions {
+			if pos.start < lastEnd {
+				t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
+			}
+			lastEnd = pos.start + uint64(pos.size)
+		}
+		return nil
+	})
+}
+
+func countUsableFreeRanges(t *testing.T, mmmm *MultiMmapManager) (count int, space int) {
+	for _, fr := range mmmm.freeRangesAll {
+		if fr.size >= LARGE_FREERANGE {
 			count++
 			space += int(fr.size)
 		}
 	}
+
+	require.Equal(t, count, len(mmmm.freeRangesLarge))
+
 	return count, space
 }
+
+func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
+	all := mmmm.freeRangesAll
+	large := mmmm.freeRangesLarge
+
+	require.True(t, slices.IsSortedFunc(all, func(a, b position) int {
+		return cmp.Compare(a.start, b.start)
+	}), "free ranges aren't sorted by start position")
+
+	for _, l := range large {
+		require.True(t, l.isLarge())
+
+		found := false
+		for _, a := range all {
+			if l.start == a.start && l.size == a.size {
+				found = true
+				break
+			}
+		}
+		require.True(t, found, "large range %v not found in all ranges", l)
+	}
+
+	for i := 1; i < len(all); i++ {
+		require.Greater(t, all[i].start, all[i-1].start, "all ranges should be sorted by start")
+	}
+
+	for i, fr := range all {
+		for j := i + 1; j < len(all); j++ {
+			end1 := all[i].start + uint64(all[i].size)
+			end2 := all[j].start + uint64(all[j].size)
+			require.False(t, (all[i].start >= all[j].start && all[i].start < end2) ||
+				(all[j].start >= all[i].start && all[j].start < end1),
+				"ranges %v and %v overlap", all[i], all[j])
+		}
+
+		foundInLarge := false
+		for _, l := range large {
+			if l.start == fr.start {
+				foundInLarge = true
+				break
+			}
+		}
+		if !foundInLarge {
+			require.False(t, fr.isLarge())
+		}
+	}
+
+	mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		before := make(positions, len(mmmm.freeRangesAll))
+		copy(before, mmmm.freeRangesAll)
+		err := mmmm.gatherFreeRanges(txn)
+		require.NoError(t, err)
+		require.Equal(t, before, mmmm.freeRangesAll, "recomputing free ranges should yield the same result")
+		return nil
+	})
+}
+
+func FuzzDefragment(f *testing.F) {
+	f.Add(0)
+	f.Fuzz(func(t *testing.T, seed int) {
+		tmpDir, err := os.MkdirTemp("", "mmm-defrag-fuzz-*")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		logger := zerolog.Nop()
+		rnd := rand.New(rand.NewPCG(uint64(seed), 0))
+
+		mmmm := &MultiMmapManager{
+			Dir:    tmpDir,
+			Logger: &logger,
+		}
+		err = mmmm.Init()
+		require.NoError(t, err)
+		defer mmmm.Close()
+
+		layerNames := []string{"a", "b", "c"}
+		var layers []*IndexingLayer
+		for _, name := range layerNames {
+			il, err := mmmm.EnsureLayer(name)
+			require.NoError(t, err)
+			defer il.Close()
+			layers = append(layers, il)
+		}
+
+		type indexedEvent struct {
+			evt nostr.Event
+			tag string
+		}
+		layerEvents := make([][]indexedEvent, len(layers))
+
+		sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
+		pk := sk.Public()
+
+		totalEvents := rnd.IntN(500)
+		tChoices := []string{"foo", "bar", "banana"}
+		var written int
+
+		for written < totalEvents {
+			n := rnd.IntN(50) + 1
+			if n > totalEvents-written {
+				n = totalEvents - written
+			}
+
+			for i := 0; i < n; i++ {
+				sizeParam := rnd.IntN(2000)
+				content := strings.Repeat("z", sizeParam)
+
+				chosenTag := tChoices[rnd.IntN(3)]
+				evt := nostr.Event{
+					CreatedAt: nostr.Timestamp(rnd.Uint32()),
+					Kind:      nostr.KindTextNote,
+					Content:   content,
+					Tags:      nostr.Tags{{"t", chosenTag}},
+				}
+				evt.Sign(sk)
+
+				nLayers := rnd.IntN(len(layers)) + 1
+				perm := rnd.Perm(len(layers))
+				for pi := 0; pi < nLayers; pi++ {
+					li := perm[pi]
+					err := layers[li].SaveEvent(evt)
+					require.NoError(t, err)
+					layerEvents[li] = append(layerEvents[li], indexedEvent{evt, chosenTag})
+				}
+				written++
+			}
+
+			if n > 0 {
+				totalRemaining := 0
+				for _, levts := range layerEvents {
+					totalRemaining += len(levts)
+				}
+				if totalRemaining > 0 {
+					m := rnd.IntN(n)
+					if m > totalRemaining {
+						m = totalRemaining
+					}
+					for i := 0; i < m; i++ {
+						var nonEmpty []int
+						for li, levts := range layerEvents {
+							if len(levts) > 0 {
+								nonEmpty = append(nonEmpty, li)
+							}
+						}
+						if len(nonEmpty) == 0 {
+							break
+						}
+						li := nonEmpty[rnd.IntN(len(nonEmpty))]
+						idx := rnd.IntN(len(layerEvents[li]))
+						evtInfo := layerEvents[li][idx]
+						err := layers[li].DeleteEvent(evtInfo.evt.ID)
+						require.NoError(t, err)
+						layerEvents[li] = append(layerEvents[li][:idx], layerEvents[li][idx+1:]...)
+					}
+				}
+			}
+
+			if n > 0 {
+				o := rnd.IntN(n)
+				for i := 0; i < o; i++ {
+					if len(mmmm.freeRangesAll) > 1 {
+						param := rnd.IntN(len(mmmm.freeRangesAll))
+						err := mmmm.Defragment(param)
+						require.NoError(t, err)
+					}
+
+					verifyFreeRangesInvariants(t, mmmm)
+				}
+			}
+		}
+
+		// query each layer
+		for li, il := range layers {
+			levts := layerEvents[li]
+
+			// query by author
+			evts := slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}}, 10000))
+			require.Equal(t, len(levts), len(evts))
+
+			// query by author and kind
+			evts = slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}, Kinds: []nostr.Kind{nostr.KindTextNote}}, 10000))
+			require.Equal(t, len(levts), len(evts))
+
+			// query by "t" tag
+			for _, tagVal := range tChoices {
+				expected := 0
+				for _, ie := range levts {
+					if ie.tag == tagVal {
+						expected++
+					}
+				}
+				evts = slices.Collect(il.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"t": []string{tagVal}}}, 10000))
+				require.Equal(t, expected, len(evts))
+			}
+
+			// query with no parameters
+			allEvts := slices.Collect(il.QueryEvents(nostr.Filter{}, 10000))
+			require.Equal(t, len(levts), len(allEvts))
+		}
+
+		// build union of all events across all layers
+		allEventSet := make(map[string]nostr.Event)
+		for _, levts := range layerEvents {
+			for _, ie := range levts {
+				allEventSet[ie.evt.ID.String()] = ie.evt
+			}
+		}
+
+		// all events still accessible via GetByID
+		for _, evt := range allEventSet {
+			gotEvt, eventLayers := mmmm.GetByID(evt.ID)
+			require.NotNil(t, gotEvt)
+			require.NotEmpty(t, eventLayers)
+			require.Equal(t, evt.Content, gotEvt.Content)
+		}
+
+		verifyFreeRangesInvariants(t, mmmm)
+
+		mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+			cursor, err := txn.OpenCursor(mmmm.indexId)
+			require.NoError(t, err)
+			defer cursor.Close()
+
+			var allPositions []position
+			for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
+				pos := positionFromBytes(val[0:12])
+				allPositions = append(allPositions, pos)
+			}
+
+			slices.SortFunc(allPositions, func(a, b position) int {
+				return cmp.Compare(a.start, b.start)
+			})
+
+			var lastEnd uint64
+			for _, pos := range allPositions {
+				if pos.start < lastEnd {
+					t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
+				}
+				lastEnd = pos.start + uint64(pos.size)
+			}
+			return nil
+		})
+	})
+}

eventstore/mmm/fuzz_test.go

@@ -142,6 +142,11 @@ func FuzzTest(f *testing.F) {
 			mmmm.Rescan()
 		}
 
+		// perform random defrags -- shouldn't break the database
+		if rnd.UintN(3) == 1 {
+			mmmm.Defragment(len(deleted) / 3)
+		}
+
 		for id, deletedlayers := range deleted {
 			evt, foundlayers := mmmm.GetByID(id)
 

eventstore/mmm/helpers.go

@@ -41,7 +41,7 @@ func (it *iterator) pull(n int, since uint32) {
 			return
 		}
 
-		if len(it.key) != it.query.keySize || !bytes.HasPrefix(it.key, it.query.prefix) {
+		if len(it.key) != len(it.query.prefix)+4 || !bytes.HasPrefix(it.key, it.query.prefix) {
 			// we reached the end of this prefix
 			it.exhausted = true
 			return
@@ -226,7 +226,7 @@ func (il *IndexingLayer) getIndexKeysForEvent(evt nostr.Event) iter.Seq[key] {
 				return
 			}
 
-			// now the p-tag+kind+date
+			// now the p-1733934977tag+kind+date
 			if dbi == il.indexTag32 && tag[0] == "p" {
 				k := make([]byte, 8+2+4)
 				xhex.Decode(k[0:8], []byte(tag[1][0:8*2]))

eventstore/mmm/indexinglayer.go

@@ -61,7 +61,7 @@ func (il *IndexingLayer) Init() error {
 
 	env.SetMaxDBs(9)
 	env.SetMaxReaders(1000)
-	env.SetMapSize(1 << 38) // ~273GB
+	env.SetMapSize(MMAP_INFINITE_SIZE)
 
 	// create directory if it doesn't exist and open it
 	if err := os.MkdirAll(path, 0755); err != nil {

eventstore/mmm/mmmm.go

@@ -35,13 +35,15 @@ type MultiMmapManager struct {
 	mmapfEnd  uint64
 
 	writeMutex sync.Mutex
+	lockfile   *os.File
 
 	lmdbEnv     *lmdb.Env
 	stuff       lmdb.DBI
 	knownLayers lmdb.DBI
 	indexId     lmdb.DBI
 
-	freeRanges positions
+	freeRangesAll   positions  // sorted by position
+	freeRangesLarge []position // unsorted
 }
 
 func (b *MultiMmapManager) String() string {
@@ -49,33 +51,43 @@ func (b *MultiMmapManager) String() string {
 }
 
 const (
-	MMAP_INFINITE_SIZE = 1 << 40
+	MMAP_INFINITE_SIZE = 100_000_000_000
 	maxuint16          = 65535
 	maxuint32          = 4294967295
 )
 
-func (b *MultiMmapManager) Init() error {
+func (b *MultiMmapManager) Init() (err error) {
 	if b.Logger == nil {
 		nopLogger := zerolog.Nop()
 		b.Logger = &nopLogger
 	}
 
+	defer func() {
+		if err != nil {
+			b.releaseLock()
+		}
+	}()
+
 	// create directory if it doesn't exist
 	dbpath := filepath.Join(b.Dir, "mmmm")
 	if err := os.MkdirAll(dbpath, 0755); err != nil {
 		return fmt.Errorf("failed to create directory %s: %w", dbpath, err)
 	}
 
-	if !b.ReadOnly {
+	// lock database directory to prevent multiple instances
-		// create lockfile to prevent multiple instances
 	lockfilePath := filepath.Join(b.Dir, "mmmm.lock")
-		if _, err := os.OpenFile(lockfilePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644); err != nil {
+	lockfile, err := os.OpenFile(lockfilePath, os.O_CREATE|os.O_RDWR, 0644)
-			if os.IsExist(err) {
+	if err != nil {
+		return fmt.Errorf("failed to open lockfile %s: %w", lockfilePath, err)
+	}
+	if err := syscall.Flock(int(lockfile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		lockfile.Close()
+		if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
 			return fmt.Errorf("database at %s is already in use by another instance", b.Dir)
 		}
-			return fmt.Errorf("failed to create lockfile %s: %w", lockfilePath, err)
+		return fmt.Errorf("failed to lock database at %s: %w", b.Dir, err)
-		}
 	}
+	b.lockfile = lockfile
 
 	// open a huge mmapped file
 	b.mmapfPath = filepath.Join(b.Dir, "events")
@@ -83,7 +95,7 @@ func (b *MultiMmapManager) Init() error {
 	if err != nil {
 		return fmt.Errorf("failed to open events file at %s: %w", b.mmapfPath, err)
 	}
-	mmapf, err := syscall.Mmap(int(file.Fd()), 0, MMAP_INFINITE_SIZE,
+	mmapf, err := syscall.Mmap(int(file.Fd()), 0, int(MMAP_INFINITE_SIZE),
 		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
 	if err != nil {
 		return fmt.Errorf("failed to mmap events file at %s: %w", b.mmapfPath, err)
@@ -104,7 +116,7 @@ func (b *MultiMmapManager) Init() error {
 
 	env.SetMaxDBs(3)
 	env.SetMaxReaders(1000)
-	env.SetMapSize(1 << 38) // ~273GB
+	env.SetMapSize(MMAP_INFINITE_SIZE)
 
 	err = env.Open(dbpath, lmdb.NoTLS, 0644)
 	if err != nil {
@@ -139,18 +151,17 @@ func (b *MultiMmapManager) Init() error {
 
 		if !b.ReadOnly {
 			// scan index table to calculate free ranges from used positions
-			b.freeRanges, err = b.gatherFreeRanges(txn)
+			if err := b.gatherFreeRanges(txn); err != nil {
-			if err != nil {
 				return err
 			}
 
 			logOp := b.Logger.Debug()
-			for _, pos := range b.freeRanges {
+			count := 0
-				if pos.size > 20 {
+			for _, pos := range b.freeRangesLarge {
 				logOp = logOp.Uint32(fmt.Sprintf("%d", pos.start), pos.size)
+				count++
 			}
-			}
+			logOp.Int("count", count).Msg("calculated free ranges from index scan")
-			logOp.Msg("calculated free ranges from index scan")
 		}
 
 		return nil
@@ -330,6 +341,7 @@ func (b *MultiMmapManager) removeAllReferencesFromLayer(txn *lmdb.Txn, layerId u
 	return nil
 }
 
+//go:inline
 func (b *MultiMmapManager) loadEvent(pos position, eventReceiver *nostr.Event) error {
 	return betterbinary.Unmarshal(b.mmapf[pos.start:pos.start+uint64(pos.size)], eventReceiver)
 }
@@ -365,6 +377,19 @@ func (b *MultiMmapManager) getNextAvailableLayerId(txn *lmdb.Txn) (uint16, error
 	return id, nil
 }
 
+func (b *MultiMmapManager) releaseLock() {
+	if b.lockfile == nil {
+		return
+	}
+
+	_ = syscall.Flock(int(b.lockfile.Fd()), syscall.LOCK_UN)
+	_ = b.lockfile.Close()
+	b.lockfile = nil
+
+	lockfilePath := filepath.Join(b.Dir, "mmmm.lock")
+	_ = os.Remove(lockfilePath)
+}
+
 func (b *MultiMmapManager) Close() {
 	b.lmdbEnv.Close()
 	for _, il := range b.layers {
@@ -373,7 +398,5 @@ func (b *MultiMmapManager) Close() {
 
 	syscall.Munmap(b.mmapf)
 
-	// remove lockfile
+	b.releaseLock()
-	lockfilePath := filepath.Join(b.Dir, "mmmm.lock")
-	os.Remove(lockfilePath)
 }

eventstore/mmm/position.go

@@ -1,19 +1,27 @@
 package mmm
 
 import (
+	"cmp"
 	"encoding/binary"
 	"fmt"
+	"slices"
 	"strings"
 )
 
 type positions []position
 
+func (poss positions) find(start uint64) (idx int) {
+	idx, _ = slices.BinarySearchFunc(poss, start, func(item position, target uint64) int {
+		return cmp.Compare(item.start, target)
+	})
+	return idx
+}
+
 func (poss positions) String() string {
 	str := strings.Builder{}
 	str.Grow(10 + 20*len(poss))
 	str.WriteString("positions:[")
 	for _, pos := range poss {
-		str.WriteByte(' ')
 		str.WriteString(pos.String())
 	}
 	str.WriteString("]")
@@ -29,6 +37,11 @@ func (pos position) String() string {
 	return fmt.Sprintf("<%d|%d|%d>", pos.start, pos.size, pos.start+uint64(pos.size))
 }
 
+func (pos position) isLarge() bool {
+	return pos.size >= LARGE_FREERANGE
+}
+
+//go:inline
 func positionFromBytes(posb []byte) position {
 	return position{
 		size:  binary.BigEndian.Uint32(posb[0:4]),
@@ -36,6 +49,7 @@ func positionFromBytes(posb []byte) position {
 	}
 }
 
+//go:inline
 func writeBytesFromPosition(out []byte, pos position) {
 	binary.BigEndian.PutUint32(out[0:4], pos.size)
 	binary.BigEndian.PutUint64(out[4:12], pos.start)

eventstore/mmm/query.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"math"
 	"slices"
+	"sync"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore/codec/betterbinary"
@@ -14,6 +15,12 @@ import (
 	"github.com/PowerDNS/lmdb-go/lmdb"
 )
 
+var tempResultsPool = sync.Pool{
+	New: func() any {
+		return make([]nostr.Event, 0, 64)
+	},
+}
+
 // GetByID returns the event -- if found in this mmm -- and all the IndexingLayers it belongs to.
 func (b *MultiMmapManager) GetByID(id nostr.ID) (*nostr.Event, IndexingLayers) {
 	var event *nostr.Event
@@ -140,7 +147,8 @@ func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yi
 
 	numberOfIteratorsToPullOnEachRound := max(1, int(math.Ceil(float64(len(iterators))/float64(12))))
 	totalEventsEmitted := 0
-	tempResults := make([]nostr.Event, 0, batchSizePerQuery*2)
+	tempResults := tempResultsPool.Get().([]nostr.Event)
+	defer tempResultsPool.Put(tempResults[:0])
 
 	for len(iterators) > 0 {
 		// reset stuff
@@ -180,8 +188,8 @@ func (il *IndexingLayer) query(txn *lmdb.Txn, filter nostr.Filter, limit int, yi
 					// decode the entire thing
 					event := nostr.Event{}
 					if err := betterbinary.Unmarshal(bin, &event); err != nil {
-						log.Printf("lmdb: value read error (id %x) on query prefix %x sp %x dbi %v: %s\n",
+						log.Printf("mmm: value read error (id %s) on query prefix %x sp %x dbi %v: %s\n",
-							betterbinary.GetID(bin), iterators[i].query.prefix, iterators[i].query.startingPoint, iterators[i].query.dbi, err)
+							betterbinary.GetID(bin).Hex(), iterators[i].query.prefix, iterators[i].query.startingPoint, iterators[i].query.dbi, err)
 						continue
 					}
 

eventstore/mmm/query_planner.go

@@ -14,8 +14,6 @@ type query struct {
 	i             int
 	dbi           lmdb.DBI
 	prefix        []byte
-	keySize       int
-	timestampSize int
 	startingPoint []byte
 }
 
@@ -41,10 +39,10 @@ func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
 			}
 		}
 		for i, q := range queries {
-			sp := make([]byte, len(q.prefix))
+			sp := make([]byte, len(q.prefix)+4)
-			sp = sp[0:len(q.prefix)]
+			copy(sp[0:len(q.prefix)], q.prefix)
-			copy(sp, q.prefix)
+			binary.BigEndian.PutUint32(sp[len(q.prefix):], uint32(until))
-			queries[i].startingPoint = binary.BigEndian.AppendUint32(sp, uint32(until))
+			queries[i].startingPoint = sp
 		}
 	}()
 
@@ -65,10 +63,9 @@ func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
 		}
 
 		// only "p" tag has a goodness of 2, so
-		if goodness == 2 {
+		if goodness == 2 && filter.Kinds != nil {
 			// this means we got a "p" tag, so we will use the ptag-kind index
 			i := 0
-			if filter.Kinds != nil {
 			queries = make([]query, len(tagValues)*len(filter.Kinds))
 			for _, value := range tagValues {
 				if len(value) != 64 {
@@ -81,25 +78,14 @@ func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
 						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
 					}
 					binary.BigEndian.PutUint16(k[8:8+2], uint16(kind))
-						queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0 : 8+2], keySize: 8 + 2 + 4, timestampSize: 4}
+					queries[i] = query{
+						i:      i,
+						dbi:    il.indexPTagKind,
+						prefix: k[0 : 8+2],
+					}
 					i++
 				}
 			}
-			} else {
-				// even if there are no kinds, in that case we will just return any kind and not care
-				queries = make([]query, len(tagValues))
-				for i, value := range tagValues {
-					if len(value) != 64 {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
-					}
-
-					k := make([]byte, 8)
-					if err := xhex.Decode(k[0:8], []byte(value[0:8*2])); err != nil {
-						return nil, nil, nil, "", nil, 0, fmt.Errorf("invalid 'p' tag '%s'", value)
-					}
-					queries[i] = query{i: i, dbi: il.indexPTagKind, prefix: k[0:8], keySize: 8 + 2 + 4, timestampSize: 4}
-				}
-			}
 		} else {
 			// otherwise we will use a plain tag index
 			queries = make([]query, len(tagValues))
@@ -108,7 +94,11 @@ func (il *IndexingLayer) prepareQueries(filter nostr.Filter) (
 				dbi, k, offset := il.getTagIndexPrefix(tagKey, value)
 				// remove the last parts part to get just the prefix we want here
 				prefix := k[0:offset]
-				queries[i] = query{i: i, dbi: dbi, prefix: prefix, keySize: len(prefix) + 4, timestampSize: 4}
+				queries[i] = query{
+					i:      i,
+					dbi:    dbi,
+					prefix: prefix,
+				}
 			}
 
 			// add an extra kind filter if available (only do this on plain tag index, not on ptag-kind index)
@@ -143,9 +133,11 @@ pubkeyMatching:
 			// will use pubkey index
 			queries = make([]query, len(filter.Authors))
 			for i, pk := range filter.Authors {
-				prefix := make([]byte, 8)
+				queries[i] = query{
-				copy(prefix[0:8], pk[0:8])
+					i:      i,
-				queries[i] = query{i: i, dbi: il.indexPubkey, prefix: prefix[0:8], keySize: 8 + 4, timestampSize: 4}
+					dbi:    il.indexPubkey,
+					prefix: pk[0:8],
+				}
 			}
 		} else {
 			// will use pubkeyKind index
@@ -156,7 +148,11 @@ pubkeyMatching:
 					prefix := make([]byte, 8+2)
 					copy(prefix[0:8], pk[0:8])
 					binary.BigEndian.PutUint16(prefix[8:8+2], uint16(kind))
-					queries[i] = query{i: i, dbi: il.indexPubkeyKind, prefix: prefix[0 : 8+2], keySize: 10 + 4, timestampSize: 4}
+					queries[i] = query{
+						i:      i,
+						dbi:    il.indexPubkeyKind,
+						prefix: prefix[0 : 8+2],
+					}
 					i++
 				}
 			}
@@ -173,7 +169,11 @@ pubkeyMatching:
 		for i, kind := range filter.Kinds {
 			prefix := make([]byte, 2)
 			binary.BigEndian.PutUint16(prefix[0:2], uint16(kind))
-			queries[i] = query{i: i, dbi: il.indexKind, prefix: prefix[0:2], keySize: 2 + 4, timestampSize: 4}
+			queries[i] = query{
+				i:      i,
+				dbi:    il.indexKind,
+				prefix: prefix[0:2],
+			}
 		}
 
 		// potentially with an extra useless tag filtering
@@ -184,6 +184,10 @@ pubkeyMatching:
 	// if we got here our query will have nothing to filter with
 	queries = make([]query, 1)
 	prefix := make([]byte, 0)
-	queries[0] = query{i: 0, dbi: il.indexCreatedAt, prefix: prefix, keySize: 0 + 4, timestampSize: 4}
+	queries[0] = query{
+		i:      0,
+		dbi:    il.indexCreatedAt,
+		prefix: prefix,
+	}
 	return queries, nil, nil, "", nil, since, nil
 }

eventstore/mmm/replace.go

@@ -9,9 +9,9 @@ import (
 	"github.com/PowerDNS/lmdb-go/lmdb"
 )
 
-func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
+func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) (deleted []nostr.Event, err error) {
 	if il.mmmm.ReadOnly {
-		return ReadOnly
+		return nil, ReadOnly
 	}
 
 	il.mmmm.writeMutex.Lock()
@@ -29,7 +29,7 @@ func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
 	// prepare transactions
 	mmmtxn, err := il.mmmm.lmdbEnv.BeginTxn(nil, 0)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer func() {
 		// defer abort but only if we haven't committed (we'll set it to nil after committing)
@@ -41,7 +41,7 @@ func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
 
 	iltxn, err := il.lmdbEnv.BeginTxn(nil, 0)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer func() {
 		// defer abort but only if we haven't committed (we'll set it to nil after committing)
@@ -54,33 +54,35 @@ func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
 	// check if we already have this id
 	_, existsErr := mmmtxn.Get(il.mmmm.indexId, evt.ID[0:8])
 	if existsErr == nil {
-		return nil
+		return nil, nil
 	}
 	if !lmdb.IsNotFound(existsErr) {
-		return fmt.Errorf("error checking existence: %w", existsErr)
+		return nil, fmt.Errorf("error checking existence: %w", existsErr)
 	}
 
 	// now we fetch the past events, whatever they are, delete them and then save the new
+	var qerr error
 	var results iter.Seq[nostr.Event] = func(yield func(nostr.Event) bool) {
-		err = il.query(iltxn, filter, 10 /* in theory limit could be just 1 and this should work */, yield)
+		qerr = il.query(iltxn, filter, 10 /* in theory limit could be just 1 and this should work */, yield)
 	}
-	if err != nil {
+	if qerr != nil {
-		return fmt.Errorf("failed to query past events with %s: %w", filter, err)
+		return nil, fmt.Errorf("failed to query past events with %s: %w", filter, qerr)
 	}
 
 	var acquiredFreeRangeFromDelete *position
 	shouldStore := true
 	for previous := range results {
 		if nostr.IsOlder(previous, evt) {
-			if pos, shouldPurge, err := il.delete(mmmtxn, iltxn, previous.ID); err != nil {
+			if pos, shouldPurge, derr := il.delete(mmmtxn, iltxn, previous.ID); derr != nil {
-				return fmt.Errorf("failed to delete event %s for replacing: %w", previous.ID, err)
+				return nil, fmt.Errorf("failed to delete event %s for replacing: %w", previous.ID, derr)
 			} else if shouldPurge {
 				// purge
 				if err := mmmtxn.Del(il.mmmm.indexId, previous.ID[0:8], nil); err != nil {
-					return err
+					return nil, err
 				}
 				acquiredFreeRangeFromDelete = &pos
 			}
+			deleted = append(deleted, previous)
 		} else {
 			// there is a newer event already stored, so we won't store this
 			shouldStore = false
@@ -90,17 +92,17 @@ func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
 	if shouldStore {
 		_, err := il.mmmm.storeOn(mmmtxn, iltxn, il, evt)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	// commit in this order to minimize problematic inconsistencies
 	if err := mmmtxn.Commit(); err != nil {
-		return fmt.Errorf("can't commit mmmtxn: %w", err)
+		return nil, fmt.Errorf("can't commit mmmtxn: %w", err)
 	}
 	mmmtxn = nil
 	if err := iltxn.Commit(); err != nil {
-		return fmt.Errorf("can't commit iltxn: %w", err)
+		return nil, fmt.Errorf("can't commit iltxn: %w", err)
 	}
 	iltxn = nil
 
@@ -110,5 +112,5 @@ func (il *IndexingLayer) ReplaceEvent(evt nostr.Event) error {
 		il.mmmm.mergeNewFreeRange(*acquiredFreeRangeFromDelete)
 	}
 
-	return nil
+	return deleted, nil
 }

eventstore/mmm/save.go

@@ -104,25 +104,42 @@ func (b *MultiMmapManager) storeOn(
 			return false, fmt.Errorf("event too large to store, max %d, got %d", 1<<16, pos.size)
 		}
 
-		// find a suitable place for this to be stored in
+		// find a suitable place for this to be stored in (search only large free ranges)
 		appendToMmap := true
-		for f, fr := range b.freeRanges {
+		for f, fr := range b.freeRangesLarge {
 			if fr.size >= pos.size {
-				// found the smallest possible place that can fit this event
+				// found a place that can fit this event
 				appendToMmap = false
 				pos.start = fr.start
 
 				// modify the free ranges we're keeping track of
 				// (in case of conflict we lose this free range but it's ok, it will be recovered on the next startup)
 				if pos.size == fr.size {
-					// if we've used it entirely just delete it
+					// if we've used it entirely just delete it (swap-delete since it's unsorted)
-					b.freeRanges = slices.Delete(b.freeRanges, f, f+1)
+					b.freeRangesLarge[f] = b.freeRangesLarge[len(b.freeRangesLarge)-1]
+					b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
+
+					// also delete it from b.freeRangesAll
+					idx := b.freeRangesAll.find(fr.start)
+					b.freeRangesAll = slices.Delete(b.freeRangesAll, idx, idx+1)
 				} else {
 					// otherwise modify it in place
-					b.freeRanges[f] = position{
+					newFreeRange := position{
 						start: fr.start + uint64(pos.size),
 						size:  fr.size - pos.size,
 					}
+					// only keep it in freeRangesLarge if it's still large enough
+					if newFreeRange.size >= LARGE_FREERANGE {
+						b.freeRangesLarge[f] = newFreeRange
+					} else {
+						// remove it from freeRangesLarge if it's no longer large enough
+						b.freeRangesLarge[f] = b.freeRangesLarge[len(b.freeRangesLarge)-1]
+						b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
+					}
+
+					// also modify it in b.freeRangesAll
+					idx := b.freeRangesAll.find(fr.start)
+					b.freeRangesAll[idx] = newFreeRange
 				}
 
 				break
@@ -140,7 +157,7 @@ func (b *MultiMmapManager) storeOn(
 		}
 
 		// write to the mmap
-		if err := betterbinary.Marshal(evt, b.mmapf[pos.start:]); err != nil {
+		if err := betterbinary.Marshal(evt, b.mmapf[pos.start:pos.start+uint64(pos.size)]); err != nil {
 			return false, fmt.Errorf("error marshaling to %d: %w", pos.start, err)
 		}
 

eventstore/mmm/stats.go

@@ -42,6 +42,8 @@ func (il *IndexingLayer) ComputeStats(opts StatsOptions) (EventStats, error) {
 	}
 
 	err := il.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		txn.RawRead = true
+
 		cursor, err := txn.OpenCursor(il.indexPubkeyKind)
 		if err != nil {
 			return err

eventstore/mmm/testdata/fuzz/FuzzDefragment/2ce076b09281c580

@@ -0,0 +1,2 @@
+go test fuzz v1
+int(-360)

eventstore/mmm/testdata/fuzz/FuzzDefragment/d5845b29b7a801f7

@@ -0,0 +1,2 @@
+go test fuzz v1
+int(-17)

eventstore/mmm/testdata/fuzz/FuzzTest/ffd0b6bc2949a534

@@ -0,0 +1,5 @@
+go test fuzz v1
+int(46)
+uint(84)
+uint(55)
+uint(5)

eventstore/nullstore/lib.go

@@ -29,8 +29,8 @@ func (b NullStore) SaveEvent(evt nostr.Event) error {
 	return nil
 }
 
-func (b NullStore) ReplaceEvent(evt nostr.Event) error {
+func (b NullStore) ReplaceEvent(evt nostr.Event) ([]nostr.Event, error) {
-	return nil
+	return nil, nil
 }
 
 func (b NullStore) CountEvents(filter nostr.Filter) (uint32, error) {

eventstore/slicestore/lib.go

@@ -122,7 +122,7 @@ func (b *SliceStore) delete(id nostr.ID) error {
 	return nil
 }
 
-func (b *SliceStore) ReplaceEvent(evt nostr.Event) error {
+func (b *SliceStore) ReplaceEvent(evt nostr.Event) (deleted []nostr.Event, err error) {
 	b.Lock()
 	defer b.Unlock()
 
@@ -135,8 +135,9 @@ func (b *SliceStore) ReplaceEvent(evt nostr.Event) error {
 	for previous := range b.QueryEvents(filter, 1) {
 		if nostr.IsOlder(previous, evt) {
 			if err := b.delete(previous.ID); err != nil {
-				return fmt.Errorf("failed to delete event for replacing: %w", err)
+				return nil, fmt.Errorf("failed to delete event for replacing: %w", err)
 			}
+			deleted = append(deleted, previous)
 		} else {
 			shouldStore = false
 		}
@@ -144,11 +145,11 @@ func (b *SliceStore) ReplaceEvent(evt nostr.Event) error {
 
 	if shouldStore {
 		if err := b.save(evt); err != nil && err != eventstore.ErrDupEvent {
-			return fmt.Errorf("failed to save: %w", err)
+			return nil, fmt.Errorf("failed to save: %w", err)
 		}
 	}
 
-	return nil
+	return deleted, nil
 }
 
 func eventTimestampComparator(e nostr.Event, t nostr.Timestamp) int {

eventstore/store.go

@@ -26,7 +26,7 @@ type Store interface {
 
 	// ReplaceEvent atomically replaces a replaceable or addressable event.
 	// Conceptually it is like a Query->Delete->Save, but streamlined.
-	ReplaceEvent(nostr.Event) error
+	ReplaceEvent(nostr.Event) (deleted []nostr.Event, err error)
 
 	// CountEvents counts all events that match a given filter
 	CountEvents(nostr.Filter) (uint32, error)

eventstore/test/basic_test.go

@@ -128,6 +128,24 @@ func basicTest(t *testing.T, db eventstore.Store) {
 			require.Len(t, results, 1)
 			require.Equal(t, events[5].ID, results[0].ID, "author + kind query error")
 		}
+
+		// test 5: until
+		{
+			results := slices.Collect(db.QueryEvents(nostr.Filter{Until: 102}, 1000))
+			require.NoError(t, err)
+			require.Len(t, results, 3)
+
+			resultsWithTag := slices.Collect(db.QueryEvents(nostr.Filter{
+				Until: 102,
+				Tags: nostr.TagMap{
+					"e": []string{
+						"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+					},
+				},
+			}, 1000))
+			require.NoError(t, err)
+			require.Len(t, resultsWithTag, 1)
+		}
 	}
 
 	// from another-basic-test.patch
@@ -223,7 +241,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		}
 		originalProfile.Sign(sk3)
 
-		err = db.ReplaceEvent(originalProfile)
+		_, err = db.ReplaceEvent(originalProfile)
 		require.NoError(t, err)
 
 		// verify
@@ -244,7 +262,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		newProfile.Sign(sk3)
 
 		// replace with newer event
-		err = db.ReplaceEvent(newProfile)
+		_, err = db.ReplaceEvent(newProfile)
 		require.NoError(t, err)
 
 		// verify only the newer event exists
@@ -264,7 +282,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		}
 		olderProfile.Sign(sk3)
 
-		err = db.ReplaceEvent(olderProfile)
+		_, err = db.ReplaceEvent(olderProfile)
 		require.NoError(t, err)
 
 		// verify the newer event is still there
@@ -284,7 +302,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		}
 		articleV1.Sign(sk3)
 
-		err = db.ReplaceEvent(articleV1)
+		_, err = db.ReplaceEvent(articleV1)
 		require.NoError(t, err)
 
 		// verify article was saved
@@ -305,7 +323,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		}
 		articleV2.Sign(sk3)
 
-		err = db.ReplaceEvent(articleV2)
+		_, err = db.ReplaceEvent(articleV2)
 		require.NoError(t, err)
 
 		// verify only the newer version exists
@@ -327,7 +345,7 @@ func basicTest(t *testing.T, db eventstore.Store) {
 		}
 		differentArticle.Sign(sk3)
 
-		err = db.ReplaceEvent(differentArticle)
+		_, err = db.ReplaceEvent(differentArticle)
 		require.NoError(t, err)
 
 		// verify both articles exist (different d tags)

eventstore/test/db_test.go

@@ -33,6 +33,7 @@ var tests = []struct {
 	{"manyauthors", manyAuthorsTest},
 	{"unbalanced", unbalancedTest},
 	{"count", countTest},
+	{"pfilter-until", pTagUntilMismatchTest},
 }
 
 func TestSliceStore(t *testing.T) {

eventstore/test/pfilter_until_test.go

@@ -0,0 +1,92 @@
+package test
+
+import (
+	"encoding/hex"
+	"slices"
+	"testing"
+
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore"
+	"github.com/stretchr/testify/require"
+)
+
+func pTagUntilMismatchTest(t *testing.T, db eventstore.Store) {
+	err := db.Init()
+	require.NoError(t, err)
+
+	targetP := "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"
+	author := nostr.MustPubKeyFromHex("7fa56f5d6962ab1e3cd424e758c3002b8665f7b0d8dcee9fe9e288d7751ac194")
+
+	events := []nostr.Event{
+		{
+			Kind:      9802,
+			ID:        nostr.MustIDFromHex("2c997233fa580b1a831f989d8fa320c409f8412d9c75b819c9a29df102d7f901"),
+			PubKey:    author,
+			CreatedAt: 1773835689,
+			Tags: nostr.Tags{
+				{"e", "bcaa6599e69cff48ed6ab4b0b315d4f33a869a4ba8fa808287700faebe17195f", "wss://nos.lol/", "source"},
+				{"p", targetP, "", "author"},
+			},
+			Content: "With so few people donating in the zap the devs button, the incentives are quite low to produce cool new things",
+			Sig:     sigFromHex(t, "b49206476c4d2a5f44590331541c83910fd826c0f4cdab99ceffd5bcf3aca94935e3db9d7820e7db3a0f1165c43a28dd3173a81fd08bf8348629ea4efde02537"),
+		},
+		{
+			Kind:      9802,
+			ID:        nostr.MustIDFromHex("31c1eddb3a5201ef1bbce91b9fb3b7d8fe3e3eb25a66bedadcbc93c84d072c7d"),
+			PubKey:    author,
+			CreatedAt: 1773154080,
+			Tags: nostr.Tags{
+				{"p", "5ea4648045bb1ff222655ddd36e6dceddc43590c26090c486bef38ef450da5bd", "", "mention"},
+				{"p", "c8fb0d3aa788b9ace4f6cb92dd97d3f292db25b5c9f92462ef6c64926129fbaf", "", "mention"},
+				{"p", "2f29aa33c2a3b45c2ef32212879248b2f4a49a002bd0de0fa16c94e138ac6f13", "", "mention"},
+				{"p", targetP, "", "mention"},
+				{"comment", "normie"},
+				{"e", "5911eeba39a6886fe8abea82bb50612d27d1273d63904c9b64cde070c7088d48", "wss://relay.primal.net/", "source"},
+				{"p", "3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24", "", "author"},
+			},
+			Content: "grimoire is cool, but it's too nerdy for me",
+			Sig:     sigFromHex(t, "0ee01515d54293d52fa1247a395e64c8499df96eee80c30204cb7c8fc5b5977023e6ac00cc240f137ce7e594818545340bf74bce6d3de86539f1a5d26fe33f24"),
+		},
+		{
+			Kind:      9802,
+			ID:        nostr.MustIDFromHex("baeb90e2075c9d8a9b41286dbf1c52e5ef8ad6c030118839ce24d065e72df9b7"),
+			PubKey:    author,
+			CreatedAt: 1773154058,
+			Tags: nostr.Tags{
+				{"p", "c8fb0d3aa788b9ace4f6cb92dd97d3f292db25b5c9f92462ef6c64926129fbaf", "", "mention"},
+				{"p", "2f29aa33c2a3b45c2ef32212879248b2f4a49a002bd0de0fa16c94e138ac6f13", "", "mention"},
+				{"p", targetP, "", "mention"},
+				{"p", "3f770d65d3a764a9c5cb503ae123e62ec7598ad035d836e2a810f3877a745b24", "", "mention"},
+				{"comment", "no lies detected"},
+				{"e", "81171c564cedbc5f07e5b7a9d06842d1f43a81cd79c8755921190382de55c514", "wss://nos.lol/", "source"},
+				{"p", "5ea4648045bb1ff222655ddd36e6dceddc43590c26090c486bef38ef450da5bd", "", "author"},
+			},
+			Content: "i never used grimoire once in my life, but that is not the point, it is still the best client",
+			Sig:     sigFromHex(t, "3e4b855c4e3a4d2b3a078d593e728f1bbdb07af91ba5831b7866e2d16df90ce58a5f9f1db5733603911b20b334bc7fc5ae1482c7870b9f7acde4a8ccc080a79d"),
+		},
+	}
+
+	for _, evt := range events {
+		err = db.SaveEvent(evt)
+		require.NoError(t, err)
+	}
+
+	results := slices.Collect(db.QueryEvents(nostr.Filter{
+		Until: 1733934976,
+		Limit: 3,
+		Tags:  nostr.TagMap{"p": []string{targetP}},
+	}, 1000))
+	require.Len(t, results, 0)
+}
+
+func sigFromHex(t *testing.T, sigStr string) [64]byte {
+	t.Helper()
+
+	raw, err := hex.DecodeString(sigStr)
+	require.NoError(t, err)
+	require.Len(t, raw, 64)
+
+	var sig [64]byte
+	copy(sig[:], raw)
+	return sig
+}

eventstore/wrappers/dynamic_publisher.go

@@ -0,0 +1,41 @@
+package wrappers
+
+import (
+	"context"
+	"fmt"
+	"iter"
+
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore"
+)
+
+var _ nostr.Publisher = DynamicPublisher{}
+
+type DynamicPublisher struct {
+	GetStore func() eventstore.Store
+	MaxLimit int
+}
+
+func (w DynamicPublisher) QueryEvents(filter nostr.Filter) iter.Seq[nostr.Event] {
+	return w.GetStore().QueryEvents(filter, w.MaxLimit)
+}
+
+func (w DynamicPublisher) Publish(ctx context.Context, evt nostr.Event) error {
+	if evt.Kind.IsEphemeral() {
+		return nil
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if evt.Kind.IsRegular() {
+		if err := w.GetStore().SaveEvent(evt); err != nil && err != eventstore.ErrDupEvent {
+			return fmt.Errorf("failed to save: %w", err)
+		} else {
+			return err
+		}
+	}
+
+	_, err := w.GetStore().ReplaceEvent(evt)
+	return err
+}

eventstore/wrappers/publisher.go

@@ -39,5 +39,6 @@ func (w StorePublisher) Publish(ctx context.Context, evt nostr.Event) error {
 	}
 
 	// others are replaced
-	return w.Store.ReplaceEvent(evt)
+	_, err := w.Store.ReplaceEvent(evt)
+	return err
 }

filter.go

@@ -44,6 +44,7 @@ func (ef Filter) Matches(event Event) bool {
 	return true
 }
 
+//go:inline
 func (ef Filter) MatchesIgnoringTimestampConstraints(event Event) bool {
 	if ef.IDs != nil && !slices.Contains(ef.IDs, event.ID) {
 		return false

go.mod

@@ -40,8 +40,10 @@ require (
 )
 
 require (
+	fiatjaf.com/lib v0.3.7
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/go-git/go-git/v5 v5.16.3
+	github.com/pemistahl/lingua-go v1.4.0
 	github.com/sivukhin/godjot v1.0.6
 	github.com/templexxx/cpu v0.0.1
 	github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b
@@ -63,6 +65,7 @@ require (
 	github.com/blevesearch/scorch_segment_api/v2 v2.2.16 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
+	github.com/blevesearch/stempel v0.2.0 // indirect
 	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
 	github.com/blevesearch/vellum v1.0.11 // indirect
 	github.com/blevesearch/zapx/v11 v11.3.10 // indirect
@@ -93,6 +96,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+fiatjaf.com/lib v0.3.7 h1:mXZOn7NrUcjSdy4oNvwQyAmes7Ueb+Zr5hjqMIe2dxI=
+fiatjaf.com/lib v0.3.7/go.mod h1:UlHaZvPHj25PtKLh9GjZkUHRmQ2xZ8Jkoa4VRaLeeQ8=
 github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc=
 github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw=
 github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec h1:1Qb69mGp/UtRPn422BH4/Y4Q3SLUrD9KHuDkm8iodFc=
@@ -40,6 +42,8 @@ github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+j
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
+github.com/blevesearch/stempel v0.2.0 h1:CYzVPaScODMvgE9o+kf6D4RJ/VRomyi9uHF+PtB+Afc=
+github.com/blevesearch/stempel v0.2.0/go.mod h1:wjeTHqQv+nQdbPuJ/YcvOjTInA2EIc6Ks1FoSUzSLvc=
 github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
 github.com/blevesearch/vellum v1.0.11 h1:SJI97toEFTtA9WsDZxkyGTaBWFdWl1n2LEDCXLCq/AU=
@@ -190,6 +194,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=
+github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -207,6 +213,8 @@ github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 h1:D0vL7YNisV2yqE55+q0lFuGse6U8lxlg7fYTctlT5Gc=
 github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sivukhin/godjot v1.0.6 h1:yoRD+hlcDbSxP9Gd/KRVlEFXgtGyZyt0CHwhY6Gk3EQ=
 github.com/sivukhin/godjot v1.0.6/go.mod h1:wA6KdR4Z+XpwdwyViPDLWYYxT72pKjNc6XGA9I025gM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

helpers.go

@@ -92,46 +92,6 @@ func similarPublicKey(as, bs []PubKey) bool {
 	return true
 }
 
-// Escaping strings for JSON encoding according to RFC8259.
-// Also encloses result in quotation marks "".
-func escapeString(dst []byte, s string) []byte {
-	dst = append(dst, '"')
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		switch {
-		case c == '"':
-			// quotation mark
-			dst = append(dst, []byte{'\\', '"'}...)
-		case c == '\\':
-			// reverse solidus
-			dst = append(dst, []byte{'\\', '\\'}...)
-		case c >= 0x20:
-			// default, rest below are control chars
-			dst = append(dst, c)
-		case c == 0x08:
-			dst = append(dst, []byte{'\\', 'b'}...)
-		case c < 0x09:
-			dst = append(dst, []byte{'\\', 'u', '0', '0', '0', '0' + c}...)
-		case c == 0x09:
-			dst = append(dst, []byte{'\\', 't'}...)
-		case c == 0x0a:
-			dst = append(dst, []byte{'\\', 'n'}...)
-		case c == 0x0c:
-			dst = append(dst, []byte{'\\', 'f'}...)
-		case c == 0x0d:
-			dst = append(dst, []byte{'\\', 'r'}...)
-		case c < 0x10:
-			dst = append(dst, []byte{'\\', 'u', '0', '0', '0', 0x57 + c}...)
-		case c < 0x1a:
-			dst = append(dst, []byte{'\\', 'u', '0', '0', '1', 0x20 + c}...)
-		case c < 0x20:
-			dst = append(dst, []byte{'\\', 'u', '0', '0', '1', 0x47 + c}...)
-		}
-	}
-	dst = append(dst, '"')
-	return dst
-}
-
 func subIdToSerial(subId string) int64 {
 	n := strings.Index(subId, ":")
 	if n < 0 || n > len(subId) {

khatru/blossom/blob.go

@@ -13,6 +13,9 @@ type BlobIndex interface {
 	List(ctx context.Context, pubkey nostr.PubKey) iter.Seq[blossom.BlobDescriptor]
 	Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error)
 	Delete(ctx context.Context, sha256 string, pubkey nostr.PubKey) error
+
+	ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor]
+	OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey
 }
 
 var (

khatru/blossom/eventstorewrapper.go

@@ -59,6 +59,30 @@ func (es EventStoreBlobIndexWrapper) List(ctx context.Context, pubkey nostr.PubK
 	}
 }
 
+func (es EventStoreBlobIndexWrapper) ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor] {
+	return func(yield func(nostr.PubKey, blossom.BlobDescriptor) bool) {
+		for evt := range es.Store.QueryEvents(nostr.Filter{
+			Kinds: []nostr.Kind{24242},
+		}, 1000) {
+			bd := es.parseEvent(evt)
+			if !yield(evt.PubKey, bd) {
+				return
+			}
+		}
+	}
+}
+
+func (es EventStoreBlobIndexWrapper) OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey {
+	var owners []nostr.PubKey
+	for evt := range es.Store.QueryEvents(nostr.Filter{
+		Tags:  nostr.TagMap{"x": []string{sha256}},
+		Kinds: []nostr.Kind{24242},
+	}, 1000) {
+		owners = append(owners, evt.PubKey)
+	}
+	return owners
+}
+
 func (es EventStoreBlobIndexWrapper) Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error) {
 	next, stop := iter.Pull(
 		es.Store.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"x": []string{sha256}}, Kinds: []nostr.Kind{24242}, Limit: 1}, 1),

khatru/blossom/handlers.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -129,7 +128,7 @@ func (bs BlossomServer) handleUpload(w http.ResponseWriter, r *http.Request) {
 
 	hash := sha256.Sum256(b)
 	hhash := nostr.HexEncodeToString(hash[:])
-	mimeType := mime.TypeByExtension(ext)
+	mimeType := blossom.GetMIMEType(ext)
 	if mimeType == "" {
 		mimeType = "application/octet-stream"
 	}

khatru/blossom/memoryindex.go

@@ -59,6 +59,25 @@ func (x MemoryBlobIndex) List(ctx context.Context, pubkey nostr.PubKey) iter.Seq
 	}
 }
 
+func (x MemoryBlobIndex) ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor] {
+	return func(yield func(nostr.PubKey, blossom.BlobDescriptor) bool) {
+		for _, v := range x.m.Range {
+			for _, owner := range v.owners {
+				if !yield(owner, v.blob) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (x MemoryBlobIndex) OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey {
+	if val, ok := x.m.Load(sha256); ok {
+		return val.owners
+	}
+	return nil
+}
+
 func (x MemoryBlobIndex) Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error) {
 	if val, ok := x.m.Load(sha256); ok {
 		return &val.blob, nil

khatru/deleting.go

@@ -65,7 +65,7 @@ func (rl *Relay) handleDeleteRequest(ctx context.Context, evt nostr.Event) error
 			errg, ctx := errgroup.WithContext(ctx)
 			for target := range rl.QueryStored(ctx, f) {
 				// got the event, now check if the user can delete it
-				if target.PubKey == evt.PubKey {
+				if rl.AllowDeleting == nil && target.PubKey == evt.PubKey || rl.AllowDeleting != nil && rl.AllowDeleting(ctx, target, evt) {
 					// delete it
 					errg.Go(func() error {
 						if err := rl.DeleteEvent(ctx, target.ID); err != nil {
@@ -78,6 +78,9 @@ func (rl *Relay) handleDeleteRequest(ctx context.Context, evt nostr.Event) error
 						}
 
 						haveDeletedSomething = true
+						if rl.OnEventDeleted != nil {
+							rl.OnEventDeleted(ctx, target)
+						}
 						return nil
 					})
 				} else {

khatru/dispatcher_test.go

@@ -0,0 +1,265 @@
+package khatru
+
+import (
+	"slices"
+	"strconv"
+	"testing"
+
+	"fiatjaf.com/nostr"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDispatcherCandidates(t *testing.T) {
+	d := newDispatcher()
+
+	d.addSubscription(subscription{
+		id: "...",
+		filter: nostr.Filter{
+			Kinds: []nostr.Kind{9},
+			Tags:  nostr.TagMap{"h": []string{"aaa"}},
+		},
+	})
+	d.addSubscription(subscription{
+		id: "...",
+		filter: nostr.Filter{
+			Kinds: []nostr.Kind{11},
+			Tags:  nostr.TagMap{"h": []string{"aaa"}},
+		},
+	})
+	d.addSubscription(subscription{
+		id: "...",
+		filter: nostr.Filter{
+			Kinds: []nostr.Kind{9, 11, 1111},
+			Tags:  nostr.TagMap{"h": []string{"aaa"}},
+		},
+	})
+	d.addSubscription(subscription{
+		id: "...",
+		filter: nostr.Filter{
+			Kinds: []nostr.Kind{9, 11, 1111},
+			Tags:  nostr.TagMap{"h": []string{"bbb"}},
+		},
+	})
+	d.addSubscription(subscription{
+		id: "...",
+		filter: nostr.Filter{
+			Kinds: []nostr.Kind{9, 11, 1111},
+			Authors: []nostr.PubKey{
+				nostr.MustPubKeyFromHex("87f5650744bed197fcb170ae05fd8d1948a24b2aac34cedf7bdb1c47d6d93273"),
+			},
+		},
+	})
+
+	matched := 0
+	for range d.candidates(nostr.Event{
+		PubKey:    nostr.MustPubKeyFromHex("87f5650744bed197fcb170ae05fd8d1948a24b2aac34cedf7bdb1c47d6d93273"),
+		ID:        nostr.MustIDFromHex("87f5650744bed197fcb170ae05fd8d1948a24b2aac34cedf7bdb1c47d6d93273"),
+		Kind:      9,
+		CreatedAt: nostr.Now(),
+		Content:   "hello",
+		Tags: nostr.Tags{
+			{"h", "aaa"},
+		},
+	}) {
+		matched++
+	}
+
+	require.Equal(t, 3, matched)
+}
+
+func FuzzDispatcherCandidates(f *testing.F) {
+	f.Add(1, 1, uint8(8), uint8(16))
+	f.Add(2, 3, uint8(32), uint8(32))
+
+	f.Fuzz(func(t *testing.T, seed int, advance int, ops uint8, checks uint8) {
+		d := newDispatcher()
+		state := fuzzState{value: seed, advance: advance}
+
+		active := make(map[int]subscription)
+		activeSSIDs := make([]int, 0, int(ops))
+		nextSubID := 0
+
+		steps := int(ops) + 1
+		for range steps {
+			if len(activeSSIDs) == 0 || state.next(10) != 0 {
+				nextSubID++
+				sub := subscription{
+					id:     strconv.Itoa(nextSubID),
+					filter: fuzzDispatcherFilter(&state),
+				}
+
+				ssid := d.addSubscription(sub)
+
+				active[ssid] = sub
+				activeSSIDs = append(activeSSIDs, ssid)
+			} else {
+				idx := state.next(len(activeSSIDs))
+				ssid := activeSSIDs[idx]
+
+				d.removeSubscription(ssid)
+
+				delete(active, ssid)
+				activeSSIDs = append(activeSSIDs[:idx], activeSSIDs[idx+1:]...)
+			}
+
+			for range int(checks%7) + 1 {
+				event := fuzzDispatcherEvent(&state)
+
+				expected := expectedDispatcherCandidates(active, event)
+				actual := collectedDispatcherCandidates(&d, event)
+
+				require.Equalf(t, expected, actual, "seed=%d advance=%d event=%s active=%v", seed, advance, event.String(), active)
+			}
+		}
+
+		for _, ssid := range activeSSIDs {
+			d.removeSubscription(ssid)
+			delete(active, ssid)
+		}
+
+		require.Empty(t, collectedDispatcherCandidates(&d, fuzzDispatcherEvent(&state)))
+	})
+}
+
+type fuzzState struct {
+	value   int
+	advance int
+}
+
+func (state *fuzzState) next(n int) int {
+	if n <= 0 {
+		return 0
+	}
+	value := state.value % n
+	if value < 0 {
+		value += n
+	}
+	state.value += state.advance
+	return value
+}
+
+func fuzzDispatcherFilter(seed *fuzzState) nostr.Filter {
+	filter := nostr.Filter{
+		Authors: fuzzDispatcherAuthors(seed),
+		Kinds:   fuzzDispatcherKinds(seed),
+		Tags:    fuzzDispatcherTagMap(seed),
+	}
+
+	if seed.next(3) == 0 {
+		since := nostr.Timestamp(seed.next(6))
+		until := since + nostr.Timestamp(seed.next(6))
+		filter.Since = since
+		filter.Until = until
+	} else if seed.next(4) == 0 {
+		filter.Since = nostr.Timestamp(seed.next(8))
+	} else if seed.next(4) == 0 {
+		filter.Until = nostr.Timestamp(seed.next(8))
+	}
+
+	return filter
+}
+
+func fuzzDispatcherAuthors(seed *fuzzState) []nostr.PubKey {
+	switch seed.next(4) {
+	case 0:
+		return nil
+	case 1:
+		return []nostr.PubKey{}
+	}
+
+	count := seed.next(3) + 1
+	authors := make([]nostr.PubKey, 0, count)
+	for range count {
+		pk := nostr.PubKey{byte(seed.next(4) + 1)}
+		if !slices.Contains(authors, pk) {
+			authors = append(authors, pk)
+		}
+	}
+	return authors
+}
+
+func fuzzDispatcherKinds(seed *fuzzState) []nostr.Kind {
+	switch seed.next(4) {
+	case 0:
+		return nil
+	case 1:
+		return []nostr.Kind{}
+	}
+
+	count := seed.next(3) + 1
+	kinds := make([]nostr.Kind, 0, count)
+	for range count {
+		kind := nostr.Kind(seed.next(5) + 1)
+		if !slices.Contains(kinds, kind) {
+			kinds = append(kinds, kind)
+		}
+	}
+	return kinds
+}
+
+func fuzzDispatcherTagMap(seed *fuzzState) nostr.TagMap {
+	if seed.next(3) == 0 {
+		return nil
+	}
+
+	keys := []string{"e", "p", "t"}
+	values := []string{"a", "b", "c", "d"}
+
+	count := seed.next(3)
+	if count == 0 {
+		return nil
+	}
+
+	tags := make(nostr.TagMap, count)
+	start := seed.next(len(keys))
+	for i := range count {
+		idx := (start + i) % len(keys)
+		valueCount := seed.next(3) + 1
+		entries := make([]string, 0, valueCount)
+		for range valueCount {
+			value := values[seed.next(len(values))]
+			if !slices.Contains(entries, value) {
+				entries = append(entries, value)
+			}
+		}
+		tags[keys[idx]] = entries
+	}
+
+	return tags
+}
+
+func fuzzDispatcherEvent(seed *fuzzState) nostr.Event {
+	tags := make(nostr.Tags, 0, seed.next(4))
+	keys := []string{"e", "p", "t"}
+	values := []string{"a", "b", "c", "d"}
+	for range cap(tags) {
+		tags = append(tags, nostr.Tag{keys[seed.next(len(keys))], values[seed.next(len(values))]})
+	}
+
+	return nostr.Event{
+		PubKey:    nostr.PubKey{byte(seed.next(4) + 1)},
+		Kind:      nostr.Kind(seed.next(5) + 1),
+		CreatedAt: nostr.Timestamp(seed.next(8)),
+		Tags:      tags,
+	}
+}
+
+func expectedDispatcherCandidates(active map[int]subscription, event nostr.Event) []string {
+	ids := make([]string, 0, len(active))
+	for _, sub := range active {
+		if sub.filter.Matches(event) {
+			ids = append(ids, sub.id)
+		}
+	}
+	slices.Sort(ids)
+	return ids
+}
+
+func collectedDispatcherCandidates(d *dispatcher, event nostr.Event) []string {
+	ids := make([]string, 0, d.subscriptions.Size())
+	for sub := range d.candidates(event) {
+		ids = append(ids, sub.id)
+	}
+	slices.Sort(ids)
+	return ids
+}

khatru/docs/core/routing.md

@@ -1,58 +0,0 @@
----
-outline: deep
----
-
-# Request Routing
-
-If you have one (or more) set of policies that have to be executed in sequence (for example, first you check for the presence of a tag, then later in the next policies you use that tag without checking) and they only apply to some class of events, but you still want your relay to deal with other classes of events that can lead to cumbersome sets of rules, always having to check if an event meets the requirements and so on. There is where routing can help you.
-
-```go
-sk := os.Getenv("RELAY_SECRET_KEY")
-
-// a relay for NIP-29 groups
-groupsStore := boltdb.BoltBackend{}
-groupsStore.Init()
-groupsRelay, _ := khatru29.Init(relay29.Options{Domain: "example.com", DB: groupsStore, SecretKey: sk})
-// ...
-
-// a relay for everything else
-publicStore := slicestore.SliceStore{}
-publicStore.Init()
-publicRelay := khatru.NewRelay()
-publicRelay.UseEventStore(publicStore, 1000)
-// ...
-
-// a higher-level relay that just routes between the two above
-router := khatru.NewRouter()
-
-// route requests and events to the groups relay
-router.Route().
-	Req(func (filter nostr.Filter) bool {
-		_, hasHTag := filter.Tags["h"]
-		if hasHTag {
-			return true
-		}
-		return slices.Contains(filter.Kinds, func (k int) bool { return k == 39000 || k == 39001 || k == 39002 })
-	}).
-	Event(func (event *nostr.Event) bool {
-		switch {
-		case event.Kind <= 9021 && event.Kind >= 9000:
-			return true
-		case event.Kind <= 39010 && event.Kind >= 39000:
-			return true
-		case event.Kind <= 12 && event.Kind >= 9:
-			return true
-		case event.Tags.Find("h") != nil:
-			return true
-		default:
-			return false
-		}
-	}).
-	Relay(groupsRelay)
-
-// route requests and events to the other
-router.Route().
-	Req(func (filter nostr.Filter) bool { return true }).
-	Event(func (event *nostr.Event) bool { return true }).
-	Relay(publicRelay)
-```

khatru/examples/routing/main.go

@@ -1,61 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"net/http"
-	"slices"
-
-	"fiatjaf.com/nostr"
-	"fiatjaf.com/nostr/eventstore/lmdb"
-	"fiatjaf.com/nostr/eventstore/slicestore"
-	"fiatjaf.com/nostr/khatru"
-)
-
-func main() {
-	db1 := &slicestore.SliceStore{}
-	db1.Init()
-	r1 := khatru.NewRelay()
-	r1.UseEventstore(db1, 400)
-
-	db2 := &lmdb.LMDBBackend{Path: "/tmp/t"}
-	db2.Init()
-	r2 := khatru.NewRelay()
-	r2.UseEventstore(db2, 400)
-
-	db3 := &slicestore.SliceStore{}
-	db3.Init()
-	r3 := khatru.NewRelay()
-	r3.UseEventstore(db3, 400)
-
-	router := khatru.NewRouter()
-
-	router.Route().
-		Req(func(filter nostr.Filter) bool {
-			return slices.Contains(filter.Kinds, 30023)
-		}).
-		Event(func(event *nostr.Event) bool {
-			return event.Kind == 30023
-		}).
-		Relay(r1)
-
-	router.Route().
-		Req(func(filter nostr.Filter) bool {
-			return slices.Contains(filter.Kinds, 1) && slices.Contains(filter.Tags["t"], "spam")
-		}).
-		Event(func(event *nostr.Event) bool {
-			return event.Kind == 1 && event.Tags.FindWithValue("t", "spam") != nil
-		}).
-		Relay(r2)
-
-	router.Route().
-		Req(func(filter nostr.Filter) bool {
-			return slices.Contains(filter.Kinds, 1)
-		}).
-		Event(func(event *nostr.Event) bool {
-			return event.Kind == 1
-		}).
-		Relay(r3)
-
-	fmt.Println("running on :3334")
-	http.ListenAndServe(":3334", router)
-}

khatru/expiration.go

@@ -39,9 +39,15 @@ type expirationManager struct {
 	events expiringEventHeap
 	mu     sync.Mutex
 
+	// a function to query the relay database, generally the same as relay.queryStored
 	queryStored func(ctx context.Context, filter nostr.Filter) iter.Seq[nostr.Event]
+
+	// a function to delete an event from the relay database, generally the same as relay.DeleteEvent
 	deleteEvent func(ctx context.Context, id nostr.ID) error
 
+	// a function to call after an event has been deleted, generally the same as relay.OnEventDeleted
+	deleteCallback func(ctx context.Context, id nostr.Event)
+
 	interval        time.Duration
 	initialScanDone bool
 	kill            chan struct{} // used for manually killing this
@@ -109,7 +115,11 @@ func (em *expirationManager) checkExpiredEvents(ctx context.Context) {
 		heap.Pop(&em.events)
 
 		ctx := context.WithValue(ctx, internalCallKey, struct{}{})
-		em.deleteEvent(ctx, next.id)
+		if nil == em.deleteEvent(ctx, next.id) && em.deleteCallback != nil {
+			for evt := range em.queryStored(ctx, nostr.Filter{IDs: []nostr.ID{next.id}}) {
+				em.deleteCallback(ctx, evt)
+			}
+		}
 	}
 }
 
@@ -142,12 +152,14 @@ func (em *expirationManager) removeEvent(id nostr.ID) {
 func (rl *Relay) StartExpirationManager(
 	queryStored func(ctx context.Context, filter nostr.Filter) iter.Seq[nostr.Event],
 	deleteEvent func(ctx context.Context, id nostr.ID) error,
+	onDeleteCallback func(ctx context.Context, evt nostr.Event),
 ) {
 	rl.expirationManager = &expirationManager{
 		events: make(expiringEventHeap, 0),
 
 		queryStored:    queryStored,
 		deleteEvent:    deleteEvent,
+		deleteCallback: onDeleteCallback,
 
 		interval: time.Hour,
 		kill:     make(chan struct{}),

khatru/get-started.go

@@ -1,65 +0,0 @@
-package khatru
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/fasthttp/websocket"
-	"github.com/rs/cors"
-)
-
-func (rl *Relay) Router() *http.ServeMux {
-	return rl.serveMux
-}
-
-func (rl *Relay) SetRouter(mux *http.ServeMux) {
-	rl.serveMux = mux
-}
-
-// Start creates an http server and starts listening on given host and port.
-func (rl *Relay) Start(host string, port int, started ...chan bool) error {
-	addr := net.JoinHostPort(host, strconv.Itoa(port))
-	ln, err := net.Listen("tcp", addr)
-	if err != nil {
-		return err
-	}
-
-	rl.Addr = ln.Addr().String()
-	rl.httpServer = &http.Server{
-		Handler:      cors.Default().Handler(rl),
-		Addr:         addr,
-		WriteTimeout: 2 * time.Second,
-		ReadTimeout:  2 * time.Second,
-		IdleTimeout:  30 * time.Second,
-	}
-
-	// notify caller that we're starting
-	for _, started := range started {
-		close(started)
-	}
-
-	if err := rl.httpServer.Serve(ln); err == http.ErrServerClosed {
-		return nil
-	} else if err != nil {
-		return err
-	} else {
-		return nil
-	}
-}
-
-// Shutdown sends a websocket close control message to all connected clients.
-func (rl *Relay) Shutdown(ctx context.Context) {
-	rl.httpServer.Shutdown(ctx)
-	rl.clientsMutex.Lock()
-	defer rl.clientsMutex.Unlock()
-	for ws := range rl.clients {
-		ws.conn.WriteControl(websocket.CloseMessage, nil, time.Now().Add(time.Second))
-		ws.cancel()
-		ws.conn.Close()
-	}
-	clear(rl.clients)
-	rl.listeners = rl.listeners[:0]
-}

khatru/handlers.go

@@ -43,8 +43,8 @@ func (rl *Relay) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	})
 
 	relayPathMatches := true
-	if rl.ServiceURL != "" {
+	if serviceURL := rl.getServiceURL(r); serviceURL != "" {
-		p, err := url.Parse(rl.ServiceURL)
+		p, err := url.Parse(serviceURL)
 		if err == nil {
 			relayPathMatches = strings.TrimSuffix(r.URL.Path, "/") == strings.TrimSuffix(p.Path, "/")
 		}
@@ -108,7 +108,9 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 		),
 	)
 
+	killOnce := sync.Once{}
 	kill := func() {
+		killOnce.Do(func() {
 			if nil != rl.OnDisconnect {
 				rl.OnDisconnect(ctx)
 			}
@@ -119,6 +121,7 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 			ws.conn.Close()
 
 			rl.removeClientAndListeners(ws)
+		})
 	}
 
 	go func() {
@@ -214,35 +217,33 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						return
 					}
 
-					srl := rl
-					if rl.getSubRelayFromEvent != nil {
-						srl = rl.getSubRelayFromEvent(&env.Event)
-					}
-
 					var ok bool
 					var writeErr error
 					var skipBroadcast bool
 
 					if env.Event.Kind == nostr.KindDeletion {
 						// store the delete event first
-						skipBroadcast, writeErr = srl.handleNormal(ctx, env.Event)
+						skipBroadcast, writeErr = rl.handleNormal(ctx, env.Event)
 						if writeErr == nil {
 							// this always returns "blocked: " whenever it returns an error
-							writeErr = srl.handleDeleteRequest(ctx, env.Event)
+							writeErr = rl.handleDeleteRequest(ctx, env.Event)
+							if writeErr == ErrNothingToDelete {
+								writeErr = nil
+							}
 						}
 					} else if env.Event.Kind.IsEphemeral() {
 						// this will also always return a prefixed reason
-						writeErr = srl.handleEphemeral(ctx, env.Event)
+						writeErr = rl.handleEphemeral(ctx, env.Event)
 					} else {
 						// this will also always return a prefixed reason
-						skipBroadcast, writeErr = srl.handleNormal(ctx, env.Event)
+						skipBroadcast, writeErr = rl.handleNormal(ctx, env.Event)
 					}
 
 					var reason string
 					if writeErr == nil {
 						ok = true
 						if !skipBroadcast {
-							n := srl.notifyListeners(env.Event, false)
+							n := rl.notifyListeners(env.Event, false)
 
 							// the number of notified listeners matters in ephemeral events
 							if env.Event.Kind.IsEphemeral() {
@@ -275,15 +276,10 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 					var total uint32
 					var hll *hyperloglog.HyperLogLog
 
-					srl := rl
-					if rl.getSubRelayFromFilter != nil {
-						srl = rl.getSubRelayFromFilter(env.Filter)
-					}
-
 					if offset := nip45.HyperLogLogEventPubkeyOffsetForFilter(env.Filter); offset != -1 {
-						total, hll = srl.handleCountRequestWithHLL(ctx, ws, env.Filter, offset)
+						total, hll = rl.handleCountRequestWithHLL(ctx, ws, env.Filter, offset)
 					} else {
-						total = srl.handleCountRequest(ctx, ws, env.Filter)
+						total = rl.handleCountRequest(ctx, ws, env.Filter)
 					}
 
 					resp := nostr.CountEnvelope{
@@ -297,8 +293,7 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 					ws.WriteJSON(resp)
 
 				case *nostr.ReqEnvelope:
-					eose := sync.WaitGroup{}
+					rl.removeListenerId(ws, env.SubscriptionID)
-					eose.Add(len(env.Filters))
 
 					// a context just for the "stored events" request handler
 					reqCtx, cancelReqCtx := context.WithCancelCause(ctx)
@@ -308,11 +303,7 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 
 					// handle each filter separately -- dispatching events as they're loaded from databases
 					for _, filter := range env.Filters {
-						srl := rl
+						err := rl.handleRequest(reqCtx, env.SubscriptionID, ws, filter)
-						if rl.getSubRelayFromFilter != nil {
-							srl = rl.getSubRelayFromFilter(filter)
-						}
-						err := srl.handleRequest(reqCtx, env.SubscriptionID, &eose, ws, filter)
 						if err != nil {
 							// fail everything if any filter is rejected
 							reason := err.Error()
@@ -322,16 +313,15 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 							ws.WriteJSON(nostr.ClosedEnvelope{SubscriptionID: env.SubscriptionID, Reason: reason})
 							cancelReqCtx(errors.New("filter rejected"))
 							return
-						} else {
+						} else if filter.IDs == nil {
-							rl.addListener(ws, env.SubscriptionID, srl, filter, cancelReqCtx)
+							// a query that is just a bunch of "ids": [...] will not add listeners.
+							// is this a bug? maybe, but I don't think anyone is listening for an ID
+							// that hasn't been published yet anywhere -- if yes we can change later
+							rl.addListener(ws, env.SubscriptionID, filter, cancelReqCtx)
 						}
 					}
 
-					go func() {
-						// when all events have been loaded from databases and dispatched we can fire the EOSE message
-						eose.Wait()
 					ws.WriteJSON(nostr.EOSEEnvelope(env.SubscriptionID))
-					}()
 				case *nostr.CloseEnvelope:
 					id := string(*env)
 					rl.removeListenerId(ws, id)
@@ -356,19 +346,18 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						}
 						ws.authLock.Unlock()
 						ws.WriteJSON(nostr.OKEnvelope{EventID: env.Event.ID, OK: true})
+						if rl.OnAuth != nil {
+							rl.OnAuth(ctx, pubkey)
+						}
 					} else {
 						ws.WriteJSON(nostr.OKEnvelope{EventID: env.Event.ID, OK: false, Reason: "error: failed to authenticate: " + err.Error()})
 					}
 				case *nip77.OpenEnvelope:
-					srl := rl
+					if !rl.Negentropy {
-					if rl.getSubRelayFromFilter != nil {
-						srl = rl.getSubRelayFromFilter(env.Filter)
-						if !srl.Negentropy {
 						// ignore
 						return
 					}
-					}
+					vec, err := rl.startNegentropySession(ctx, env.Filter)
-					vec, err := srl.startNegentropySession(ctx, env.Filter)
 					if err != nil {
 						// fail everything if any filter is rejected
 						reason := err.Error()

khatru/listener.go

@@ -3,18 +3,20 @@ package khatru
 import (
 	"context"
 	"errors"
-	"slices"
+	"iter"
+	"sync"
 
+	"fiatjaf.com/lib/set"
 	"fiatjaf.com/nostr"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 var ErrSubscriptionClosedByClient = errors.New("subscription closed by client")
 
 type listenerSpec struct {
-	id       string // kept here so we can easily match against it removeListenerId
+	ssid   int    // internal numeric id for a listener
+	sid    string // client-provided subscription id
 	cancel context.CancelCauseFunc
-	index    int
-	subrelay *Relay // this is important when we're dealing with routing, otherwise it will be always the same
 }
 
 type listener struct {
@@ -23,10 +25,262 @@ type listener struct {
 	ws     *WebSocket
 }
 
+type subscription struct {
+	id     string
+	filter nostr.Filter
+	ws     *WebSocket
+}
+
+type dispatcher struct {
+	serial          int
+	subscriptions   *xsync.MapOf[int, subscription]
+	byAuthor        *xsync.MapOf[nostr.PubKey, set.Set[int]]
+	byKind          *xsync.MapOf[nostr.Kind, set.Set[int]]
+	fallbackTags    set.Set[int]
+	fallbackNothing set.Set[int]
+}
+
+var setPool = sync.Pool{
+	New: func() any {
+		return set.NewEmptySliceSetReusing[int](make([]int, 0, 10))
+	},
+}
+
+func newDispatcher() dispatcher {
+	return dispatcher{
+		subscriptions:   xsync.NewMapOf[int, subscription](),
+		byAuthor:        xsync.NewMapOf[nostr.PubKey, set.Set[int]](),
+		byKind:          xsync.NewMapOf[nostr.Kind, set.Set[int]](),
+		fallbackTags:    setPool.Get().(set.Set[int]),
+		fallbackNothing: setPool.Get().(set.Set[int]),
+	}
+}
+
+func (d *dispatcher) addSubscription(sub subscription) int {
+	d.serial++
+	ssid := d.serial
+
+	d.subscriptions.Store(ssid, sub)
+
+	indexed := false
+	if sub.filter.Authors != nil {
+		indexed = true
+		for _, author := range sub.filter.Authors {
+			d.byAuthor.Compute(author, func(s set.Set[int], loaded bool) (set.Set[int], bool) {
+				if !loaded {
+					s = setPool.Get().(set.Set[int])
+				}
+				s.Add(ssid)
+				return s, false
+			})
+		}
+	}
+
+	if sub.filter.Kinds != nil {
+		indexed = true
+		for _, kind := range sub.filter.Kinds {
+			d.byKind.Compute(kind, func(s set.Set[int], loaded bool) (set.Set[int], bool) {
+				if !loaded {
+					s = setPool.Get().(set.Set[int])
+				}
+				s.Add(ssid)
+				return s, false
+			})
+		}
+	}
+
+	if !indexed {
+		if sub.filter.Tags != nil {
+			d.fallbackTags.Add(ssid)
+		} else {
+			d.fallbackNothing.Add(ssid)
+		}
+	}
+
+	return ssid
+}
+
+func (d *dispatcher) removeSubscription(ssid int) nostr.Filter {
+	var filter nostr.Filter
+
+	d.subscriptions.Compute(ssid, func(sub subscription, loaded bool) (subscription, bool) {
+		indexed := false
+
+		filter = sub.filter
+
+		if sub.filter.Authors != nil {
+			indexed = true
+			for _, author := range sub.filter.Authors {
+				d.byAuthor.Compute(author, func(s set.Set[int], loaded bool) (set.Set[int], bool) {
+					if !loaded {
+						return s, true
+					}
+					s.Remove(ssid)
+
+					delete := s.Len() == 0
+					if delete {
+						setPool.Put(s)
+					}
+					return s, delete
+				})
+			}
+		}
+
+		if sub.filter.Kinds != nil {
+			indexed = true
+			for _, kind := range sub.filter.Kinds {
+				d.byKind.Compute(kind, func(s set.Set[int], loaded bool) (set.Set[int], bool) {
+					if !loaded {
+						return s, true
+					}
+					s.Remove(ssid)
+
+					delete := s.Len() == 0
+					if delete {
+						setPool.Put(s)
+					}
+					return s, delete
+				})
+			}
+		}
+
+		if !indexed {
+			if sub.filter.Tags != nil {
+				d.fallbackTags.Remove(ssid)
+			} else {
+				d.fallbackNothing.Remove(ssid)
+			}
+		}
+
+		return sub, true
+	})
+
+	return filter
+}
+
+func (d *dispatcher) candidates(event nostr.Event) iter.Seq[subscription] {
+	return func(yield func(subscription) bool) {
+		authorSubs, hasAuthorSubs := d.byAuthor.Load(event.PubKey)
+		kindSubs, hasKindSubs := d.byKind.Load(event.Kind)
+
+		if hasAuthorSubs && hasKindSubs {
+			for _, ssid := range authorSubs.Slice() {
+				sub, _ := d.subscriptions.Load(ssid)
+
+				if kindSubs.Has(ssid) || sub.filter.Kinds == nil {
+					if filterMatchesTimestampConstraintsAndTags(sub.filter, event) {
+						if !yield(sub) {
+							return
+						}
+					}
+				}
+			}
+
+			for _, ssid := range kindSubs.Slice() {
+				sub, _ := d.subscriptions.Load(ssid)
+
+				if sub.filter.Authors != nil {
+					continue
+				}
+
+				if filterMatchesTimestampConstraintsAndTags(sub.filter, event) {
+					if !yield(sub) {
+						return
+					}
+				}
+			}
+		} else if hasAuthorSubs {
+			for _, ssid := range authorSubs.Slice() {
+				sub, _ := d.subscriptions.Load(ssid)
+
+				if sub.filter.Kinds != nil {
+					// if there are any kinds in the filter we already know this doesn't qualify
+					continue
+				}
+
+				if filterMatchesTimestampConstraintsAndTags(sub.filter, event) {
+					if !yield(sub) {
+						return
+					}
+				}
+			}
+		} else if hasKindSubs {
+			for _, ssid := range kindSubs.Slice() {
+				sub, _ := d.subscriptions.Load(ssid)
+
+				if sub.filter.Authors != nil {
+					// if there are any authors in the filter we already know this doesn't qualify
+					continue
+				}
+
+				if filterMatchesTimestampConstraintsAndTags(sub.filter, event) {
+					if !yield(sub) {
+						return
+					}
+				}
+			}
+		}
+
+		if len(event.Tags) > 0 {
+			for _, ssid := range d.fallbackTags.Slice() {
+				sub, _ := d.subscriptions.Load(ssid)
+
+				if filterMatchesTimestampConstraintsAndTags(sub.filter, event) {
+					if !yield(sub) {
+						return
+					}
+				}
+			}
+		}
+
+		for _, ssid := range d.fallbackNothing.Slice() {
+			sub, _ := d.subscriptions.Load(ssid)
+			if filterMatchesTimestampConstraints(sub.filter, event) {
+				if !yield(sub) {
+					return
+				}
+			}
+		}
+	}
+}
+
+//go:inline
+func filterMatchesTimestampConstraints(filter nostr.Filter, event nostr.Event) bool {
+	if filter.Since != 0 && event.CreatedAt < filter.Since {
+		return false
+	}
+
+	if filter.Until != 0 && event.CreatedAt > filter.Until {
+		return false
+	}
+
+	return true
+}
+
+//go:inline
+func filterMatchesTimestampConstraintsAndTags(filter nostr.Filter, event nostr.Event) bool {
+	if !filterMatchesTimestampConstraints(filter, event) {
+		return false
+	}
+
+	for f, v := range filter.Tags {
+		if !event.Tags.ContainsAny(f, v) {
+			return false
+		}
+	}
+
+	return true
+}
+
+//go:inline
+func tagKeyValueKey(tagKey, tagValue string) string {
+	return tagKey + "\x00" + tagValue
+}
+
 func (rl *Relay) GetListeningFilters() []nostr.Filter {
-	respfilters := make([]nostr.Filter, len(rl.listeners))
+	respfilters := make([]nostr.Filter, 0, rl.dispatcher.subscriptions.Size())
-	for i, l := range rl.listeners {
+	for _, sub := range rl.dispatcher.subscriptions.Range {
-		respfilters[i] = l.filter
+		respfilters = append(respfilters, sub.filter)
 	}
 	return respfilters
 }
@@ -36,26 +290,31 @@ func (rl *Relay) GetListeningFilters() []nostr.Filter {
 func (rl *Relay) addListener(
 	ws *WebSocket,
 	id string,
-	subrelay *Relay,
 	filter nostr.Filter,
 	cancel context.CancelCauseFunc,
 ) {
-	rl.clientsMutex.Lock()
+	select {
+	case <-rl.clientsMutex.C():
 		defer rl.clientsMutex.Unlock()
+	case <-ws.Context.Done():
+		return
+	}
 
 	if specs, ok := rl.clients[ws]; ok /* this will always be true unless client has disconnected very rapidly */ {
-		idx := len(subrelay.listeners)
+		ssid := rl.dispatcher.addSubscription(subscription{
-		rl.clients[ws] = append(specs, listenerSpec{
-			id:       id,
-			cancel:   cancel,
-			subrelay: subrelay,
-			index:    idx,
-		})
-		subrelay.listeners = append(subrelay.listeners, listener{
 			ws:     ws,
 			id:     id,
 			filter: filter,
 		})
+		rl.clients[ws] = append(specs, listenerSpec{
+			ssid:   ssid,
+			cancel: cancel,
+			sid:    id,
+		})
+
+		if rl.OnListenerAdded != nil {
+			rl.OnListenerAdded(ws, ssid, id, filter)
+		}
 	}
 }
 
@@ -66,35 +325,21 @@ func (rl *Relay) removeListenerId(ws *WebSocket, id string) {
 	defer rl.clientsMutex.Unlock()
 
 	if specs, ok := rl.clients[ws]; ok {
-		// swap delete specs that match this id
+		kept := specs[:0]
-		for s := len(specs) - 1; s >= 0; s-- {
+		for _, spec := range specs {
-			spec := specs[s]
+			if spec.sid == id {
-			if spec.id == id {
 				spec.cancel(ErrSubscriptionClosedByClient)
-				specs[s] = specs[len(specs)-1]
+				filter := rl.dispatcher.removeSubscription(spec.ssid)
-				specs = specs[0 : len(specs)-1]
-				rl.clients[ws] = specs
 
-				// swap delete listeners one at a time, as they may be each in a different subrelay
+				if rl.OnListenerRemoved != nil {
-				srl := spec.subrelay // == rl in normal cases, but different when this came from a route
+					rl.OnListenerRemoved(ws, spec.ssid, id, filter)
+				}
 
-				if spec.index != len(srl.listeners)-1 {
+				continue
-					movedFromIndex := len(srl.listeners) - 1
-					moved := srl.listeners[movedFromIndex] // this wasn't removed, but will be moved
-					srl.listeners[spec.index] = moved
-
-					// now we must update the the listener we just moved
-					// so its .index reflects its new position on srl.listeners
-					movedSpecs := rl.clients[moved.ws]
-					idx := slices.IndexFunc(movedSpecs, func(ls listenerSpec) bool {
-						return ls.index == movedFromIndex && ls.subrelay == srl
-					})
-					movedSpecs[idx].index = spec.index
-					rl.clients[moved.ws] = movedSpecs
-				}
-				srl.listeners = srl.listeners[0 : len(srl.listeners)-1] // finally reduce the slice length
 			}
+			kept = append(kept, spec)
 		}
+		rl.clients[ws] = kept
 	}
 }
 
@@ -102,31 +347,13 @@ func (rl *Relay) removeClientAndListeners(ws *WebSocket) {
 	rl.clientsMutex.Lock()
 	defer rl.clientsMutex.Unlock()
 	if specs, ok := rl.clients[ws]; ok {
-		// swap delete listeners and delete client (all specs will be deleted)
+		for _, spec := range specs {
-		for s, spec := range specs {
 			// no need to cancel contexts since they inherit from the main connection context
-			// just delete the listeners (swap-delete)
+			filter := rl.dispatcher.removeSubscription(spec.ssid)
-			srl := spec.subrelay
 
-			if spec.index != len(srl.listeners)-1 {
+			if rl.OnListenerRemoved != nil {
-				movedFromIndex := len(srl.listeners) - 1
+				rl.OnListenerRemoved(ws, spec.ssid, spec.sid, filter)
-				moved := srl.listeners[movedFromIndex] // this wasn't removed, but will be moved
-				srl.listeners[spec.index] = moved
-
-				// temporarily update the spec of the listener being removed to have index == -1
-				// (since it was removed) so it doesn't match in the search below
-				rl.clients[ws][s].index = -1
-
-				// now we must update the the listener we just moved
-				// so its .index reflects its new position on srl.listeners
-				movedSpecs := rl.clients[moved.ws]
-				idx := slices.IndexFunc(movedSpecs, func(ls listenerSpec) bool {
-					return ls.index == movedFromIndex && ls.subrelay == srl
-				})
-				movedSpecs[idx].index = spec.index
-				rl.clients[moved.ws] = movedSpecs
 			}
-			srl.listeners = srl.listeners[0 : len(srl.listeners)-1] // finally reduce the slice length
 		}
 	}
 	delete(rl.clients, ws)
@@ -136,16 +363,14 @@ func (rl *Relay) removeClientAndListeners(ws *WebSocket) {
 func (rl *Relay) notifyListeners(event nostr.Event, skipPrevent bool) int {
 	count := 0
 listenersloop:
-	for _, listener := range rl.listeners {
+	for sub := range rl.dispatcher.candidates(event) {
-		if listener.filter.Matches(event) {
 		if !skipPrevent && nil != rl.PreventBroadcast {
-				if rl.PreventBroadcast(listener.ws, listener.filter, event) {
+			if rl.PreventBroadcast(sub.ws, sub.filter, event) {
 				continue listenersloop
 			}
 		}
-			listener.ws.WriteJSON(nostr.EventEnvelope{SubscriptionID: &listener.id, Event: event})
+		sub.ws.WriteJSON(nostr.EventEnvelope{SubscriptionID: &sub.id, Event: event})
 		count++
 	}
-	}
 	return count
 }

khatru/listener_fuzz_test.go

@@ -1,7 +1,6 @@
 package khatru
 
 import (
-	"math/rand"
 	"testing"
 
 	"fiatjaf.com/nostr"
@@ -25,7 +24,7 @@ func FuzzRandomListenerClientRemoving(f *testing.F) {
 		l := 0
 
 		for i := 0; i < totalWebsockets; i++ {
-			ws := &WebSocket{}
+			ws := &WebSocket{Context: rl.ctx}
 			websockets = append(websockets, ws)
 			rl.clients[ws] = nil
 		}
@@ -38,7 +37,7 @@ func FuzzRandomListenerClientRemoving(f *testing.F) {
 
 				if s%addListenerFreq == 0 {
 					l++
-					rl.addListener(ws, w+":"+idFromSeqLower(j), rl, f, cancel)
+					rl.addListener(ws, w+":"+idFromSeqLower(j), f, cancel)
 				}
 
 				s++
@@ -46,14 +45,22 @@ func FuzzRandomListenerClientRemoving(f *testing.F) {
 		}
 
 		require.Len(t, rl.clients, totalWebsockets)
-		require.Len(t, rl.listeners, l)
+		ssidCount := 0
+		for _, specs := range rl.clients {
+			ssidCount += len(specs)
+		}
+		require.Equal(t, l, ssidCount)
 
 		for ws := range rl.clients {
 			rl.removeClientAndListeners(ws)
 		}
 
 		require.Len(t, rl.clients, 0)
-		require.Len(t, rl.listeners, 0)
+		ssidCount = 0
+		for _, specs := range rl.clients {
+			ssidCount += len(specs)
+		}
+		require.Equal(t, 0, ssidCount)
 	})
 }
 
@@ -84,7 +91,7 @@ func FuzzRandomListenerIdRemoving(f *testing.F) {
 		extra := 0
 
 		for i := 0; i < totalWebsockets; i++ {
-			ws := &WebSocket{}
+			ws := &WebSocket{Context: rl.ctx}
 			websockets = append(websockets, ws)
 			rl.clients[ws] = nil
 		}
@@ -97,11 +104,11 @@ func FuzzRandomListenerIdRemoving(f *testing.F) {
 
 				if s%addListenerFreq == 0 {
 					id := w + ":" + idFromSeqLower(j)
-					rl.addListener(ws, id, rl, f, cancel)
+					rl.addListener(ws, id, f, cancel)
 					subs = append(subs, wsid{ws, id})
 
 					if s%addExtraListenerFreq == 0 {
-						rl.addListener(ws, id, rl, f, cancel)
+						rl.addListener(ws, id, f, cancel)
 						extra++
 					}
 				}
@@ -111,16 +118,21 @@ func FuzzRandomListenerIdRemoving(f *testing.F) {
 		}
 
 		require.Len(t, rl.clients, totalWebsockets)
-		require.Len(t, rl.listeners, len(subs)+extra)
+		ssidCount := 0
+		for _, specs := range rl.clients {
+			ssidCount += len(specs)
+		}
+		require.Equal(t, len(subs)+extra, ssidCount)
 
-		rand.Shuffle(len(subs), func(i, j int) {
+		for _, wsidToRemove := range moduloOrder(subs, int(utw+ubs+ualf+ualef)) {
-			subs[i], subs[j] = subs[j], subs[i]
-		})
-		for _, wsidToRemove := range subs {
 			rl.removeListenerId(wsidToRemove.ws, wsidToRemove.id)
 		}
 
-		require.Len(t, rl.listeners, 0)
+		ssidCount = 0
+		for _, specs := range rl.clients {
+			ssidCount += len(specs)
+		}
+		require.Equal(t, 0, ssidCount)
 		require.Len(t, rl.clients, totalWebsockets)
 		for _, specs := range rl.clients {
 			require.Len(t, specs, 0)
@@ -129,23 +141,17 @@ func FuzzRandomListenerIdRemoving(f *testing.F) {
 }
 
 func FuzzRouterListenersPabloCrash(f *testing.F) {
-	f.Add(uint(3), uint(6), uint(2), uint(20))
+	f.Add(uint(6), uint(2), uint(20))
-	f.Fuzz(func(t *testing.T, totalRelays uint, totalConns uint, subFreq uint, subIterations uint) {
+	f.Fuzz(func(t *testing.T, totalConns uint, subFreq uint, subIterations uint) {
-		totalRelays++
 		totalConns++
 		subFreq++
 		subIterations++
 
 		rl := NewRelay()
 
-		relays := make([]*Relay, int(totalRelays))
-		for i := 0; i < int(totalRelays); i++ {
-			relays[i] = NewRelay()
-		}
-
 		conns := make([]*WebSocket, int(totalConns))
 		for i := 0; i < int(totalConns); i++ {
-			ws := &WebSocket{}
+			ws := &WebSocket{Context: rl.ctx}
 			conns[i] = ws
 			rl.clients[ws] = make([]listenerSpec, 0, subIterations)
 		}
@@ -159,20 +165,18 @@ func FuzzRouterListenersPabloCrash(f *testing.F) {
 		}
 
 		s := 0
-		subs := make([]wsid, 0, subIterations*totalConns*totalRelays)
+		subs := make([]wsid, 0, subIterations*totalConns)
 		for i, conn := range conns {
 			w := idFromSeqUpper(i)
 			for j := 0; j < int(subIterations); j++ {
 				id := w + ":" + idFromSeqLower(j)
-				for _, rlt := range relays {
 				if s%int(subFreq) == 0 {
-						rl.addListener(conn, id, rlt, f, cancel)
+					rl.addListener(conn, id, f, cancel)
 					subs = append(subs, wsid{conn, id})
 				}
 				s++
 			}
 		}
-		}
 
 		for _, wsid := range subs {
 			rl.removeListenerId(wsid.ws, wsid.id)
@@ -181,8 +185,5 @@ func FuzzRouterListenersPabloCrash(f *testing.F) {
 		for _, wsid := range subs {
 			require.Len(t, rl.clients[wsid.ws], 0)
 		}
-		for _, rlt := range relays {
-			require.Len(t, rlt.listeners, 0)
-		}
 	})
 }

khatru/listener_test.go

@@ -1,7 +1,6 @@
 package khatru
 
 import (
-	"math/rand"
 	"strings"
 	"testing"
 
@@ -23,11 +22,23 @@ func idFromSeq(seq int, min, max int) string {
 	return result.String()
 }
 
+func moduloOrder[T any](items []T, seed int) []T {
+	remaining := append([]T(nil), items...)
+	ordered := make([]T, 0, len(items))
+	for len(remaining) > 0 {
+		idx := seed % len(remaining)
+		ordered = append(ordered, remaining[idx])
+		remaining = append(remaining[:idx], remaining[idx+1:]...)
+		seed++
+	}
+	return ordered
+}
+
 func TestListenerSetupAndRemoveOnce(t *testing.T) {
 	rl := NewRelay()
 
-	ws1 := &WebSocket{}
+	ws1 := &WebSocket{Context: rl.ctx}
-	ws2 := &WebSocket{}
+	ws2 := &WebSocket{Context: rl.ctx}
 
 	f1 := nostr.Filter{Kinds: []nostr.Kind{1}}
 	f2 := nostr.Filter{Kinds: []nostr.Kind{2}}
@@ -39,28 +50,21 @@ func TestListenerSetupAndRemoveOnce(t *testing.T) {
 	var cancel func(cause error) = nil
 
 	t.Run("adding listeners", func(t *testing.T) {
-		rl.addListener(ws1, "1a", rl, f1, cancel)
+		rl.addListener(ws1, "1a", f1, cancel)
-		rl.addListener(ws1, "1b", rl, f2, cancel)
+		rl.addListener(ws1, "1b", f2, cancel)
-		rl.addListener(ws2, "2a", rl, f3, cancel)
+		rl.addListener(ws2, "2a", f3, cancel)
-		rl.addListener(ws1, "1c", rl, f3, cancel)
+		rl.addListener(ws1, "1c", f3, cancel)
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"1a", cancel, 0, rl},
+				{1, "1a", cancel},
-				{"1b", cancel, 1, rl},
+				{2, "1b", cancel},
-				{"1c", cancel, 3, rl},
+				{4, "1c", cancel},
 			},
 			ws2: {
-				{"2a", cancel, 2, rl},
+				{3, "2a", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"1a", f1, ws1},
-			{"1b", f2, ws1},
-			{"2a", f3, ws2},
-			{"1c", f3, ws1},
-		}, rl.listeners)
 	})
 
 	t.Run("removing a client", func(t *testing.T) {
@@ -68,23 +72,19 @@ func TestListenerSetupAndRemoveOnce(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws2: {
-				{"2a", cancel, 0, rl},
+				{3, "2a", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"2a", f3, ws2},
-		}, rl.listeners)
 	})
 }
 
 func TestListenerMoreConvolutedCase(t *testing.T) {
 	rl := NewRelay()
 
-	ws1 := &WebSocket{}
+	ws1 := &WebSocket{Context: rl.ctx}
-	ws2 := &WebSocket{}
+	ws2 := &WebSocket{Context: rl.ctx}
-	ws3 := &WebSocket{}
+	ws3 := &WebSocket{Context: rl.ctx}
-	ws4 := &WebSocket{}
+	ws4 := &WebSocket{Context: rl.ctx}
 
 	f1 := nostr.Filter{Kinds: []nostr.Kind{1}}
 	f2 := nostr.Filter{Kinds: []nostr.Kind{2}}
@@ -98,35 +98,27 @@ func TestListenerMoreConvolutedCase(t *testing.T) {
 	var cancel func(cause error) = nil
 
 	t.Run("adding listeners", func(t *testing.T) {
-		rl.addListener(ws1, "c", rl, f1, cancel)
+		rl.addListener(ws1, "c", f1, cancel)
-		rl.addListener(ws2, "b", rl, f2, cancel)
+		rl.addListener(ws2, "b", f2, cancel)
-		rl.addListener(ws3, "a", rl, f3, cancel)
+		rl.addListener(ws3, "a", f3, cancel)
-		rl.addListener(ws4, "d", rl, f3, cancel)
+		rl.addListener(ws4, "d", f3, cancel)
-		rl.addListener(ws2, "b", rl, f1, cancel)
+		rl.addListener(ws2, "b", f1, cancel)
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rl},
+				{1, "c", cancel},
 			},
 			ws2: {
-				{"b", cancel, 1, rl},
+				{2, "b", cancel},
-				{"b", cancel, 4, rl},
+				{5, "b", cancel},
 			},
 			ws3: {
-				{"a", cancel, 2, rl},
+				{3, "a", cancel},
 			},
 			ws4: {
-				{"d", cancel, 3, rl},
+				{4, "d", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"b", f2, ws2},
-			{"a", f3, ws3},
-			{"d", f3, ws4},
-			{"b", f1, ws2},
-		}, rl.listeners)
 	})
 
 	t.Run("removing a client", func(t *testing.T) {
@@ -134,85 +126,62 @@ func TestListenerMoreConvolutedCase(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rl},
+				{1, "c", cancel},
 			},
 			ws3: {
-				{"a", cancel, 2, rl},
+				{3, "a", cancel},
 			},
 			ws4: {
-				{"d", cancel, 1, rl},
+				{4, "d", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"d", f3, ws4},
-			{"a", f3, ws3},
-		}, rl.listeners)
 	})
 
 	t.Run("reorganize the first case differently and then remove again", func(t *testing.T) {
 		rl.clients = map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 1, rl},
+				{2, "c", cancel},
 			},
 			ws2: {
-				{"b", cancel, 2, rl},
+				{3, "b", cancel},
-				{"b", cancel, 4, rl},
+				{5, "b", cancel},
 			},
 			ws3: {
-				{"a", cancel, 0, rl},
+				{1, "a", cancel},
 			},
 			ws4: {
-				{"d", cancel, 3, rl},
+				{4, "d", cancel},
 			},
 		}
-		rl.listeners = []listener{
-			{"a", f3, ws3},
-			{"c", f1, ws1},
-			{"b", f2, ws2},
-			{"d", f3, ws4},
-			{"b", f1, ws2},
-		}
 
 		rl.removeClientAndListeners(ws2)
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 1, rl},
+				{2, "c", cancel},
 			},
 			ws3: {
-				{"a", cancel, 0, rl},
+				{1, "a", cancel},
 			},
 			ws4: {
-				{"d", cancel, 2, rl},
+				{4, "d", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"a", f3, ws3},
-			{"c", f1, ws1},
-			{"d", f3, ws4},
-		}, rl.listeners)
 	})
 }
 
 func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 	rl := NewRelay()
 
-	ws1 := &WebSocket{}
+	ws1 := &WebSocket{Context: rl.ctx}
-	ws2 := &WebSocket{}
+	ws2 := &WebSocket{Context: rl.ctx}
-	ws3 := &WebSocket{}
+	ws3 := &WebSocket{Context: rl.ctx}
-	ws4 := &WebSocket{}
+	ws4 := &WebSocket{Context: rl.ctx}
 
 	f1 := nostr.Filter{Kinds: []nostr.Kind{1}}
 	f2 := nostr.Filter{Kinds: []nostr.Kind{2}}
 	f3 := nostr.Filter{Kinds: []nostr.Kind{3}}
 
-	rlx := NewRelay()
-	rly := NewRelay()
-	rlz := NewRelay()
-
 	rl.clients[ws1] = nil
 	rl.clients[ws2] = nil
 	rl.clients[ws3] = nil
@@ -221,56 +190,37 @@ func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 	var cancel func(cause error) = nil
 
 	t.Run("adding listeners", func(t *testing.T) {
-		rl.addListener(ws1, "c", rlx, f1, cancel)
+		rl.addListener(ws1, "c", f1, cancel)
-		rl.addListener(ws2, "b", rly, f2, cancel)
+		rl.addListener(ws2, "b", f2, cancel)
-		rl.addListener(ws3, "a", rlz, f3, cancel)
+		rl.addListener(ws3, "a", f3, cancel)
-		rl.addListener(ws4, "d", rlx, f3, cancel)
+		rl.addListener(ws4, "d", f3, cancel)
-		rl.addListener(ws4, "e", rlx, f3, cancel)
+		rl.addListener(ws4, "e", f3, cancel)
-		rl.addListener(ws3, "a", rlx, f3, cancel)
+		rl.addListener(ws3, "a", f3, cancel)
-		rl.addListener(ws4, "e", rly, f3, cancel)
+		rl.addListener(ws4, "e", f3, cancel)
-		rl.addListener(ws3, "f", rly, f3, cancel)
+		rl.addListener(ws3, "f", f3, cancel)
-		rl.addListener(ws1, "g", rlz, f1, cancel)
+		rl.addListener(ws1, "g", f1, cancel)
-		rl.addListener(ws2, "g", rlz, f2, cancel)
+		rl.addListener(ws2, "g", f2, cancel)
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rlx},
+				{1, "c", cancel},
-				{"g", cancel, 1, rlz},
+				{9, "g", cancel},
 			},
 			ws2: {
-				{"b", cancel, 0, rly},
+				{2, "b", cancel},
-				{"g", cancel, 2, rlz},
+				{10, "g", cancel},
 			},
 			ws3: {
-				{"a", cancel, 0, rlz},
+				{3, "a", cancel},
-				{"a", cancel, 3, rlx},
+				{6, "a", cancel},
-				{"f", cancel, 2, rly},
+				{8, "f", cancel},
 			},
 			ws4: {
-				{"d", cancel, 1, rlx},
+				{4, "d", cancel},
-				{"e", cancel, 2, rlx},
+				{5, "e", cancel},
-				{"e", cancel, 1, rly},
+				{7, "e", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"d", f3, ws4},
-			{"e", f3, ws4},
-			{"a", f3, ws3},
-		}, rlx.listeners)
-
-		require.Equal(t, []listener{
-			{"b", f2, ws2},
-			{"e", f3, ws4},
-			{"f", f3, ws3},
-		}, rly.listeners)
-
-		require.Equal(t, []listener{
-			{"a", f3, ws3},
-			{"g", f1, ws1},
-			{"g", f2, ws2},
-		}, rlz.listeners)
 	})
 
 	t.Run("removing a subscription id", func(t *testing.T) {
@@ -280,41 +230,23 @@ func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rlx},
+				{1, "c", cancel},
-				{"g", cancel, 1, rlz},
+				{9, "g", cancel},
 			},
 			ws2: {
-				{"b", cancel, 0, rly},
+				{2, "b", cancel},
-				{"g", cancel, 2, rlz},
+				{10, "g", cancel},
 			},
 			ws3: {
-				{"a", cancel, 0, rlz},
+				{3, "a", cancel},
-				{"a", cancel, 1, rlx},
+				{6, "a", cancel},
-				{"f", cancel, 2, rly},
+				{8, "f", cancel},
 			},
 			ws4: {
-				{"e", cancel, 1, rly},
+				{5, "e", cancel},
-				{"e", cancel, 2, rlx},
+				{7, "e", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"a", f3, ws3},
-			{"e", f3, ws4},
-		}, rlx.listeners)
-
-		require.Equal(t, []listener{
-			{"b", f2, ws2},
-			{"e", f3, ws4},
-			{"f", f3, ws3},
-		}, rly.listeners)
-
-		require.Equal(t, []listener{
-			{"a", f3, ws3},
-			{"g", f1, ws1},
-			{"g", f2, ws2},
-		}, rlz.listeners)
 	})
 
 	t.Run("removing another subscription id", func(t *testing.T) {
@@ -325,37 +257,21 @@ func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rlx},
+				{1, "c", cancel},
-				{"g", cancel, 1, rlz},
+				{9, "g", cancel},
 			},
 			ws2: {
-				{"b", cancel, 0, rly},
+				{2, "b", cancel},
-				{"g", cancel, 0, rlz},
+				{10, "g", cancel},
 			},
 			ws3: {
-				{"f", cancel, 2, rly},
+				{8, "f", cancel},
 			},
 			ws4: {
-				{"e", cancel, 1, rly},
+				{5, "e", cancel},
-				{"e", cancel, 1, rlx},
+				{7, "e", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"e", f3, ws4},
-		}, rlx.listeners)
-
-		require.Equal(t, []listener{
-			{"b", f2, ws2},
-			{"e", f3, ws4},
-			{"f", f3, ws3},
-		}, rly.listeners)
-
-		require.Equal(t, []listener{
-			{"g", f2, ws2},
-			{"g", f1, ws1},
-		}, rlz.listeners)
 	})
 
 	t.Run("removing a connection", func(t *testing.T) {
@@ -363,31 +279,17 @@ func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rlx},
+				{1, "c", cancel},
-				{"g", cancel, 0, rlz},
+				{9, "g", cancel},
 			},
 			ws3: {
-				{"f", cancel, 0, rly},
+				{8, "f", cancel},
 			},
 			ws4: {
-				{"e", cancel, 1, rly},
+				{5, "e", cancel},
-				{"e", cancel, 1, rlx},
+				{7, "e", cancel},
 			},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-			{"e", f3, ws4},
-		}, rlx.listeners)
-
-		require.Equal(t, []listener{
-			{"f", f3, ws3},
-			{"e", f3, ws4},
-		}, rly.listeners)
-
-		require.Equal(t, []listener{
-			{"g", f1, ws1},
-		}, rlz.listeners)
 	})
 
 	t.Run("removing another subscription id", func(t *testing.T) {
@@ -398,26 +300,14 @@ func TestListenerMoreStuffWithMultipleRelays(t *testing.T) {
 
 		require.Equal(t, map[*WebSocket][]listenerSpec{
 			ws1: {
-				{"c", cancel, 0, rlx},
+				{1, "c", cancel},
-				{"g", cancel, 0, rlz},
+				{9, "g", cancel},
 			},
 			ws3: {
-				{"f", cancel, 0, rly},
+				{8, "f", cancel},
 			},
 			ws4: {},
 		}, rl.clients)
-
-		require.Equal(t, []listener{
-			{"c", f1, ws1},
-		}, rlx.listeners)
-
-		require.Equal(t, []listener{
-			{"f", f3, ws3},
-		}, rly.listeners)
-
-		require.Equal(t, []listener{
-			{"g", f1, ws1},
-		}, rlz.listeners)
 	})
 }
 
@@ -432,7 +322,7 @@ func TestRandomListenerClientRemoving(t *testing.T) {
 	l := 0
 
 	for i := 0; i < 20; i++ {
-		ws := &WebSocket{}
+		ws := &WebSocket{Context: rl.ctx}
 		websockets = append(websockets, ws)
 		rl.clients[ws] = nil
 	}
@@ -442,22 +332,30 @@ func TestRandomListenerClientRemoving(t *testing.T) {
 			ws := websockets[i]
 			w := idFromSeqUpper(i)
 
-			if rand.Intn(2) < 1 {
+			if (i+j)%2 == 0 {
 				l++
-				rl.addListener(ws, w+":"+idFromSeqLower(j), rl, f, cancel)
+				rl.addListener(ws, w+":"+idFromSeqLower(j), f, cancel)
 			}
 		}
 	}
 
 	require.Len(t, rl.clients, 20)
-	require.Len(t, rl.listeners, l)
+	ssidCount := 0
+	for _, specs := range rl.clients {
+		ssidCount += len(specs)
+	}
+	require.Equal(t, l, ssidCount)
 
 	for ws := range rl.clients {
 		rl.removeClientAndListeners(ws)
 	}
 
 	require.Len(t, rl.clients, 0)
-	require.Len(t, rl.listeners, 0)
+	ssidCount = 0
+	for _, specs := range rl.clients {
+		ssidCount += len(specs)
+	}
+	require.Equal(t, 0, ssidCount)
 }
 
 func TestRandomListenerIdRemoving(t *testing.T) {
@@ -477,7 +375,7 @@ func TestRandomListenerIdRemoving(t *testing.T) {
 	extra := 0
 
 	for i := 0; i < 20; i++ {
-		ws := &WebSocket{}
+		ws := &WebSocket{Context: rl.ctx}
 		websockets = append(websockets, ws)
 		rl.clients[ws] = nil
 	}
@@ -487,13 +385,13 @@ func TestRandomListenerIdRemoving(t *testing.T) {
 			ws := websockets[i]
 			w := idFromSeqUpper(i)
 
-			if rand.Intn(2) < 1 {
+			if (i+j)%2 == 0 {
 				id := w + ":" + idFromSeqLower(j)
-				rl.addListener(ws, id, rl, f, cancel)
+				rl.addListener(ws, id, f, cancel)
 				subs = append(subs, wsid{ws, id})
 
-				if rand.Intn(5) < 1 {
+				if (i+j)%5 == 0 {
-					rl.addListener(ws, id, rl, f, cancel)
+					rl.addListener(ws, id, f, cancel)
 					extra++
 				}
 			}
@@ -501,16 +399,21 @@ func TestRandomListenerIdRemoving(t *testing.T) {
 	}
 
 	require.Len(t, rl.clients, 20)
-	require.Len(t, rl.listeners, len(subs)+extra)
+	ssidCount := 0
+	for _, specs := range rl.clients {
+		ssidCount += len(specs)
+	}
+	require.Equal(t, len(subs)+extra, ssidCount)
 
-	rand.Shuffle(len(subs), func(i, j int) {
+	for _, wsidToRemove := range moduloOrder(subs, 20) {
-		subs[i], subs[j] = subs[j], subs[i]
-	})
-	for _, wsidToRemove := range subs {
 		rl.removeListenerId(wsidToRemove.ws, wsidToRemove.id)
 	}
 
-	require.Len(t, rl.listeners, 0)
+	ssidCount = 0
+	for _, specs := range rl.clients {
+		ssidCount += len(specs)
+	}
+	require.Equal(t, 0, ssidCount)
 	require.Len(t, rl.clients, 20)
 	for _, specs := range rl.clients {
 		require.Len(t, specs, 0)
@@ -520,12 +423,9 @@ func TestRandomListenerIdRemoving(t *testing.T) {
 func TestRouterListenersPabloCrash(t *testing.T) {
 	rl := NewRelay()
 
-	rla := NewRelay()
+	ws1 := &WebSocket{Context: rl.ctx}
-	rlb := NewRelay()
+	ws2 := &WebSocket{Context: rl.ctx}
-
+	ws3 := &WebSocket{Context: rl.ctx}
-	ws1 := &WebSocket{}
-	ws2 := &WebSocket{}
-	ws3 := &WebSocket{}
 
 	rl.clients[ws1] = nil
 	rl.clients[ws2] = nil
@@ -534,11 +434,11 @@ func TestRouterListenersPabloCrash(t *testing.T) {
 	f := nostr.Filter{Kinds: []nostr.Kind{1}}
 	cancel := func(cause error) {}
 
-	rl.addListener(ws1, ":1", rla, f, cancel)
+	rl.addListener(ws1, ":1", f, cancel)
-	rl.addListener(ws2, ":1", rlb, f, cancel)
+	rl.addListener(ws2, ":1", f, cancel)
-	rl.addListener(ws3, "a", rlb, f, cancel)
+	rl.addListener(ws3, "a", f, cancel)
-	rl.addListener(ws3, "b", rla, f, cancel)
+	rl.addListener(ws3, "b", f, cancel)
-	rl.addListener(ws3, "c", rlb, f, cancel)
+	rl.addListener(ws3, "c", f, cancel)
 
 	rl.removeClientAndListeners(ws1)
 	rl.removeClientAndListeners(ws3)

khatru/nip86.go

@@ -43,6 +43,12 @@ type RelayManagementAPI struct {
 	Stats                       func(ctx context.Context) (nip86.Response, error)
 	GrantAdmin                  func(ctx context.Context, pubkey nostr.PubKey, methods []string) error
 	RevokeAdmin                 func(ctx context.Context, pubkey nostr.PubKey, methods []string) error
+	CreateRole                  func(ctx context.Context, id string, label string, description string, color int, order int) error
+	EditRole                    func(ctx context.Context, id string, label string, description string, color int, order int) error
+	DeleteRole                  func(ctx context.Context, id string) error
+	AssignRole                  func(ctx context.Context, pubkey nostr.PubKey, roleID string) error
+	UnassignRole                func(ctx context.Context, pubkey nostr.PubKey, roleID string) error
+	SignEvent                   func(ctx context.Context, kind nostr.Kind, createdAt nostr.Timestamp, tags nostr.Tags, content string) (nostr.Event, error)
 	Generic                     func(ctx context.Context, request nip86.Request) (nip86.Response, error)
 }
 
@@ -330,6 +336,54 @@ func (rl *Relay) HandleNIP86(w http.ResponseWriter, r *http.Request) {
 			} else {
 				resp.Result = true
 			}
+		case nip86.CreateRole:
+			if rl.ManagementAPI.CreateRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.CreateRole(ctx, thing.ID, thing.Label, thing.Description, thing.Color, thing.Order); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.EditRole:
+			if rl.ManagementAPI.EditRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.EditRole(ctx, thing.ID, thing.Label, thing.Description, thing.Color, thing.Order); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.DeleteRole:
+			if rl.ManagementAPI.DeleteRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.DeleteRole(ctx, thing.ID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.AssignRole:
+			if rl.ManagementAPI.AssignRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.AssignRole(ctx, thing.PubKey, thing.RoleID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.UnassignRole:
+			if rl.ManagementAPI.UnassignRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.UnassignRole(ctx, thing.PubKey, thing.RoleID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.SignEvent:
+			if rl.ManagementAPI.SignEvent == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if result, err := rl.ManagementAPI.SignEvent(ctx, thing.Kind, thing.CreatedAt, thing.Tags, thing.Content); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = result
+			}
 		case nip86.ListDisallowedKinds:
 			if rl.ManagementAPI.ListDisallowedKinds == nil {
 				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())

khatru/policies/events.go

@@ -3,6 +3,7 @@ package policies
 import (
 	"context"
 	"fmt"
+	"iter"
 	"regexp"
 	"slices"
 	"strings"
@@ -110,6 +111,9 @@ func RejectEventsWithBase64Media(ctx context.Context, evt nostr.Event) (bool, st
 }
 
 func OnlyAllowNIP70ProtectedEvents(ctx context.Context, event nostr.Event) (reject bool, msg string) {
+	if event.Kind == 5 {
+		return false, ""
+	}
 	if nip70.IsProtected(event) {
 		return false, ""
 	}
@@ -120,6 +124,9 @@ var nostrReferencesPrefix = regexp.MustCompile(`\b(nevent1|npub1|nprofile1|note1
 
 func RejectUnprefixedNostrReferences(ctx context.Context, event nostr.Event) (bool, string) {
 	content := sdk.GetMainContent(event)
+	if content == "" {
+		content = event.Content
+	}
 
 	// only do it for stuff that wasn't parsed as blocks already
 	// (since those are already good references or URLs)
@@ -144,3 +151,55 @@ func RejectUnprefixedNostrReferences(ctx context.Context, event nostr.Event) (bo
 
 	return false, ""
 }
+
+// PreventNormalDuplicates prevents normal events that refer to the same thing from being saved.
+// For kinds 6, 7, 16, 1018 it checks "e" tags.
+// For kind 1163 it checks "p" tags.
+// For kinds 1163, 6, 16, 7516, 7517 it checks "a" tags.
+func PreventNormalDuplicates(query func(nostr.Filter, int) iter.Seq[nostr.Event]) func(ctx context.Context, event nostr.Event) (bool, string) {
+	exists := func(event nostr.Event, tagName string) bool {
+		hasAll := true
+		for t := range event.Tags.FindAll(tagName) {
+			hasThis := false
+			for range query(nostr.Filter{
+				Authors: []nostr.PubKey{event.PubKey},
+				Kinds:   []nostr.Kind{event.Kind},
+				Tags:    nostr.TagMap{tagName: []string{t[1]}},
+			}, 1) {
+				hasThis = true
+			}
+			if !hasThis {
+				hasAll = false
+				break
+			}
+		}
+		return hasAll
+	}
+
+	return func(ctx context.Context, event nostr.Event) (bool, string) {
+		reject := false
+
+		switch event.Kind {
+		case 6:
+			reject = exists(event, "e") && exists(event, "a")
+		case 7:
+			reject = exists(event, "e") && exists(event, "a")
+		case 16:
+			reject = exists(event, "e") && exists(event, "a")
+		case 1018:
+			reject = exists(event, "e")
+		case 1163:
+			reject = exists(event, "p") && exists(event, "a")
+		case 7516:
+			reject = exists(event, "a")
+		case 7517:
+			reject = exists(event, "a")
+		}
+
+		if reject {
+			return true, "an event similar to this already exists"
+		}
+
+		return false, ""
+	}
+}

khatru/relay.go

@@ -2,15 +2,18 @@ package khatru
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/binary"
 	"iter"
 	"log"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
+	"unsafe"
 
+	"fiatjaf.com/lib/channelmutex"
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore"
 	"fiatjaf.com/nostr/nip11"
@@ -40,7 +43,9 @@ func NewRelay() *Relay {
 		},
 
 		clients:      make(map[*WebSocket][]listenerSpec, 100),
-		listeners: make([]listener, 0, 100),
+		clientsMutex: channelmutex.New(),
+
+		dispatcher: newDispatcher(),
 
 		serveMux: &http.ServeMux{},
 
@@ -69,6 +74,8 @@ type Relay struct {
 	ReplaceEvent              func(ctx context.Context, event nostr.Event) error
 	DeleteEvent               func(ctx context.Context, id nostr.ID) error
 	OnEventSaved              func(ctx context.Context, event nostr.Event)
+	OnEventDeleted            func(ctx context.Context, deleted nostr.Event)
+	AllowDeleting             func(ctx context.Context, target, deletion nostr.Event) bool
 	OnEphemeralEvent          func(ctx context.Context, event nostr.Event)
 	OnRequest                 func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
 	OnCount                   func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
@@ -78,17 +85,15 @@ type Relay struct {
 	RejectConnection          func(r *http.Request) bool
 	OnConnect                 func(ctx context.Context)
 	OnDisconnect              func(ctx context.Context)
+	OnAuth                    func(ctx context.Context, pubkey nostr.PubKey)
+	OnListenerAdded           func(ws *WebSocket, ssid int, id string, filter nostr.Filter)
+	OnListenerRemoved         func(ws *WebSocket, ssid int, id string, filter nostr.Filter)
 	OverwriteRelayInformation func(ctx context.Context, r *http.Request, info nip11.RelayInformationDocument) nip11.RelayInformationDocument
 	PreventBroadcast          func(ws *WebSocket, filter nostr.Filter, event nostr.Event) bool
 
 	// this can be ignored unless you know what you're doing
 	ChallengePrefix string
 
-	// these are used when this relays acts as a router
-	routes                []Route
-	getSubRelayFromEvent  func(*nostr.Event) *Relay // used for handling EVENTs
-	getSubRelayFromFilter func(nostr.Filter) *Relay // used for handling REQs
-
 	// setting up handlers here will enable these methods
 	ManagementAPI RelayManagementAPI
 
@@ -105,8 +110,8 @@ type Relay struct {
 	// keep a connection reference to all connected clients for Server.Shutdown
 	// also used for keeping track of who is listening to what
 	clients      map[*WebSocket][]listenerSpec
-	listeners    []listener
+	dispatcher   dispatcher
-	clientsMutex sync.Mutex
+	clientsMutex *channelmutex.Mutex
 
 	// set this to true to support negentropy
 	Negentropy bool
@@ -148,19 +153,28 @@ func (rl *Relay) UseEventstore(store eventstore.Store, maxQueryLimit int) {
 		return store.SaveEvent(event)
 	}
 	rl.ReplaceEvent = func(ctx context.Context, event nostr.Event) error {
-		return store.ReplaceEvent(event)
+		_, err := store.ReplaceEvent(event)
+		return err
 	}
 	rl.DeleteEvent = func(ctx context.Context, id nostr.ID) error {
 		return store.DeleteEvent(id)
 	}
 
 	// only when using the eventstore we automatically set up the expiration manager
-	rl.StartExpirationManager(rl.QueryStored, rl.DeleteEvent)
+	rl.StartExpirationManager(func(ctx context.Context, filter nostr.Filter) iter.Seq[nostr.Event] {
+		return rl.QueryStored(ctx, filter)
+	}, func(ctx context.Context, id nostr.ID) error {
+		return rl.DeleteEvent(ctx, id)
+	}, func(ctx context.Context, evt nostr.Event) {
+		if rl.OnEventDeleted != nil {
+			rl.OnEventDeleted(ctx, evt)
+		}
+	})
 }
 
 func (rl *Relay) getBaseURL(r *http.Request) string {
-	if rl.ServiceURL != "" {
+	if serviceURL := rl.getServiceURL(r); serviceURL != "" {
-		return rl.ServiceURL
+		return serviceURL
 	}
 
 	host := r.Header.Get("X-Forwarded-Host")
@@ -184,3 +198,121 @@ func (rl *Relay) getBaseURL(r *http.Request) string {
 
 	return proto + "://" + host + r.URL.Path
 }
+
+func (rl *Relay) getServiceURL(r *http.Request) string {
+	if serviceURL, ok := r.Context().Value(serviceURLOverrideKey).(string); ok {
+		return serviceURL
+	}
+
+	return rl.ServiceURL
+}
+
+// Stats returns the current number of connected clients and open listeners.
+func (rl *Relay) Stats() (clients, listeners int) {
+	rl.clientsMutex.Lock()
+	defer rl.clientsMutex.Unlock()
+
+	for _, specs := range rl.clients {
+		listeners += len(specs)
+	}
+
+	return len(rl.clients), listeners
+}
+
+type ClientInfo struct {
+	ID                string
+	IP                string
+	UserAgent         string
+	Origin            string
+	Authenticated     []nostr.PubKey
+	SubscriptionCount int
+}
+
+type SubscriptionInfo struct {
+	ID     string
+	Filter nostr.Filter
+}
+
+type ClientSnapshot struct {
+	ClientInfo
+	Subscriptions []SubscriptionInfo
+}
+
+func (rl *Relay) ListClients() []ClientInfo {
+	rl.clientsMutex.Lock()
+	defer rl.clientsMutex.Unlock()
+
+	clients := make([]ClientInfo, 0, len(rl.clients))
+	for ws, specs := range rl.clients {
+		clients = append(clients, ClientInfo{
+			ID:                ws.GetID(),
+			IP:                GetIPFromRequest(ws.Request),
+			UserAgent:         ws.Request.UserAgent(),
+			Origin:            ws.Request.Header.Get("Origin"),
+			Authenticated:     ws.AuthedPublicKeys,
+			SubscriptionCount: len(specs),
+		})
+	}
+
+	return clients
+}
+
+func (rl *Relay) GetClientSnapshot(id string) (ClientSnapshot, bool) {
+	rl.clientsMutex.Lock()
+	defer rl.clientsMutex.Unlock()
+
+	ptrn, err := base64.RawURLEncoding.DecodeString(id)
+	if err != nil {
+		return ClientSnapshot{}, false
+	}
+	ptr := binary.LittleEndian.Uint64(ptrn)
+
+	// DANGEROUS:
+	// don't try to do anything with this `ws` object before we confirm it exists by checking the rl.clients map
+	ws := (*WebSocket)(unsafe.Pointer(uintptr(ptr)))
+	specs, ok := rl.clients[ws]
+	if !ok {
+		return ClientSnapshot{}, false
+	}
+
+	details := ClientSnapshot{
+		ClientInfo: ClientInfo{
+			ID:                id,
+			IP:                GetIPFromRequest(ws.Request),
+			UserAgent:         ws.Request.UserAgent(),
+			Origin:            ws.Request.Header.Get("Origin"),
+			Authenticated:     ws.AuthedPublicKeys,
+			SubscriptionCount: len(specs),
+		},
+		Subscriptions: make([]SubscriptionInfo, 0, len(specs)),
+	}
+
+	for _, spec := range specs {
+		filter := nostr.Filter{}
+		if sub, ok := rl.dispatcher.subscriptions.Load(spec.ssid); ok {
+			filter = sub.filter
+		}
+
+		details.Subscriptions = append(details.Subscriptions, SubscriptionInfo{
+			ID:     spec.sid,
+			Filter: filter,
+		})
+	}
+
+	return details, true
+}
+
+func (rl *Relay) Router() *http.ServeMux {
+	return rl.serveMux
+}
+
+func (rl *Relay) SetRouter(mux *http.ServeMux) {
+	rl.serveMux = mux
+}
+
+func (rl *Relay) WithServiceURL(serviceURL string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := context.WithValue(r.Context(), serviceURLOverrideKey, serviceURL)
+		rl.ServeHTTP(w, r.WithContext(ctx))
+	})
+}

khatru/relay_fuzz_test.go

@@ -3,7 +3,6 @@ package khatru
 import (
 	"context"
 	"math"
-	"math/rand/v2"
 	"net/http/httptest"
 	"testing"
 	"time"
@@ -14,13 +13,15 @@ import (
 )
 
 func FuzzReplaceableEvents(f *testing.F) {
-	f.Add(uint(1), uint(2))
+	f.Add(1, 1, uint(2))
 
-	f.Fuzz(func(t *testing.T, seed uint, nevents uint) {
+	f.Fuzz(func(t *testing.T, seed int, advance int, nevents uint) {
 		if nevents == 0 {
 			return
 		}
 
+		state := fuzzState{value: seed, advance: advance}
+
 		relay := NewRelay()
 		store := &lmdb.LMDBBackend{Path: "/tmp/fuzz"}
 		store.Init()
@@ -67,12 +68,10 @@ func FuzzReplaceableEvents(f *testing.F) {
 			ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
 			defer cancel()
 
-			rnd := rand.New(rand.NewPCG(uint64(seed), 0))
-
 			newest := nostr.Timestamp(0)
 			for range nevents {
 				evt := createEvent(sk1, 0, `{"name":"blblbl"}`, nil)
-				evt.CreatedAt = nostr.Timestamp(rnd.Int64() % math.MaxUint32)
+				evt.CreatedAt = nostr.Timestamp(state.next(math.MaxUint32))
 				evt.Sign(sk1)
 				err = client1.Publish(ctx, evt)
 				if err != nil {

khatru/relay_test.go

@@ -2,6 +2,9 @@ package khatru
 
 import (
 	"context"
+	"encoding/json"
+	"io"
+	"net/http"
 	"net/http/httptest"
 	"strconv"
 	"testing"
@@ -9,8 +12,60 @@ import (
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore/slicestore"
+	"fiatjaf.com/nostr/nip11"
+	"github.com/stretchr/testify/require"
 )
 
+func TestWithServiceURL(t *testing.T) {
+	relay := NewRelay()
+	relay.Info.Icon = "icon.png"
+	relay.Info.Banner = "banner.png"
+	relay.Router().HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusAccepted)
+		io.WriteString(w, "fallback")
+	})
+
+	handlerA := relay.WithServiceURL("https://a.example/relay")
+	handlerB := relay.WithServiceURL("https://b.example/relay")
+
+	t.Run("uses override for nip11 base url", func(t *testing.T) {
+		for _, tc := range []struct {
+			name         string
+			handler      http.Handler
+			expectedBase string
+		}{
+			{name: "first interface", handler: handlerA, expectedBase: "https://a.example/relay"},
+			{name: "second interface", handler: handlerB, expectedBase: "https://b.example/relay"},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				req := httptest.NewRequest(http.MethodGet, "http://internal/relay", nil)
+				req.Header.Set("Accept", "application/nostr+json")
+				rr := httptest.NewRecorder()
+
+				tc.handler.ServeHTTP(rr, req)
+
+				require.Equal(t, http.StatusOK, rr.Code)
+
+				var info nip11.RelayInformationDocument
+				require.NoError(t, json.NewDecoder(rr.Body).Decode(&info))
+				require.Equal(t, tc.expectedBase+"/icon.png", info.Icon)
+				require.Equal(t, tc.expectedBase+"/banner.png", info.Banner)
+			})
+		}
+	})
+
+	t.Run("uses override for relay path matching", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "http://internal/not-relay", nil)
+		req.Header.Set("Accept", "application/nostr+json")
+		rr := httptest.NewRecorder()
+
+		handlerA.ServeHTTP(rr, req)
+
+		require.Equal(t, http.StatusAccepted, rr.Code)
+		require.Equal(t, "fallback", rr.Body.String())
+	})
+}
+
 func TestBasicRelayFunctionality(t *testing.T) {
 	// setup relay with in-memory store
 	relay := NewRelay()
@@ -46,15 +101,11 @@ func TestBasicRelayFunctionality(t *testing.T) {
 	// connect two test clients
 	url := "ws" + server.URL[4:]
 	client1, err := nostr.RelayConnect(t.Context(), url, nostr.RelayOptions{})
-	if err != nil {
+	require.NoError(t, err, "failed to connect client1")
-		t.Fatalf("failed to connect client1: %v", err)
-	}
 	defer client1.Close()
 
 	client2, err := nostr.RelayConnect(t.Context(), url, nostr.RelayOptions{})
-	if err != nil {
+	require.NoError(t, err, "failed to connect client2")
-		t.Fatalf("failed to connect client2: %v", err)
-	}
 	defer client2.Close()
 
 	// test 1: store and query events
@@ -64,18 +115,14 @@ func TestBasicRelayFunctionality(t *testing.T) {
 
 		evt1 := createEvent(sk1, 1, "hello world", nil)
 		err := client1.Publish(ctx, evt1)
-		if err != nil {
+		require.NoError(t, err, "failed to publish event")
-			t.Fatalf("failed to publish event: %v", err)
-		}
 
 		// Query the event back
 		sub, err := client2.Subscribe(ctx, nostr.Filter{
 			Authors: []nostr.PubKey{pk1},
 			Kinds:   []nostr.Kind{1},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		// Wait for event
@@ -85,7 +132,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				t.Errorf("got wrong event: %v", env.ID)
 			}
 		case <-ctx.Done():
-			t.Fatal("timeout waiting for event")
+			require.FailNow(t, "timeout waiting for event")
 		}
 	})
 
@@ -99,17 +146,13 @@ func TestBasicRelayFunctionality(t *testing.T) {
 			Authors: []nostr.PubKey{pk2},
 			Kinds:   []nostr.Kind{1},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		// Publish event from client2
 		evt2 := createEvent(sk2, 1, "testing live events", nil)
 		err = client2.Publish(ctx, evt2)
-		if err != nil {
+		require.NoError(t, err, "failed to publish event")
-			t.Fatalf("failed to publish event: %v", err)
-		}
 
 		// Wait for event on subscription
 		select {
@@ -118,7 +161,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				t.Errorf("got wrong event: %v", env.ID)
 			}
 		case <-ctx.Done():
-			t.Fatal("timeout waiting for live event")
+			require.FailNow(t, "timeout waiting for live event")
 		}
 	})
 
@@ -130,24 +173,18 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		// Create an event to be deleted
 		evt3 := createEvent(sk1, 1, "delete me", nil)
 		err = client1.Publish(ctx, evt3)
-		if err != nil {
+		require.NoError(t, err, "failed to publish event")
-			t.Fatalf("failed to publish event: %v", err)
-		}
 
 		// Create deletion event
 		delEvent := createEvent(sk1, 5, "deleting", nostr.Tags{{"e", evt3.ID.Hex()}})
 		err = client1.Publish(ctx, delEvent)
-		if err != nil {
+		require.NoError(t, err, "failed to publish deletion event")
-			t.Fatalf("failed to publish deletion event: %v", err)
-		}
 
 		// Try to query the deleted event
 		sub, err := client2.Subscribe(ctx, nostr.Filter{
 			IDs: []nostr.ID{evt3.ID},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		// Should get EOSE without receiving the deleted event
@@ -162,7 +199,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				}
 				goto checkDeleteStored
 			case <-ctx.Done():
-				t.Fatal("timeout waiting for EOSE")
+				require.FailNow(t, "timeout waiting for EOSE")
 			}
 		}
 
@@ -171,9 +208,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		subDelete, err := client2.Subscribe(ctx, nostr.Filter{
 			IDs: []nostr.ID{delEvent.ID},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe to delete event")
-			t.Fatalf("failed to subscribe to delete event: %v", err)
-		}
 		defer subDelete.Unsub()
 
 		gotDeleteEvent := false
@@ -189,7 +224,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				}
 				return
 			case <-ctx.Done():
-				t.Fatal("timeout waiting for EOSE on delete event")
+				require.FailNow(t, "timeout waiting for EOSE on delete event")
 			}
 		}
 	})
@@ -204,36 +239,28 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		evt1.CreatedAt = 1000 // Set specific timestamp for testing
 		evt1.Sign(sk1)
 		err = client1.Publish(ctx, evt1)
-		if err != nil {
+		require.NoError(t, err, "failed to publish initial event")
-			t.Fatalf("failed to publish initial event: %v", err)
-		}
 
 		// create newer event that should replace the first
 		evt2 := createEvent(sk1, 0, `{"name":"newer"}`, nil)
 		evt2.CreatedAt = 2004 // Newer timestamp
 		evt2.Sign(sk1)
 		err = client1.Publish(ctx, evt2)
-		if err != nil {
+		require.NoError(t, err, "failed to publish newer event")
-			t.Fatalf("failed to publish newer event: %v", err)
-		}
 
 		// create older event that should not replace the current one
 		evt3 := createEvent(sk1, 0, `{"name":"older"}`, nil)
 		evt3.CreatedAt = 1500 // Older than evt2
 		evt3.Sign(sk1)
 		err = client1.Publish(ctx, evt3)
-		if err != nil {
+		require.NoError(t, err, "failed to publish older event")
-			t.Fatalf("failed to publish older event: %v", err)
-		}
 
 		// query to verify only the newest event exists
 		sub, err := client2.Subscribe(ctx, nostr.Filter{
 			Authors: []nostr.PubKey{pk1},
 			Kinds:   []nostr.Kind{0},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		// should only get one event back (the newest one)
@@ -251,7 +278,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				}
 				return
 			case <-ctx.Done():
-				t.Fatal("timeout waiting for events")
+				require.FailNow(t, "timeout waiting for events")
 			}
 		}
 	})
@@ -281,26 +308,20 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		// connect test client
 		url := "ws" + server.URL[4:]
 		client, err := nostr.RelayConnect(t.Context(), url, nostr.RelayOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to connect client")
-			t.Fatalf("failed to connect client: %v", err)
-		}
 		defer client.Close()
 
 		// create event that expires in 2 seconds
 		expiration := strconv.FormatInt(int64(nostr.Now()+2), 10)
 		evt := createEvent(sk1, 1, "i will expire soon", nostr.Tags{{"expiration", expiration}})
 		err = client.Publish(ctx, evt)
-		if err != nil {
+		require.NoError(t, err, "failed to publish event")
-			t.Fatalf("failed to publish event: %v", err)
-		}
 
 		// verify event exists initially
 		sub, err := client.Subscribe(ctx, nostr.Filter{
 			IDs: []nostr.ID{evt.ID},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 
 		// should get the event
 		select {
@@ -309,7 +330,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				t.Error("got wrong event")
 			}
 		case <-ctx.Done():
-			t.Fatal("timeout waiting for event")
+			require.FailNow(t, "timeout waiting for event")
 		}
 		sub.Unsub()
 
@@ -320,9 +341,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		sub, err = client.Subscribe(ctx, nostr.Filter{
 			IDs: []nostr.ID{evt.ID},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err, "failed to subscribe")
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		// should get EOSE without receiving the expired event
@@ -337,7 +356,7 @@ func TestBasicRelayFunctionality(t *testing.T) {
 				}
 				return
 			case <-ctx.Done():
-				t.Fatal("timeout waiting for EOSE")
+				require.FailNow(t, "timeout waiting for EOSE")
 			}
 		}
 	})
@@ -350,33 +369,26 @@ func TestBasicRelayFunctionality(t *testing.T) {
 		// create an event from client1
 		evt4 := createEvent(sk1, 1, "try to delete me", nil)
 		err = client1.Publish(ctx, evt4)
-		if err != nil {
+		require.NoError(t, err)
-			t.Fatalf("failed to publish event: %v", err)
-		}
 
-		// Try to delete it with client2
+		// try to delete it with client2
 		delEvent := createEvent(sk2, 5, "trying to delete", nostr.Tags{{"e", evt4.ID.Hex()}})
 		err = client2.Publish(ctx, delEvent)
-		if err == nil {
+		require.Error(t, err)
-			t.Fatalf("should have failed to publish deletion event: %v", err)
-		}
 
-		// Verify event still exists
+		// verify event still exists
 		sub, err := client1.Subscribe(ctx, nostr.Filter{
 			IDs: []nostr.ID{evt4.ID},
 		}, nostr.SubscriptionOptions{})
-		if err != nil {
+		require.NoError(t, err)
-			t.Fatalf("failed to subscribe: %v", err)
-		}
 		defer sub.Unsub()
 
 		select {
-		case env := <-sub.Events:
+		case env, more := <-sub.Events:
-			if env.ID != evt4.ID {
+			require.True(t, more, "should get an event, got nothing")
-				t.Error("got wrong event")
+			require.Equal(t, env.ID, evt4.ID, "got wrong event")
-			}
 		case <-ctx.Done():
-			t.Fatal("event should still exist")
+			require.FailNow(t, "event should still exist")
 		}
 	})
 }

khatru/responding.go

@@ -3,15 +3,12 @@ package khatru
 import (
 	"context"
 	"errors"
-	"sync"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/nip45/hyperloglog"
 )
 
-func (rl *Relay) handleRequest(ctx context.Context, id string, eose *sync.WaitGroup, ws *WebSocket, filter nostr.Filter) error {
+func (rl *Relay) handleRequest(ctx context.Context, id string, ws *WebSocket, filter nostr.Filter) error {
-	defer eose.Done()
-
 	// then check if we'll reject this filter (we apply this after overwriting
 	// because we may, for example, remove some things from the incoming filters
 	// that we know we don't support, and then if the end result is an empty
@@ -30,7 +27,9 @@ func (rl *Relay) handleRequest(ctx context.Context, id string, eose *sync.WaitGr
 	// run the function to query events
 	if nil != rl.QueryStored {
 		for event := range rl.QueryStored(ctx, filter) {
-			ws.WriteJSON(nostr.EventEnvelope{SubscriptionID: &id, Event: event})
+			if nil != ws.WriteJSON(nostr.EventEnvelope{SubscriptionID: &id, Event: event}) {
+				break
+			}
 		}
 	}
 

khatru/router.go

@@ -1,77 +0,0 @@
-package khatru
-
-import (
-	"fiatjaf.com/nostr"
-)
-
-type Router struct{ *Relay }
-
-type Route struct {
-	eventMatcher  func(*nostr.Event) bool
-	filterMatcher func(nostr.Filter) bool
-	relay         *Relay
-}
-
-type routeBuilder struct {
-	router        *Router
-	eventMatcher  func(*nostr.Event) bool
-	filterMatcher func(nostr.Filter) bool
-}
-
-func NewRouter() *Router {
-	rr := &Router{Relay: NewRelay()}
-	rr.routes = make([]Route, 0, 3)
-	rr.getSubRelayFromFilter = func(f nostr.Filter) *Relay {
-		for _, route := range rr.routes {
-			if route.filterMatcher == nil || route.filterMatcher(f) {
-				return route.relay
-			}
-		}
-		return rr.Relay
-	}
-	rr.getSubRelayFromEvent = func(e *nostr.Event) *Relay {
-		for _, route := range rr.routes {
-			if route.eventMatcher == nil || route.eventMatcher(e) {
-				return route.relay
-			}
-		}
-		return rr.Relay
-	}
-	return rr
-}
-
-func (rr *Router) Route() routeBuilder {
-	return routeBuilder{
-		router:        rr,
-		filterMatcher: func(f nostr.Filter) bool { return false },
-		eventMatcher:  func(e *nostr.Event) bool { return false },
-	}
-}
-
-func (rb routeBuilder) Req(fn func(nostr.Filter) bool) routeBuilder {
-	rb.filterMatcher = fn
-	return rb
-}
-
-func (rb routeBuilder) AnyReq() routeBuilder {
-	rb.filterMatcher = nil
-	return rb
-}
-
-func (rb routeBuilder) Event(fn func(*nostr.Event) bool) routeBuilder {
-	rb.eventMatcher = fn
-	return rb
-}
-
-func (rb routeBuilder) AnyEvent() routeBuilder {
-	rb.eventMatcher = nil
-	return rb
-}
-
-func (rb routeBuilder) Relay(relay *Relay) {
-	rb.router.routes = append(rb.router.routes, Route{
-		filterMatcher: rb.filterMatcher,
-		eventMatcher:  rb.eventMatcher,
-		relay:         relay,
-	})
-}

khatru/testdata/fuzz/FuzzDispatcherCandidates/030fbe034592967a

@@ -0,0 +1,5 @@
+go test fuzz v1
+int(-180)
+int(92)
+byte('{')
+byte('\n')

khatru/testdata/fuzz/FuzzDispatcherCandidates/0be355dc7fe62f98

@@ -0,0 +1,5 @@
+go test fuzz v1
+int(140)
+int(-52)
+byte('"')
+byte('h')

khatru/testdata/fuzz/FuzzReplaceableEvents/00ff79377dab077d

@@ -1,3 +1,4 @@
 go test fuzz v1
-uint(25)
+int(25)
+int(1)
 uint(223)

khatru/utils.go

@@ -11,6 +11,7 @@ const (
 	subscriptionIdKey
 	nip86HeaderAuthKey
 	internalCallKey
+	serviceURLOverrideKey
 )
 
 func RequestAuth(ctx context.Context) {
@@ -73,6 +74,13 @@ func IsAuthed(ctx context.Context, pubkey nostr.PubKey) bool {
 	return false
 }
 
+// ForceSetAuthed modifies the context to insert a custom authed public key.
+// It can be used in testing or other rare scenarios for making requests as if a given public key
+// was authenticated when in fact it didn't perform any of the authentication rituals.
+func ForceSetAuthed(ctx context.Context, pubkey nostr.PubKey) context.Context {
+	return context.WithValue(ctx, nip86HeaderAuthKey, pubkey)
+}
+
 // IsInternalCall returns true when a call to QueryEvents, for example, is being made because of a deletion
 // or expiration request.
 func IsInternalCall(ctx context.Context) bool {

khatru/websocket.go

@@ -2,8 +2,12 @@ package khatru
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
 	"net/http"
 	"sync"
+	"unsafe"
 
 	"fiatjaf.com/nostr"
 	"github.com/fasthttp/websocket"
@@ -30,7 +34,17 @@ type WebSocket struct {
 	negentropySessions *xsync.MapOf[string, *NegentropySession]
 }
 
+func (ws *WebSocket) GetID() string {
+	ptr := uintptr(unsafe.Pointer(ws))
+	var id [8]byte
+	binary.LittleEndian.PutUint64(id[:], uint64(ptr))
+	return base64.RawURLEncoding.EncodeToString(id[:])
+}
+
 func (ws *WebSocket) WriteJSON(any any) error {
+	if ws == nil {
+		return fmt.Errorf("connection doesn't exist")
+	}
 	ws.mutex.Lock()
 	err := ws.conn.WriteJSON(any)
 	ws.mutex.Unlock()

kinds.go

@@ -38,10 +38,20 @@ func (kind Kind) Name() string {
 		return "Seal"
 	case KindDirectMessage:
 		return "DirectMessage"
+	case KindFileMessage:
+		return "FileMessage"
 	case KindGenericRepost:
 		return "GenericRepost"
 	case KindReactionToWebsite:
 		return "ReactionToWebsite"
+	case KindPhoto:
+		return "Photo"
+	case KindNormalVideoEvent:
+		return "NormalVideoEvent"
+	case KindShortVideoEvent:
+		return "ShortVideoEvent"
+	case KindPublicMessage:
+		return "PublicMessage"
 	case KindChannelCreation:
 		return "ChannelCreation"
 	case KindChannelMetadata:
@@ -52,12 +62,14 @@ func (kind Kind) Name() string {
 		return "ChannelHideMessage"
 	case KindChannelMuteUser:
 		return "ChannelMuteUser"
+	case KindPodcastEpisode:
+		return "PodcastEpisode"
 	case KindChess:
 		return "Chess"
 	case KindMergeRequests:
 		return "MergeRequests"
-	case KindComment:
+	case KindPollResponse:
-		return "Comment"
+		return "PollResponse"
 	case KindBid:
 		return "Bid"
 	case KindBidConfirmation:
@@ -68,10 +80,26 @@ func (kind Kind) Name() string {
 		return "GiftWrap"
 	case KindFileMetadata:
 		return "FileMetadata"
+	case KindPoll:
+		return "Poll"
+	case KindComment:
+		return "Comment"
+	case KindVoiceMessage:
+		return "VoiceMessage"
+	case KindScroll:
+		return "Scroll"
+	case KindVoiceMessageComment:
+		return "VoiceMessageComment"
 	case KindLiveChatMessage:
 		return "LiveChatMessage"
+	case KindCodeSnippet:
+		return "CodeSnippet"
 	case KindPatch:
 		return "Patch"
+	case KindGitPullRequest:
+		return "GitPullRequest"
+	case KindGitPullRequestUpdate:
+		return "GitPullRequestUpdate"
 	case KindIssue:
 		return "Issue"
 	case KindReply:
@@ -100,10 +128,24 @@ func (kind Kind) Name() string {
 		return "TorrentComment"
 	case KindCoinjoinPool:
 		return "CoinjoinPool"
+	case KindDecoupledKeyClientAnnouncement:
+		return "DecoupledKeyClientAnnouncement"
+	case KindDecoupledEncryptionKeyDistribution:
+		return "DecoupledEncryptionKeyDistribution"
 	case KindCommunityPostApproval:
 		return "CommunityPostApproval"
 	case KindJobFeedback:
 		return "JobFeedback"
+	case KindReservedCashuWalletTokens:
+		return "ReservedCashuWalletTokens"
+	case KindCashuWalletTokens:
+		return "CashuWalletTokens"
+	case KindCashuWalletHistory:
+		return "CashuWalletHistory"
+	case KindGeocacheLog:
+		return "GeocacheLog"
+	case KindGeocacheProofOfFind:
+		return "GeocacheProofOfFind"
 	case KindSimpleGroupPutUser:
 		return "SimpleGroupPutUser"
 	case KindSimpleGroupRemoveUser:
@@ -152,14 +194,24 @@ func (kind Kind) Name() string {
 		return "SearchRelayList"
 	case KindSimpleGroupList:
 		return "SimpleGroupList"
+	case KindFavoriteRelaysList:
+		return "FavoriteRelaysList"
+	case KindPrivateEventRelayList:
+		return "PrivateEventRelayList"
 	case KindInterestList:
 		return "InterestList"
 	case KindNutZapInfo:
 		return "NutZapInfo"
+	case KindMediaFollows:
+		return "MediaFollows"
 	case KindEmojiList:
 		return "EmojiList"
+	case KindDecoupledKeyAnnouncement:
+		return "DecoupledKeyAnnouncement"
 	case KindDMRelayList:
 		return "DMRelayList"
+	case KindFavoritePodcasts:
+		return "FavoritePodcasts"
 	case KindUserServerList:
 		return "UserServerList"
 	case KindFileStorageServerList:
@@ -168,8 +220,26 @@ func (kind Kind) Name() string {
 		return "GoodWikiAuthorList"
 	case KindGoodWikiRelayList:
 		return "GoodWikiRelayList"
+	case KindPodcastMetadata:
+		return "PodcastMetadata"
+	case KindAuthoredPodcasts:
+		return "AuthoredPodcasts"
+	case KindRelayMonitorAnnouncement:
+		return "RelayMonitorAnnouncement"
+	case KindRoomPresence:
+		return "RoomPresence"
+	case KindUserGraspList:
+		return "UserGraspList"
+	case KindProxyAnnouncement:
+		return "ProxyAnnouncement"
+	case KindTransportMethodAnnouncement:
+		return "TransportMethodAnnouncement"
 	case KindNWCWalletInfo:
 		return "NWCWalletInfo"
+	case KindNsiteRoot:
+		return "NsiteRoot"
+	case KindCashuWalletEvent:
+		return "CashuWalletEvent"
 	case KindLightningPubRPC:
 		return "LightningPubRPC"
 	case KindClientAuthentication:
@@ -226,10 +296,20 @@ func (kind Kind) Name() string {
 		return "ReleaseArtifactSets"
 	case KindApplicationSpecificData:
 		return "ApplicationSpecificData"
+	case KindRelayDiscovery:
+		return "RelayDiscovery"
+	case KindAppCurationSet:
+		return "AppCurationSet"
 	case KindLiveEvent:
 		return "LiveEvent"
+	case KindInteractiveRoom:
+		return "InteractiveRoom"
+	case KindConferenceEvent:
+		return "ConferenceEvent"
 	case KindUserStatuses:
 		return "UserStatuses"
+	case KindSlideSet:
+		return "SlideSet"
 	case KindClassifiedListing:
 		return "ClassifiedListing"
 	case KindDraftClassifiedListing:
@@ -238,20 +318,14 @@ func (kind Kind) Name() string {
 		return "RepositoryAnnouncement"
 	case KindRepositoryState:
 		return "RepositoryState"
-	case KindSimpleGroupMetadata:
-		return "SimpleGroupMetadata"
-	case KindSimpleGroupAdmins:
-		return "SimpleGroupAdmins"
-	case KindSimpleGroupMembers:
-		return "SimpleGroupMembers"
-	case KindSimpleGroupRoles:
-		return "SimpleGroupRoles"
-	case KindSimpleGroupLiveKitParticipants:
-		return "SimpleGroupLiveKitParticipants"
 	case KindWikiArticle:
 		return "WikiArticle"
 	case KindRedirects:
 		return "Redirects"
+	case KindDraftEvent:
+		return "DraftEvent"
+	case KindLinkSet:
+		return "LinkSet"
 	case KindFeed:
 		return "Feed"
 	case KindDateCalendarEvent:
@@ -266,14 +340,43 @@ func (kind Kind) Name() string {
 		return "HandlerRecommendation"
 	case KindHandlerInformation:
 		return "HandlerInformation"
-	case KindVideoEvent:
+	case KindSoftwareApplication:
-		return "VideoEvent"
+		return "SoftwareApplication"
-	case KindShortVideoEvent:
+	case KindLegacyNsiteFile:
-		return "ShortVideoEvent"
+		return "LegacyNsiteFile"
 	case KindVideoViewEvent:
 		return "VideoViewEvent"
 	case KindCommunityDefinition:
 		return "CommunityDefinition"
+	case KindNsiteNamed:
+		return "NsiteNamed"
+	case KindGeocacheListing:
+		return "GeocacheListing"
+	case KindGeocacheLogEntry:
+		return "GeocacheLogEntry"
+	case KindCashuMintAnnouncement:
+		return "CashuMintAnnouncement"
+	case KindFedimintAnnouncement:
+		return "FedimintAnnouncement"
+	case KindPeerToPeerOrderEvents:
+		return "PeerToPeerOrderEvents"
+	case KindSimpleGroupMetadata:
+		return "SimpleGroupMetadata"
+	case KindSimpleGroupAdmins:
+		return "SimpleGroupAdmins"
+	case KindSimpleGroupMembers:
+		return "SimpleGroupMembers"
+	case KindSimpleGroupRoles:
+		return "SimpleGroupRoles"
+	case KindSimpleGroupLiveKitParticipants:
+		return "SimpleGroupLiveKitParticipants"
+	case KindStarterPacks:
+		return "StarterPacks"
+	case KindMediaStarterPacks:
+		return "MediaStarterPacks"
+	case KindWebBookmarks:
+		return "WebBookmarks"
+
 	}
 	return "unknown"
 }
@@ -294,23 +397,37 @@ const (
 	KindSimpleGroupReply                   Kind = 12
 	KindSeal                               Kind = 13
 	KindDirectMessage                      Kind = 14
+	KindFileMessage                        Kind = 15
 	KindGenericRepost                      Kind = 16
 	KindReactionToWebsite                  Kind = 17
+	KindPhoto                              Kind = 20
+	KindNormalVideoEvent                   Kind = 21
+	KindShortVideoEvent                    Kind = 22
+	KindPublicMessage                      Kind = 24
 	KindChannelCreation                    Kind = 40
 	KindChannelMetadata                    Kind = 41
 	KindChannelMessage                     Kind = 42
 	KindChannelHideMessage                 Kind = 43
 	KindChannelMuteUser                    Kind = 44
+	KindPodcastEpisode                     Kind = 54
 	KindChess                              Kind = 64
 	KindMergeRequests                      Kind = 818
-	KindComment                        Kind = 1111
+	KindPollResponse                       Kind = 1018
 	KindBid                                Kind = 1021
 	KindBidConfirmation                    Kind = 1022
 	KindOpenTimestamps                     Kind = 1040
 	KindGiftWrap                           Kind = 1059
 	KindFileMetadata                       Kind = 1063
+	KindPoll                               Kind = 1068
+	KindComment                            Kind = 1111
+	KindVoiceMessage                       Kind = 1222
+	KindScroll                             Kind = 1227
+	KindVoiceMessageComment                Kind = 1244
 	KindLiveChatMessage                    Kind = 1311
+	KindCodeSnippet                        Kind = 1337
 	KindPatch                              Kind = 1617
+	KindGitPullRequest                     Kind = 1618
+	KindGitPullRequestUpdate               Kind = 1619
 	KindIssue                              Kind = 1621
 	KindReply                              Kind = 1622
 	KindStatusOpen                         Kind = 1630
@@ -325,8 +442,15 @@ const (
 	KindTorrent                            Kind = 2003
 	KindTorrentComment                     Kind = 2004
 	KindCoinjoinPool                       Kind = 2022
+	KindDecoupledKeyClientAnnouncement     Kind = 4454
+	KindDecoupledEncryptionKeyDistribution Kind = 4455
 	KindCommunityPostApproval              Kind = 4550
 	KindJobFeedback                        Kind = 7000
+	KindReservedCashuWalletTokens          Kind = 7374
+	KindCashuWalletTokens                  Kind = 7375
+	KindCashuWalletHistory                 Kind = 7376
+	KindGeocacheLog                        Kind = 7516
+	KindGeocacheProofOfFind                Kind = 7517
 	KindSimpleGroupPutUser                 Kind = 9000
 	KindSimpleGroupRemoveUser              Kind = 9001
 	KindSimpleGroupEditMetadata            Kind = 9002
@@ -351,15 +475,29 @@ const (
 	KindBlockedRelayList                   Kind = 10006
 	KindSearchRelayList                    Kind = 10007
 	KindSimpleGroupList                    Kind = 10009
+	KindFavoriteRelaysList                 Kind = 10012
+	KindPrivateEventRelayList              Kind = 10013
 	KindInterestList                       Kind = 10015
 	KindNutZapInfo                         Kind = 10019
+	KindMediaFollows                       Kind = 10020
 	KindEmojiList                          Kind = 10030
+	KindDecoupledKeyAnnouncement           Kind = 10044
 	KindDMRelayList                        Kind = 10050
+	KindFavoritePodcasts                   Kind = 10054
 	KindUserServerList                     Kind = 10063
 	KindFileStorageServerList              Kind = 10096
 	KindGoodWikiAuthorList                 Kind = 10101
 	KindGoodWikiRelayList                  Kind = 10102
+	KindPodcastMetadata                    Kind = 10154
+	KindAuthoredPodcasts                   Kind = 10164
+	KindRelayMonitorAnnouncement           Kind = 10166
+	KindRoomPresence                       Kind = 10312
+	KindUserGraspList                      Kind = 10317
+	KindProxyAnnouncement                  Kind = 10377
+	KindTransportMethodAnnouncement        Kind = 11111
 	KindNWCWalletInfo                      Kind = 13194
+	KindNsiteRoot                          Kind = 15128
+	KindCashuWalletEvent                   Kind = 17375
 	KindLightningPubRPC                    Kind = 21000
 	KindClientAuthentication               Kind = 22242
 	KindNWCWalletRequest                   Kind = 23194
@@ -388,19 +526,21 @@ const (
 	KindModularArticleContent              Kind = 30041
 	KindReleaseArtifactSets                Kind = 30063
 	KindApplicationSpecificData            Kind = 30078
+	KindRelayDiscovery                     Kind = 30166
+	KindAppCurationSet                     Kind = 30267
 	KindLiveEvent                          Kind = 30311
+	KindInteractiveRoom                    Kind = 30312
+	KindConferenceEvent                    Kind = 30313
 	KindUserStatuses                       Kind = 30315
+	KindSlideSet                           Kind = 30388
 	KindClassifiedListing                  Kind = 30402
 	KindDraftClassifiedListing             Kind = 30403
 	KindRepositoryAnnouncement             Kind = 30617
 	KindRepositoryState                    Kind = 30618
-	KindSimpleGroupMetadata            Kind = 39000
-	KindSimpleGroupAdmins              Kind = 39001
-	KindSimpleGroupMembers             Kind = 39002
-	KindSimpleGroupRoles               Kind = 39003
-	KindSimpleGroupLiveKitParticipants Kind = 39004
 	KindWikiArticle                        Kind = 30818
 	KindRedirects                          Kind = 30819
+	KindDraftEvent                         Kind = 31234
+	KindLinkSet                            Kind = 31388
 	KindFeed                               Kind = 31890
 	KindDateCalendarEvent                  Kind = 31922
 	KindTimeCalendarEvent                  Kind = 31923
@@ -408,10 +548,24 @@ const (
 	KindCalendarEventRSVP                  Kind = 31925
 	KindHandlerRecommendation              Kind = 31989
 	KindHandlerInformation                 Kind = 31990
-	KindVideoEvent                     Kind = 34235
+	KindSoftwareApplication                Kind = 32267
-	KindShortVideoEvent                Kind = 34236
+	KindLegacyNsiteFile                    Kind = 34128
 	KindVideoViewEvent                     Kind = 34237
 	KindCommunityDefinition                Kind = 34550
+	KindNsiteNamed                         Kind = 35128
+	KindGeocacheListing                    Kind = 37515
+	KindGeocacheLogEntry                   Kind = 37516
+	KindCashuMintAnnouncement              Kind = 38172
+	KindFedimintAnnouncement               Kind = 38173
+	KindPeerToPeerOrderEvents              Kind = 38383
+	KindSimpleGroupMetadata                Kind = 39000
+	KindSimpleGroupAdmins                  Kind = 39001
+	KindSimpleGroupMembers                 Kind = 39002
+	KindSimpleGroupRoles                   Kind = 39003
+	KindSimpleGroupLiveKitParticipants     Kind = 39004
+	KindStarterPacks                       Kind = 39089
+	KindMediaStarterPacks                  Kind = 39092
+	KindWebBookmarks                       Kind = 39701
 )
 
 func (kind Kind) IsRegular() bool {

nip29/group.go

@@ -4,14 +4,22 @@ import (
 	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"strings"
 
 	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/nip19"
 )
 
 type GroupAddress struct {
+	// URL of the relay that is hosting the group
 	Relay string
+
+	// Group identifier ("d"/"h" tag)
 	ID string
+
+	// Public key of the relay, used to publish kind:39000/etc events
+	Self nostr.PubKey
 }
 
 func (gid GroupAddress) String() string {
@@ -19,6 +27,10 @@ func (gid GroupAddress) String() string {
 	return fmt.Sprintf("%s'%s", p.Host, gid.ID)
 }
 
+func (gid GroupAddress) Code() string {
+	return nip19.EncodeNaddr(gid.Self, 39000, gid.ID, []string{gid.Relay})
+}
+
 func (gid GroupAddress) IsValid() bool {
 	return gid.Relay != "" && gid.ID != ""
 }
@@ -27,14 +39,6 @@ func (gid GroupAddress) Equals(gid2 GroupAddress) bool {
 	return gid.Relay == gid2.Relay && gid.ID == gid2.ID
 }
 
-func ParseGroupAddress(raw string) (GroupAddress, error) {
-	spl := strings.Split(raw, "'")
-	if len(spl) != 2 {
-		return GroupAddress{}, fmt.Errorf("invalid group id")
-	}
-	return GroupAddress{ID: spl[1], Relay: nostr.NormalizeURL(spl[0])}, nil
-}
-
 type Group struct {
 	Address GroupAddress
 
@@ -42,6 +46,7 @@ type Group struct {
 	Picture             string
 	About               string
 	Members             map[nostr.PubKey][]*Role
+	LiveKitParticipants []nostr.PubKey
 
 	// indicates that only members can read group messages
 	Private bool
@@ -55,6 +60,18 @@ type Group struct {
 	// indicates that relays should hide group metadata from non-members
 	Hidden bool
 
+	// indicates that the group supports audio/video live chat
+	LiveKit bool
+
+	// indicates which event kinds this group supports
+	SupportedKinds []nostr.Kind
+
+	// arbitrary string indicating the parent group
+	Parent string
+
+	// ordered list of identifiers of child groups
+	Children []string
+
 	Roles       []*Role
 	InviteCodes []string
 
@@ -62,6 +79,7 @@ type Group struct {
 	LastAdminsUpdate              nostr.Timestamp
 	LastMembersUpdate             nostr.Timestamp
 	LastRolesUpdate               nostr.Timestamp
+	LastLiveKitParticipantsUpdate nostr.Timestamp
 }
 
 func (group Group) String() string {
@@ -83,6 +101,11 @@ func (group Group) String() string {
 		maybeClosed = " closed"
 	}
 
+	maybeLiveKit := ""
+	if group.LiveKit {
+		maybeLiveKit = " livekit"
+	}
+
 	members := make([]string, len(group.Members))
 	i := 0
 	for pubkey, roles := range group.Members {
@@ -101,13 +124,14 @@ func (group Group) String() string {
 		i++
 	}
 
-	return fmt.Sprintf(`<Group %s name="%s"%s%s%s%s picture="%s" about="%s" members=[%v]>`,
+	return fmt.Sprintf(`<Group %s name="%s"%s%s%s%s%s picture="%s" about="%s" members=[%v]>`,
 		group.Address,
 		group.Name,
 		maybePrivate,
 		maybeRestricted,
 		maybeHidden,
 		maybeClosed,
+		maybeLiveKit,
 		group.Picture,
 		group.About,
 		strings.Join(members, " "),
@@ -115,16 +139,17 @@ func (group Group) String() string {
 }
 
 // NewGroup takes a group address in the form "<id>'<relay-hostname>"
-func NewGroup(gadstr string) (Group, error) {
+func NewGroup(relayHost, groupId string) (Group, error) {
-	gad, err := ParseGroupAddress(gadstr)
+	relayHost = nostr.NormalizeURL(relayHost)
-	if err != nil {
-		return Group{}, fmt.Errorf("invalid group id '%s': %w", gadstr, err)
-	}
 
 	return Group{
-		Address: gad,
+		Address: GroupAddress{
-		Name:    gad.ID,
+			Relay: relayHost,
+			ID:    groupId,
+		},
+		Name:                groupId,
 		Members:             make(map[nostr.PubKey][]*Role),
+		LiveKitParticipants: make([]nostr.PubKey, 0),
 	}, nil
 }
 
@@ -136,6 +161,7 @@ func NewGroupFromMetadataEvent(relayURL string, evt *nostr.Event) (Group, error)
 		},
 		Name:                evt.Tags.GetD(),
 		Members:             make(map[nostr.PubKey][]*Role),
+		LiveKitParticipants: make([]nostr.PubKey, 0),
 	}
 
 	err := g.MergeInMetadataEvent(evt)
@@ -173,6 +199,26 @@ func (group Group) ToMetadataEvent() nostr.Event {
 	if group.Closed {
 		evt.Tags = append(evt.Tags, nostr.Tag{"closed"})
 	}
+	if group.LiveKit {
+		evt.Tags = append(evt.Tags, nostr.Tag{"livekit"})
+	}
+
+	if group.SupportedKinds != nil {
+		tag := make(nostr.Tag, 1, 1+len(group.SupportedKinds))
+		tag[0] = "supported_kinds"
+		for _, kind := range group.SupportedKinds {
+			tag = append(tag, strconv.Itoa(int(kind)))
+		}
+		evt.Tags = append(evt.Tags, tag)
+	}
+
+	if group.Parent != "" {
+		evt.Tags = append(evt.Tags, nostr.Tag{"parent", group.Parent})
+	}
+
+	for _, child := range group.Children {
+		evt.Tags = append(evt.Tags, nostr.Tag{"child", child})
+	}
 
 	return evt
 }
@@ -236,6 +282,22 @@ func (group Group) ToRolesEvent() nostr.Event {
 	return evt
 }
 
+func (group Group) ToLiveKitParticipantsEvent() nostr.Event {
+	evt := nostr.Event{
+		Kind:      nostr.KindSimpleGroupLiveKitParticipants,
+		CreatedAt: group.LastLiveKitParticipantsUpdate,
+		Tags:      make(nostr.Tags, 1, 1+len(group.LiveKitParticipants)),
+	}
+	evt.Tags[0] = nostr.Tag{"d", group.Address.ID}
+
+	for _, member := range group.LiveKitParticipants {
+		tag := nostr.Tag{"participant", member.Hex()}
+		evt.Tags = append(evt.Tags, tag)
+	}
+
+	return evt
+}
+
 func (group *Group) MergeInMetadataEvent(evt *nostr.Event) error {
 	if evt.Kind != nostr.KindSimpleGroupMetadata {
 		return fmt.Errorf("expected kind %d, got %d", nostr.KindSimpleGroupMetadata, evt.Kind)
@@ -247,27 +309,46 @@ func (group *Group) MergeInMetadataEvent(evt *nostr.Event) error {
 	group.LastMetadataUpdate = evt.CreatedAt
 	group.Name = group.Address.ID
 
-	if tag := evt.Tags.Find("name"); tag != nil {
+	for _, tag := range evt.Tags {
-		group.Name = tag[1]
+		if len(tag) >= 1 {
-	}
+			switch tag[0] {
-	if tag := evt.Tags.Find("about"); tag != nil {
+			case "private":
-		group.About = tag[1]
-	}
-	if tag := evt.Tags.Find("picture"); tag != nil {
-		group.Picture = tag[1]
-	}
-
-	if tag := evt.Tags.Find("private"); tag != nil {
 				group.Private = true
-	}
+			case "restricted":
-	if tag := evt.Tags.Find("restricted"); tag != nil {
 				group.Restricted = true
-	}
+			case "closed":
-	if tag := evt.Tags.Find("hidden"); tag != nil {
-		group.Hidden = true
-	}
-	if tag := evt.Tags.Find("closed"); tag != nil {
 				group.Closed = true
+			case "hidden":
+				group.Hidden = true
+			case "livekit":
+				group.LiveKit = true
+			case "supported_kinds":
+				kinds := make([]nostr.Kind, 0, len(tag)-1)
+				for _, raw := range tag[1:] {
+					kind, err := strconv.Atoi(raw)
+					if err != nil {
+						continue
+					}
+					kinds = append(kinds, nostr.Kind(kind))
+				}
+				group.SupportedKinds = kinds
+			default:
+				if len(tag) >= 2 {
+					switch tag[0] {
+					case "name":
+						group.Name = tag[1]
+					case "about":
+						group.About = tag[1]
+					case "picture":
+						group.Picture = tag[1]
+					case "parent":
+						group.Parent = tag[1]
+					case "child":
+						group.Children = append(group.Children, tag[1])
+					}
+				}
+			}
+		}
 	}
 
 	return nil
@@ -368,3 +449,34 @@ func (group *Group) MergeInRolesEvent(evt *nostr.Event) error {
 
 	return nil
 }
+
+func (group *Group) MergeInLiveKitParticipantsEvent(evt *nostr.Event) error {
+	if evt.Kind != nostr.KindSimpleGroupLiveKitParticipants {
+		return fmt.Errorf("expected kind %d, got %d", nostr.KindSimpleGroupLiveKitParticipants, evt.Kind)
+	}
+	if evt.CreatedAt < group.LastLiveKitParticipantsUpdate {
+		return fmt.Errorf("event is older than our last update (%d vs %d)", evt.CreatedAt, group.LastLiveKitParticipantsUpdate)
+	}
+
+	group.LastLiveKitParticipantsUpdate = evt.CreatedAt
+	group.LiveKitParticipants = make([]nostr.PubKey, 0, len(evt.Tags))
+	for _, tag := range evt.Tags {
+		if len(tag) < 2 {
+			continue
+		}
+		if tag[0] != "participant" {
+			continue
+		}
+
+		member, err := nostr.PubKeyFromHex(tag[1])
+		if err != nil {
+			continue
+		}
+		if slices.Contains(group.LiveKitParticipants, member) {
+			continue
+		}
+		group.LiveKitParticipants = append(group.LiveKitParticipants, member)
+	}
+
+	return nil
+}

nip29/moderation_actions.go

@@ -3,6 +3,7 @@ package nip29
 import (
 	"fmt"
 	"slices"
+	"strconv"
 
 	"fiatjaf.com/nostr"
 )
@@ -78,48 +79,63 @@ var moderationActionFactories = map[nostr.Kind]func(nostr.Event) (Action, error)
 	nostr.KindSimpleGroupEditMetadata: func(evt nostr.Event) (Action, error) {
 		ok := false
 		edit := EditMetadata{When: evt.CreatedAt}
-		if t := evt.Tags.Find("name"); t != nil {
-			edit.NameValue = &t[1]
-			ok = true
-		}
-		if t := evt.Tags.Find("picture"); t != nil {
-			edit.PictureValue = &t[1]
-			ok = true
-		}
-		if t := evt.Tags.Find("about"); t != nil {
-			edit.AboutValue = &t[1]
-			ok = true
-		}
 
-		y := true
+		for _, tag := range evt.Tags {
-		n := false
+			if len(tag) >= 1 {
-		if evt.Tags.Has("closed") {
+				switch tag[0] {
-			edit.ClosedValue = &y
+				case "name":
-			ok = true
+					if len(tag) >= 2 {
-		} else if evt.Tags.Has("open") {
+						edit.Group.Name = tag[1]
-			edit.ClosedValue = &n
 						ok = true
 					}
-		if evt.Tags.Has("restricted") {
+				case "picture":
-			edit.RestrictedValue = &y
+					if len(tag) >= 2 {
-			ok = true
+						edit.Group.Picture = tag[1]
-		} else if evt.Tags.Has("unrestricted") {
-			edit.RestrictedValue = &n
 						ok = true
 					}
-		if evt.Tags.Has("hidden") {
+				case "about":
-			edit.HiddenValue = &y
+					if len(tag) >= 2 {
-			ok = true
+						edit.Group.About = tag[1]
-		} else if evt.Tags.Has("visible") {
-			edit.HiddenValue = &n
 						ok = true
 					}
-		if evt.Tags.Has("private") {
+				case "supported_kinds":
-			edit.PrivateValue = &y
+					kinds := make([]nostr.Kind, 0, len(tag)-1)
+					for _, kstr := range tag[1:] {
+						if kind, err := strconv.ParseUint(kstr, 10, 16); err != nil {
+							return nil, fmt.Errorf("invalid kind: %w", err)
+						} else {
+							kinds = append(kinds, nostr.Kind(kind))
+						}
+					}
+					edit.Group.SupportedKinds = kinds
 					ok = true
-		} else if evt.Tags.Has("public") {
+				case "closed":
-			edit.PrivateValue = &n
+					edit.Group.Closed = true
 					ok = true
+				case "restricted":
+					edit.Group.Restricted = true
+					ok = true
+				case "hidden":
+					edit.Group.Hidden = true
+					ok = true
+				case "private":
+					edit.Group.Private = true
+					ok = true
+				case "parent":
+					if len(tag) >= 2 {
+						edit.Group.Parent = tag[1]
+						ok = true
+					}
+				case "livekit":
+					edit.Group.LiveKit = true
+					ok = true
+				case "child":
+					if len(tag) >= 2 {
+						edit.Group.Children = append(edit.Group.Children, tag[1])
+						ok = true
+					}
+				}
+			}
 		}
 
 		if ok {
@@ -226,40 +242,26 @@ func (a RemoveUser) Apply(group *Group) {
 }
 
 type EditMetadata struct {
-	NameValue       *string
+	Group
-	PictureValue    *string
+
-	AboutValue      *string
-	RestrictedValue *bool
-	ClosedValue     *bool
-	HiddenValue     *bool
-	PrivateValue    *bool
 	When nostr.Timestamp
 }
 
 func (_ EditMetadata) Name() string { return "edit-metadata" }
 func (a EditMetadata) Apply(group *Group) {
 	group.LastMetadataUpdate = a.When
-	if a.NameValue != nil {
+
-		group.Name = *a.NameValue
+	group.Name = a.Group.Name
-	}
+	group.Picture = a.Group.Picture
-	if a.PictureValue != nil {
+	group.About = a.Group.About
-		group.Picture = *a.PictureValue
+	group.Restricted = a.Group.Restricted
-	}
+	group.Closed = a.Group.Closed
-	if a.AboutValue != nil {
+	group.Hidden = a.Group.Hidden
-		group.About = *a.AboutValue
+	group.Private = a.Group.Private
-	}
+	group.LiveKit = a.Group.LiveKit
-	if a.RestrictedValue != nil {
+	group.SupportedKinds = a.Group.SupportedKinds
-		group.Restricted = *a.RestrictedValue
+	group.Parent = a.Group.Parent
-	}
+	group.Children = a.Group.Children
-	if a.ClosedValue != nil {
-		group.Closed = *a.ClosedValue
-	}
-	if a.HiddenValue != nil {
-		group.Hidden = *a.HiddenValue
-	}
-	if a.PrivateValue != nil {
-		group.Private = *a.PrivateValue
-	}
 }
 
 type CreateGroup struct {
@@ -272,6 +274,7 @@ func (a CreateGroup) Apply(group *Group) {
 	group.LastMetadataUpdate = a.When
 	group.LastAdminsUpdate = a.When
 	group.LastMembersUpdate = a.When
+	group.LastLiveKitParticipantsUpdate = a.When
 }
 
 type DeleteGroup struct {
@@ -281,6 +284,7 @@ type DeleteGroup struct {
 func (_ DeleteGroup) Name() string { return "delete-group" }
 func (a DeleteGroup) Apply(group *Group) {
 	group.Members = make(map[nostr.PubKey][]*Role)
+	group.LiveKitParticipants = make([]nostr.PubKey, 0)
 	group.Closed = true
 	group.Private = true
 	group.Restricted = true
@@ -288,9 +292,11 @@ func (a DeleteGroup) Apply(group *Group) {
 	group.Name = "[deleted]"
 	group.About = ""
 	group.Picture = ""
+	group.LiveKit = false
 	group.LastMetadataUpdate = a.When
 	group.LastAdminsUpdate = a.When
 	group.LastMembersUpdate = a.When
+	group.LastLiveKitParticipantsUpdate = a.When
 }
 
 type CreateInvite struct {

nip29/nip29_test.go

@@ -15,7 +15,7 @@ const (
 )
 
 func TestGroupEventBackAndForth(t *testing.T) {
-	group1, _ := NewGroup("relay.com'xyz")
+	group1, _ := NewGroup("relay.com", "xyz")
 	group1.Name = "banana"
 	group1.Private = true
 	meta1 := group1.ToMetadataEvent()
@@ -31,7 +31,7 @@ func TestGroupEventBackAndForth(t *testing.T) {
 	}
 	require.True(t, hasPrivate, "translation of group1 to metadata event failed: %s", meta1)
 
-	group2, _ := NewGroup("groups.com'abc")
+	group2, _ := NewGroup("groups.com", "abc")
 	alicePub, _ := nostr.PubKeyFromHex(ALICE)
 	group2.Members[alicePub] = []*Role{{Name: "nada"}}
 	admins2 := group2.ToAdminsEvent()

nip34/gitnaturalapi/commits.go

@@ -0,0 +1,106 @@
+package gitnaturalapi
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type Person struct {
+	Name      string
+	Email     string
+	Timestamp int64
+	Timezone  string
+}
+
+type Commit struct {
+	Hash      string
+	Tree      string
+	Parents   []string
+	Author    Person
+	Committer Person
+	Message   string
+}
+
+func ParseCommit(data []byte, hash string) (*Commit, error) {
+	content := string(data)
+
+	headerEndIndex := strings.Index(content, "\n\n")
+	if headerEndIndex == -1 {
+		return nil, fmt.Errorf("invalid commit format for %s: no message separator found", hash)
+	}
+
+	header := content[:headerEndIndex]
+	message := content[headerEndIndex+2:]
+
+	lines := strings.Split(header, "\n")
+	result := &Commit{
+		Hash:    hash,
+		Parents: []string{},
+		Message: message,
+	}
+
+	for _, line := range lines {
+		if strings.HasPrefix(line, "tree ") {
+			result.Tree = line[5:]
+		} else if strings.HasPrefix(line, "parent ") {
+			result.Parents = append(result.Parents, line[7:])
+		} else if strings.HasPrefix(line, "author ") {
+			person, err := parsePerson(line[7:])
+			if err != nil {
+				return nil, fmt.Errorf("invalid author in commit %s: %w", hash, err)
+			}
+			result.Author = person
+		} else if strings.HasPrefix(line, "committer ") {
+			person, err := parsePerson(line[10:])
+			if err != nil {
+				return nil, fmt.Errorf("invalid committer in commit %s: %w", hash, err)
+			}
+			result.Committer = person
+		}
+	}
+
+	if result.Tree == "" {
+		return nil, fmt.Errorf("invalid commit format for %s: missing tree", hash)
+	}
+	if result.Author.Name == "" {
+		return nil, fmt.Errorf("invalid commit format for %s: missing author", hash)
+	}
+	if result.Committer.Name == "" {
+		return nil, fmt.Errorf("invalid commit format for %s: missing committer", hash)
+	}
+
+	return result, nil
+}
+
+func parsePerson(line string) (Person, error) {
+	emailStart := strings.Index(line, " <")
+	if emailStart == -1 {
+		return Person{}, fmt.Errorf("invalid person format: %s", line)
+	}
+	name := line[:emailStart]
+
+	emailEnd := strings.Index(line[emailStart+2:], ">")
+	if emailEnd == -1 {
+		return Person{}, fmt.Errorf("invalid person format: %s", line)
+	}
+	email := line[emailStart+2 : emailStart+2+emailEnd]
+
+	rest := strings.TrimSpace(line[emailStart+2+emailEnd+1:])
+	parts := strings.SplitN(rest, " ", 2)
+	if len(parts) != 2 {
+		return Person{}, fmt.Errorf("invalid person format: %s", line)
+	}
+
+	timestamp, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil {
+		return Person{}, fmt.Errorf("invalid timestamp in person: %s", parts[0])
+	}
+
+	return Person{
+		Name:      name,
+		Email:     email,
+		Timestamp: timestamp,
+		Timezone:  parts[1],
+	}, nil
+}

nip34/gitnaturalapi/diff.go

@@ -0,0 +1,413 @@
+package gitnaturalapi
+
+import (
+	"strings"
+	"sync"
+)
+
+type DiffLine struct {
+	Index  int
+	Status string
+	Text   string
+	Change string
+}
+
+type DiffFile struct {
+	Path    string
+	Status  string
+	Content []byte
+	Lines   []DiffLine
+}
+
+type changedEntry struct {
+	newVersion  TreeEntry
+	oldVersions []TreeEntry
+}
+
+func GetCommitDiff(url string, commitOrRef string) ([]DiffFile, error) {
+	commit, err := GetSingleCommit(url, commitOrRef)
+	if err != nil {
+		return nil, err
+	}
+
+	added := make(map[string]TreeEntry)
+	deleted := make(map[string]TreeEntry)
+	changed := make(map[string]*changedEntry)
+	unchanged := make(map[string]bool)
+
+	for _, parent := range commit.Parents {
+		parentCommit, err := GetSingleCommit(url, parent)
+		if err != nil {
+			return nil, err
+		}
+		err = computeTreeDiffs(url, commit.Tree, parentCommit.Tree, "", added, deleted, changed, unchanged)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var diff []DiffFile
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	var firstErr error
+
+	for path, entry := range changed {
+		p := path
+		e := entry
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			curr, err := GetObject(url, e.newVersion.Hash)
+			if err != nil {
+				mu.Lock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				mu.Unlock()
+				return
+			}
+			if curr == nil {
+				return
+			}
+
+			if len(e.oldVersions) == 0 {
+				return
+			}
+
+			oldObj, err := GetObject(url, e.oldVersions[0].Hash)
+			if err != nil || oldObj == nil {
+				return
+			}
+
+			if isBinary(curr.Data) || isBinary(oldObj.Data) {
+				mu.Lock()
+				diff = append(diff, DiffFile{
+					Path:   p,
+					Status: "changed-binary",
+				})
+				mu.Unlock()
+				return
+			}
+
+			lines := diffTextLines(oldObj.Data, curr.Data)
+			mu.Lock()
+			diff = append(diff, DiffFile{
+				Path:   p,
+				Status: "changed",
+				Lines:  lines,
+			})
+			mu.Unlock()
+		}()
+	}
+
+	for path, entry := range deleted {
+		p := path
+		e := entry
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			obj, err := GetObject(url, e.Hash)
+			if err != nil || obj == nil {
+				return
+			}
+			mu.Lock()
+			diff = append(diff, DiffFile{
+				Path:    p,
+				Status:  "deleted",
+				Content: obj.Data,
+			})
+			mu.Unlock()
+		}()
+	}
+
+	for path, entry := range added {
+		p := path
+		e := entry
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			obj, err := GetObject(url, e.Hash)
+			if err != nil || obj == nil {
+				return
+			}
+			mu.Lock()
+			diff = append(diff, DiffFile{
+				Path:    p,
+				Status:  "added",
+				Content: obj.Data,
+			})
+			mu.Unlock()
+		}()
+	}
+
+	wg.Wait()
+
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	return diff, nil
+}
+
+func isBinary(data []byte) bool {
+	for _, b := range data {
+		if b == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func diffTextLines(oldData []byte, newData []byte) []DiffLine {
+	oldText := string(oldData)
+	newText := string(newData)
+	oldLines := splitLines(oldText)
+	newLines := splitLines(newText)
+
+	ops := lcsOperations(oldLines, newLines)
+	allLines := make([]DiffLine, 0, len(ops))
+	oldIndex := 1
+	newIndex := 1
+
+	for i := 0; i < len(ops); i++ {
+		op := ops[i]
+		var next *lcsOp
+		if i+1 < len(ops) {
+			next = &ops[i+1]
+		}
+
+		if op.typ == "del" && next != nil && next.typ == "add" {
+			allLines = append(allLines, DiffLine{
+				Status: "changed",
+				Index:  newIndex,
+				Text:   next.line,
+			})
+			oldIndex++
+			newIndex++
+			i++
+			continue
+		}
+
+		if op.typ == "add" && next != nil && next.typ == "del" {
+			allLines = append(allLines, DiffLine{
+				Status: "changed",
+				Index:  newIndex,
+				Text:   op.line,
+			})
+			oldIndex++
+			newIndex++
+			i++
+			continue
+		}
+
+		if op.typ == "add" {
+			allLines = append(allLines, DiffLine{
+				Status: "added",
+				Index:  newIndex,
+				Text:   op.line,
+			})
+			newIndex++
+			continue
+		}
+
+		if op.typ == "del" {
+			allLines = append(allLines, DiffLine{
+				Status: "deleted",
+				Index:  oldIndex,
+				Text:   op.line,
+			})
+			oldIndex++
+			continue
+		}
+
+		oldIndex++
+		newIndex++
+	}
+
+	if len(allLines) == 0 {
+		return allLines
+	}
+
+	keep := make([]bool, len(allLines))
+	for i := 0; i < len(allLines); i++ {
+		start := i - 3
+		if start < 0 {
+			start = 0
+		}
+		end := i + 3
+		if end >= len(allLines) {
+			end = len(allLines) - 1
+		}
+		for j := start; j <= end; j++ {
+			keep[j] = true
+		}
+	}
+
+	result := make([]DiffLine, 0, len(allLines))
+	for i := 0; i < len(allLines); i++ {
+		if keep[i] {
+			result = append(result, allLines[i])
+		}
+	}
+
+	return result
+}
+
+type lcsOp struct {
+	typ  string
+	line string
+}
+
+func splitLines(text string) []string {
+	lines := strings.Split(text, "\n")
+	if len(lines) > 0 && lines[len(lines)-1] == "" {
+		lines = lines[:len(lines)-1]
+	}
+	return lines
+}
+
+func lcsOperations(oldLines []string, newLines []string) []lcsOp {
+	n := len(oldLines)
+	m := len(newLines)
+
+	dp := make([][]uint32, n+1)
+	for i := range dp {
+		dp[i] = make([]uint32, m+1)
+	}
+
+	for i := 1; i <= n; i++ {
+		for j := 1; j <= m; j++ {
+			if oldLines[i-1] == newLines[j-1] {
+				dp[i][j] = dp[i-1][j-1] + 1
+			} else if dp[i-1][j] >= dp[i][j-1] {
+				dp[i][j] = dp[i-1][j]
+			} else {
+				dp[i][j] = dp[i][j-1]
+			}
+		}
+	}
+
+	ops := make([]lcsOp, 0, n+m)
+	i := n
+	j := m
+	for i > 0 || j > 0 {
+		if i > 0 && j > 0 && oldLines[i-1] == newLines[j-1] {
+			ops = append(ops, lcsOp{typ: "equal", line: oldLines[i-1]})
+			i--
+			j--
+			continue
+		}
+
+		if i > 0 && (j == 0 || dp[i-1][j] >= dp[i][j-1]) {
+			ops = append(ops, lcsOp{typ: "del", line: oldLines[i-1]})
+			i--
+			continue
+		}
+
+		if j > 0 {
+			ops = append(ops, lcsOp{typ: "add", line: newLines[j-1]})
+			j--
+		}
+	}
+
+	for i, j := 0, len(ops)-1; i < j; i, j = i+1, j-1 {
+		ops[i], ops[j] = ops[j], ops[i]
+	}
+
+	return ops
+}
+
+func computeTreeDiffs(
+	url string,
+	treeHash string,
+	parentTreeHash string,
+	basePath string,
+	added map[string]TreeEntry,
+	deleted map[string]TreeEntry,
+	changed map[string]*changedEntry,
+	unchanged map[string]bool,
+) error {
+	var newTree []TreeEntry
+	var oldTree []TreeEntry
+
+	if treeHash != "" {
+		obj, err := GetObject(url, treeHash)
+		if err != nil {
+			return err
+		}
+		if obj != nil {
+			newTree = ParseTree(obj.Data)
+		}
+	}
+
+	if parentTreeHash != "" {
+		obj, err := GetObject(url, parentTreeHash)
+		if err != nil {
+			return err
+		}
+		if obj != nil {
+			oldTree = ParseTree(obj.Data)
+		}
+	}
+
+	for _, entry := range newTree {
+		var old *TreeEntry
+		for _, o := range oldTree {
+			if o.Path == entry.Path {
+				o := o
+				old = &o
+				break
+			}
+		}
+
+		if old != nil {
+			delete(added, basePath+entry.Path)
+
+			if old.Hash == entry.Hash {
+				unchanged[basePath+entry.Path] = true
+			} else {
+				if entry.IsDir {
+					err := computeTreeDiffs(url, entry.Hash, old.Hash, basePath+entry.Path+"/", added, deleted, changed, unchanged)
+					if err != nil {
+						return err
+					}
+				} else {
+					if existing, exists := changed[basePath+entry.Path]; !exists {
+						changed[basePath+entry.Path] = &changedEntry{
+							newVersion:  entry,
+							oldVersions: []TreeEntry{*old},
+						}
+					} else {
+						existing.oldVersions = append(existing.oldVersions, *old)
+					}
+				}
+			}
+		} else {
+			if entry.IsDir {
+				err := computeTreeDiffs(url, entry.Hash, "", basePath+entry.Path+"/", added, deleted, changed, unchanged)
+				if err != nil {
+					return err
+				}
+			} else {
+				added[basePath+entry.Path] = entry
+			}
+		}
+	}
+
+	for _, old := range oldTree {
+		if unchanged[basePath+old.Path] || changed[basePath+old.Path] != nil {
+			continue
+		}
+
+		if old.IsDir {
+			err := computeTreeDiffs(url, "", old.Hash, basePath+old.Path+"/", added, deleted, changed, unchanged)
+			if err != nil {
+				return err
+			}
+		} else {
+			deleted[basePath+old.Path] = old
+		}
+	}
+
+	return nil
+}

nip34/gitnaturalapi/lib.go

@@ -0,0 +1,264 @@
+package gitnaturalapi
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+)
+
+type MissingCapability struct {
+	URL        string
+	Capability string
+}
+
+func (e *MissingCapability) Error() string {
+	return fmt.Sprintf("server at %s is missing required capability %s", e.URL, e.Capability)
+}
+
+func prepareRequest(url string, commitOrRef string, needFilter bool) (resolvedRef string, capabilities []string, err error) {
+	var info *InfoRefsUploadPackResponse
+	if strings.HasPrefix(commitOrRef, "refs/") {
+		info, err = GetInfoRefs(url)
+		if err != nil {
+			return "", nil, err
+		}
+		resolved, ok := info.Refs[commitOrRef]
+		if !ok {
+			return "", nil, fmt.Errorf("ref %s not found", commitOrRef)
+		}
+		commitOrRef = resolved
+	}
+
+	caps, err := GetCapabilities(url, info)
+	if err != nil {
+		return "", nil, err
+	}
+
+	for _, c := range DefaultCapabilities {
+		if slices.Contains(caps, c) {
+			capabilities = append(capabilities, c)
+		}
+	}
+	for _, c := range NecessaryCapabilities {
+		if slices.Contains(caps, c) {
+			capabilities = append(capabilities, c)
+		} else {
+			return "", nil, &MissingCapability{URL: url, Capability: c}
+		}
+	}
+	for _, c := range RequiredCapabilities {
+		if !slices.Contains(caps, c) {
+			return "", nil, &MissingCapability{URL: url, Capability: c}
+		}
+	}
+	if needFilter {
+		if slices.Contains(caps, "filter") {
+			capabilities = append(capabilities, "filter")
+		} else {
+			return "", nil, &MissingCapability{URL: url, Capability: "filter"}
+		}
+	}
+
+	return commitOrRef, capabilities, nil
+}
+
+func GetObject(url string, blobHash string) (*ParsedObject, error) {
+	ref, caps, err := prepareRequest(url, blobHash, false)
+	if err != nil {
+		return nil, err
+	}
+
+	deepen := 1
+	want, err := CreateWantRequest(ref, caps, &deepen, "")
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := FetchPackfile(url, want)
+	if err != nil {
+		return nil, err
+	}
+
+	return result.Objects[blobHash], nil
+}
+
+func GetDirectoryTreeAt(url string, commitOrRef string, nestLimit *int) (*Tree, error) {
+	ref, caps, err := prepareRequest(url, commitOrRef, true)
+	if err != nil {
+		return nil, err
+	}
+
+	want, err := CreateWantRequest(ref, caps, nestLimit, "blob:none")
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := FetchPackfile(url, want)
+	if err != nil {
+		return nil, err
+	}
+
+	commit := result.Objects[ref]
+	if commit == nil {
+		return nil, fmt.Errorf("commit %s not found in packfile", ref)
+	}
+
+	treeHash := string(commit.Data[5:45])
+	rootTree := result.Objects[treeHash]
+	if rootTree == nil {
+		return nil, fmt.Errorf("root tree %s not found in packfile", treeHash)
+	}
+
+	return LoadTree(rootTree, result.Objects, nestLimit), nil
+}
+
+func ShallowCloneRepositoryAt(url string, commitOrRef string) (*Commit, *Tree, error) {
+	ref, caps, err := prepareRequest(url, commitOrRef, false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	deepen := 1
+	want, err := CreateWantRequest(ref, caps, &deepen, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	result, err := FetchPackfile(url, want)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	commitObj := result.Objects[ref]
+	if commitObj == nil {
+		return nil, nil, fmt.Errorf("commit %s not found in packfile", ref)
+	}
+
+	treeHash := string(commitObj.Data[5:45])
+	rootTree := result.Objects[treeHash]
+	if rootTree == nil {
+		return nil, nil, fmt.Errorf("root tree %s not found in packfile", treeHash)
+	}
+
+	commit, err := ParseCommit(commitObj.Data, commitObj.Hash)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	tree := LoadTree(rootTree, result.Objects, nil)
+	return commit, tree, nil
+}
+
+func FetchCommitsOnly(url string, commitOrRef string, maxCommits *int) ([]*Commit, error) {
+	ref, caps, err := prepareRequest(url, commitOrRef, true)
+	if err != nil {
+		return nil, err
+	}
+
+	want, err := CreateWantRequest(ref, caps, maxCommits, "tree:0")
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := FetchPackfile(url, want)
+	if err != nil {
+		return nil, err
+	}
+
+	commitMap := make(map[string]*Commit, len(result.Objects))
+	for hash, obj := range result.Objects {
+		commit, err := ParseCommit(obj.Data, hash)
+		if err != nil {
+			return nil, err
+		}
+		commitMap[hash] = commit
+	}
+
+	// sort topologically starting from the requested ref
+	sorted := make([]*Commit, 0, len(commitMap))
+	visited := make(map[string]bool, len(commitMap))
+	var visit func(hash string)
+	visit = func(hash string) {
+		if visited[hash] {
+			return
+		}
+		c, ok := commitMap[hash]
+		if !ok {
+			return
+		}
+		visited[hash] = true
+		sorted = append(sorted, c)
+		for _, parent := range c.Parents {
+			visit(parent)
+		}
+	}
+	visit(ref)
+
+	for _, c := range commitMap {
+		if !visited[c.Hash] {
+			sorted = append(sorted, c)
+		}
+	}
+
+	return sorted, nil
+}
+
+func GetSingleCommit(url string, commitOrRef string) (*Commit, error) {
+	maxCommits := 1
+	commits, err := FetchCommitsOnly(url, commitOrRef, &maxCommits)
+	if err != nil {
+		return nil, err
+	}
+	if len(commits) == 0 {
+		return nil, fmt.Errorf("no commit found for reference: %s", commitOrRef)
+	}
+	return commits[0], nil
+}
+
+func GetObjectByPath(url string, commitOrRef string, path string) (*TreeEntry, error) {
+	normalizedPath := strings.ReplaceAll(path, "\\", "/")
+	normalizedPath = strings.TrimLeft(normalizedPath, "/")
+	normalizedPath = strings.TrimRight(normalizedPath, "/")
+
+	var pathSegments []string
+	if normalizedPath != "" {
+		pathSegments = strings.Split(normalizedPath, "/")
+	}
+	requiredDepth := len(pathSegments)
+
+	tree, err := GetDirectoryTreeAt(url, commitOrRef, &requiredDepth)
+	if err != nil {
+		return nil, err
+	}
+
+	currentLevel := tree
+nextSegment:
+	for i, segment := range pathSegments {
+		isLastSegment := i == len(pathSegments)-1
+
+		for _, dir := range currentLevel.Directories {
+			if dir.Name == segment {
+				if isLastSegment {
+					return &TreeEntry{Path: segment, Mode: "40000", IsDir: true, Hash: dir.Hash}, nil
+				}
+				if dir.Content != nil {
+					currentLevel = dir.Content
+					continue nextSegment
+				}
+				return nil, nil
+			}
+		}
+
+		if isLastSegment {
+			for _, file := range currentLevel.Files {
+				if file.Name == segment {
+					return &TreeEntry{Path: segment, Mode: "100644", IsDir: false, Hash: file.Hash}, nil
+				}
+			}
+		}
+
+		return nil, nil
+	}
+
+	return nil, nil
+}

nip34/gitnaturalapi/lib_test.go

@@ -0,0 +1,289 @@
+package gitnaturalapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetRefs(t *testing.T) {
+	info, err := GetInfoRefs("https://codeberg.org/dluvian/gitplaza.git")
+	require.NoError(t, err)
+
+	require.Contains(t, info.Capabilities, "shallow")
+	require.Contains(t, info.Capabilities, "object-format=sha1")
+	require.Greater(t, len(info.Refs), 5)
+	require.Contains(t, info.Refs, "refs/heads/master")
+	require.Equal(t, "a04d0761564b0d23c5edbadf494ab4f1cc4656f4", info.Refs["refs/tags/v0.1.0"])
+	require.Equal(t, "refs/heads/master", info.Symrefs["HEAD"])
+}
+
+func TestGetOnlyTreeAtCurrentCommit(t *testing.T) {
+	urls := []string{
+		"https://codeberg.org/dluvian/gitplaza.git",
+		"https://github.com/fiatjaf/pyramid.git",
+		"https://pyramid.fiatjaf.com/npub180cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsyjh6w6/nostrlib.git",
+	}
+
+	for _, url := range urls {
+		t.Run(url, func(t *testing.T) {
+			tree, err := GetDirectoryTreeAt(url, "refs/heads/master", nil)
+			require.NoError(t, err)
+			for _, file := range tree.Files {
+				require.Nil(t, file.Content, "file %q should have nil content at %s", file.Name, url)
+			}
+			require.Greater(t, len(tree.Directories), 2, "at %s", url)
+		})
+	}
+}
+
+func TestCloneRepositoryAtCurrentCommit(t *testing.T) {
+	urls := []string{
+		"https://codeberg.org/dluvian/gitplaza.git",
+		"https://github.com/fiatjaf/pyramid.git",
+		"https://pyramid.fiatjaf.com/npub180cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsyjh6w6/nostrlib.git",
+	}
+
+	for _, url := range urls {
+		t.Run(url, func(t *testing.T) {
+			commit, tree, err := ShallowCloneRepositoryAt(url, "refs/heads/master")
+			require.NoError(t, err)
+			require.Greater(t, len(tree.Files), 5, "at %s", url)
+			require.Greater(t, len(tree.Directories), 2, "at %s", url)
+
+			info, err := GetInfoRefs(url)
+			require.NoError(t, err)
+			require.Equal(t, info.Refs["refs/heads/master"], commit.Hash, "at %s", url)
+		})
+	}
+}
+
+func TestGetSpecificObject(t *testing.T) {
+	url := "https://codeberg.org/dluvian/gitplaza.git"
+	hash := "0f9438a8fd68594cd663fb8dbd23c5f5139f5263" // shell.nix
+
+	blob, err := GetObject(url, hash)
+	require.NoError(t, err)
+	require.NotNil(t, blob)
+	require.Equal(t, ObjectTypeBlob, blob.Type)
+
+	expected := "(builtins.getFlake\n  (\"git+file://\" + toString ./.)).devShells.${builtins.currentSystem}.default\n"
+	require.Equal(t, expected, string(blob.Data))
+}
+
+func TestGetNonExistentCommit(t *testing.T) {
+	urls := []string{
+		"https://codeberg.org/dluvian/gitplaza.git",
+		"https://pyramid.fiatjaf.com/npub180cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsyjh6w6/nostrlib.git",
+		"https://github.com/fiatjaf/nak.git",
+	}
+
+	commit := "1d4438a8fd68594cd663fb8dbd23c5f5139fabcd" // doesn't exist
+
+	for _, url := range urls {
+		t.Run(url, func(t *testing.T) {
+			_, err := GetDirectoryTreeAt(url, commit, nil)
+			require.Error(t, err)
+			var missingRef *MissingRef
+			require.ErrorAs(t, err, &missingRef)
+		})
+	}
+}
+
+func TestFetchListOfCommits(t *testing.T) {
+	commits, err := FetchCommitsOnly(
+		"https://pyramid.fiatjaf.com/npub180cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsyjh6w6/nostrlib.git",
+		"refs/heads/master",
+		nil,
+	)
+	require.NoError(t, err)
+	require.Greater(t, len(commits), 10)
+}
+
+func TestFetch10PastCommits(t *testing.T) {
+	maxCommits := 10
+	commits, err := FetchCommitsOnly(
+		"https://github.com/fiatjaf/pyramid.git",
+		"57712756e37d7c60d1ac53e0f6b59e9ecad67c9a",
+		&maxCommits,
+	)
+	require.NoError(t, err)
+	require.Len(t, commits, 10)
+
+	c := commits[1]
+	require.Equal(t, "49c1b48f5120bad4089535a190d2233c96188fa2", c.Hash)
+	require.Equal(t, "286786a6f1072a2ef5ae057fbb611858b8e88bc4", c.Tree)
+	require.Equal(t, []string{"1599e46c0ee6f460e25048880754868d4f9644fd"}, c.Parents)
+	require.Equal(t, "fiatjaf", c.Author.Name)
+	require.Equal(t, "fiatjaf@gmail.com", c.Author.Email)
+	require.Equal(t, int64(1767157644), c.Author.Timestamp)
+	require.Equal(t, "-0300", c.Author.Timezone)
+	require.Equal(t, "fiatjaf", c.Committer.Name)
+	require.Equal(t, "fiatjaf@gmail.com", c.Committer.Email)
+	require.Equal(t, int64(1767157724), c.Committer.Timestamp)
+	require.Equal(t, "-0300", c.Committer.Timezone)
+	require.Equal(t, "scheduled events.\n", c.Message)
+
+	expectedMsg5 := "turn off groups logic on QueryStore and PreventBroadcast when groups is turned off.\n\nthis was causing crashes that Golang's bizarre iter API showed as happening inside SortedMerge.\n"
+	require.Equal(t, expectedMsg5, commits[5].Message)
+}
+
+func TestGetSingleCommit(t *testing.T) {
+	url := "https://github.com/fiatjaf/pyramid.git"
+	commit, err := GetSingleCommit(url, "5e982dd1122a0bb1b0154c222ec4ba841f3820c6")
+	require.NoError(t, err)
+
+	require.Equal(t, "5e982dd1122a0bb1b0154c222ec4ba841f3820c6", commit.Hash)
+	require.Equal(t, "fiatjaf", commit.Author.Name)
+	require.Equal(t, "validate incoming git-related stuff.\n", commit.Message)
+}
+
+func TestGetDirectoryTreeWithDepthLimit(t *testing.T) {
+	url := "https://github.com/fiatjaf/pyramid.git"
+
+	fullTree, err := GetDirectoryTreeAt(url, "refs/heads/master", nil)
+	require.NoError(t, err)
+
+	depth := 1
+	shallowTree, err := GetDirectoryTreeAt(url, "refs/heads/master", &depth)
+	require.NoError(t, err)
+
+	require.Equal(t, len(fullTree.Directories), len(shallowTree.Directories))
+
+	for _, dir := range shallowTree.Directories {
+		require.NotNil(t, dir.Content, "directory %q content should not be nil at depth 1", dir.Name)
+		for _, file := range dir.Content.Files {
+			require.NotEmpty(t, file.Name)
+			require.Nil(t, file.Content, "file %q content should be nil", file.Name)
+		}
+		for _, subdir := range dir.Content.Directories {
+			require.NotEmpty(t, subdir.Name)
+			require.Nil(t, subdir.Content, "subdir %q content should be nil at depth 1", subdir.Name)
+		}
+	}
+
+	require.Equal(t, len(fullTree.Files), len(shallowTree.Files))
+}
+
+func TestGetObjectByPathExistingFile(t *testing.T) {
+	url := "https://codeberg.org/dluvian/gitplaza.git"
+	entry, err := GetObjectByPath(url, "refs/heads/master", "README.md")
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	require.Equal(t, "README.md", entry.Path)
+	require.False(t, entry.IsDir)
+	require.Equal(t, "100644", entry.Mode)
+	require.NotEmpty(t, entry.Hash)
+}
+
+func TestGetObjectByPathExistingDirectory(t *testing.T) {
+	url := "https://codeberg.org/dluvian/gitplaza.git"
+	entry, err := GetObjectByPath(url, "refs/heads/master", "src")
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	require.Equal(t, "src", entry.Path)
+	require.True(t, entry.IsDir)
+	require.Equal(t, "40000", entry.Mode)
+	require.NotEmpty(t, entry.Hash)
+}
+
+func TestGetObjectByPathNestedFile(t *testing.T) {
+	url := "https://github.com/fiatjaf/pyramid.git"
+	entry, err := GetObjectByPath(url, "d567c18cd5c144a58b0214216f454b3caf49d4ff", "grasp/grasp.templ")
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	require.Equal(t, "grasp.templ", entry.Path)
+	require.False(t, entry.IsDir)
+	require.Equal(t, "100644", entry.Mode)
+	require.Equal(t, "05bce14339ece5f48c670d0592faa8dece9e8957", entry.Hash)
+}
+
+func TestGetObjectByPathNonExistent(t *testing.T) {
+	url := "https://codeberg.org/dluvian/gitplaza.git"
+	entry, err := GetObjectByPath(url, "refs/heads/master", "whatever/something/x/y/z/non-existent-file.txt")
+	require.NoError(t, err)
+	require.Nil(t, entry)
+}
+
+func TestGetCommitDiff(t *testing.T) {
+	url := "https://github.com/smallhelm/diff-lines.git"
+	diff, err := GetCommitDiff(url, "a73592653fe9d01f948ca3035e088e45f722eca7")
+	require.NoError(t, err)
+	require.NotNil(t, diff)
+	require.Len(t, diff, 5)
+
+	byPath := make(map[string]DiffFile, len(diff))
+	for _, f := range diff {
+		byPath[f.Path] = f
+	}
+
+	for _, tc := range []struct {
+		path   string
+		status string
+	}{
+		{".travis.yml", "added"},
+		{".gitignore", "added"},
+		{"index.js", "added"},
+		{"tests.js", "added"},
+		{"package.json", "changed"},
+	} {
+		f, ok := byPath[tc.path]
+		require.True(t, ok, "missing diff file %q", tc.path)
+		require.Equal(t, tc.status, f.Status, "%s status", tc.path)
+	}
+
+	gitignore, ok := byPath[".gitignore"]
+	require.True(t, ok, "missing .gitignore in diff")
+	require.Equal(t, "/node_modules\n", string(gitignore.Content))
+
+	pkg, ok := byPath["package.json"]
+	require.True(t, ok, "missing package.json in diff")
+	require.NotEmpty(t, pkg.Lines, "package.json should have diff lines")
+
+	normalizeLineStatus := func(status string) string {
+		if status == "same" {
+			return "changed"
+		}
+		return status
+	}
+
+	lineByIndex := make(map[int]DiffLine, len(pkg.Lines))
+	for _, line := range pkg.Lines {
+		lineByIndex[line.Index] = line
+	}
+
+	expectedLines := []struct {
+		Index  int
+		Status string
+		Text   string
+	}{
+		{Index: 22, Status: "added", Text: "  },"},
+		{Index: 23, Status: "added", Text: "  \"homepage\": \"https://github.com/smallhelm/diff-lines#readme\","},
+		{Index: 24, Status: "added", Text: "  \"devDependencies\": {"},
+		{Index: 25, Status: "added", Text: "    \"tape\": \"^4.6.0\""},
+		{Index: 27, Status: "added", Text: "  \"dependencies\": {"},
+		{Index: 28, Status: "added", Text: "    \"diff\": \"^2.2.3\""},
+		{Index: 29, Status: "changed", Text: "  }"},
+	}
+
+	actualLines := make([]struct {
+		Index  int
+		Status string
+		Text   string
+	}, 0, len(expectedLines))
+	for _, expected := range expectedLines {
+		line, ok := lineByIndex[expected.Index]
+		require.True(t, ok, "missing package.json diff line %d", expected.Index)
+		actualLines = append(actualLines, struct {
+			Index  int
+			Status string
+			Text   string
+		}{
+			Index:  line.Index,
+			Status: normalizeLineStatus(line.Status),
+			Text:   line.Text,
+		})
+	}
+
+	require.Equal(t, expectedLines, actualLines)
+}

nip34/gitnaturalapi/packs.go

@@ -0,0 +1,154 @@
+package gitnaturalapi
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+var NecessaryCapabilities = []string{
+	"multi_ack_detailed",
+	"side-band-64k",
+}
+
+var RequiredCapabilities = []string{
+	"shallow",
+	"object-format=sha1",
+}
+
+var DefaultCapabilities = []string{
+	"ofs-delta",
+	"no-progress",
+}
+
+type MissingRef struct{}
+
+func (e *MissingRef) Error() string { return "missing ref" }
+
+type InvalidCommit struct {
+	Commit string
+}
+
+func (e *InvalidCommit) Error() string {
+	return fmt.Sprintf("invalid commit '%s', must be 20 byte hex", e.Commit)
+}
+
+func FetchPackfile(url string, want string) (*PackfileResult, error) {
+	req, err := http.NewRequest("POST", url+"/git-upload-pack", strings.NewReader(want))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create git-upload-pack request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
+	req.Header.Set("Accept", "application/x-git-upload-pack-result")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to call git-upload-pack: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("failed to call git-upload-pack: %s", string(body))
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read git-upload-pack response: %w", err)
+	}
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("empty response")
+	}
+
+	offset := 0
+	for offset < len(data) {
+		prev := offset
+		if prev+1 >= len(data) {
+			break
+		}
+		nlIdx := bytes.IndexByte(data[prev+1:], '\n')
+		if nlIdx == -1 {
+			if len(data) >= 32 && string(data[4:32]) == "ERR upload-pack: not our ref" {
+				return nil, &MissingRef{}
+			}
+			end := len(data)
+			if end > 63 {
+				end = 63
+			}
+			return nil, fmt.Errorf("unexpected '%s'", string(data[:end]))
+		}
+		offset = prev + nlIdx + 1
+		if offset >= 3 && string(data[offset-3:offset]) == "NAK" {
+			break
+		}
+	}
+	offset++
+
+	var packfileData []byte
+	for offset < len(data) {
+		if offset+5 > len(data) {
+			break
+		}
+		pktLen, err := strconv.ParseInt(string(data[offset:offset+4]), 16, 32)
+		if err != nil {
+			break
+		}
+		length := int(pktLen)
+		if length == 0 {
+			break
+		}
+		if offset+length > len(data) {
+			break
+		}
+		if data[offset+4] == 2 {
+			// progress message, ignore
+		} else if data[offset+4] == 1 {
+			packfileData = append(packfileData, data[offset+5:offset+length]...)
+		}
+		offset += length
+	}
+
+	if len(packfileData) == 0 {
+		return nil, &MissingRef{}
+	}
+
+	return ParsePackfile(packfileData)
+}
+
+func CreateWantRequest(commitSha string, capabilities []string, deepen *int, filter string) (string, error) {
+	if len(commitSha) != 40 {
+		return "", &InvalidCommit{Commit: commitSha}
+	}
+
+	var buf strings.Builder
+
+	wantLine := fmt.Sprintf("want %s %s agent=nsa/1.0.0\n", commitSha, strings.Join(capabilities, " "))
+	buf.WriteString(pktEncode(wantLine))
+
+	if deepen != nil {
+		deepenLine := fmt.Sprintf("deepen %d\n", *deepen)
+		buf.WriteString(pktEncode(deepenLine))
+	}
+
+	if filter != "" {
+		filterLine := fmt.Sprintf("filter %s\n", filter)
+		buf.WriteString(pktEncode(filterLine))
+	}
+
+	buf.WriteString("0000")
+	buf.WriteString(pktEncode("done\n"))
+
+	return buf.String(), nil
+}
+
+func pktEncode(data string) string {
+	if len(data) == 0 {
+		return "0000"
+	}
+	length := len(data) + 4
+	return fmt.Sprintf("%04x%s", length, data)
+}

nip34/gitnaturalapi/parse_packfile.go

@@ -0,0 +1,307 @@
+package gitnaturalapi
+
+import (
+	"bytes"
+	"compress/zlib"
+	"crypto/sha1"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"io"
+)
+
+const (
+	ObjectTypeCommit   = 1
+	ObjectTypeTree     = 2
+	ObjectTypeBlob     = 3
+	ObjectTypeTag      = 4
+	ObjectTypeOfsDelta = 6
+	ObjectTypeRefDelta = 7
+)
+
+type ParsedObject struct {
+	Type   int
+	Size   int
+	Data   []byte
+	Offset int
+	Hash   string
+}
+
+type PackfileResult struct {
+	Version int
+	Count   int
+	Objects map[string]*ParsedObject
+}
+
+func ParsePackfile(data []byte) (*PackfileResult, error) {
+	if len(data) < 12 {
+		return nil, fmt.Errorf("packfile too short")
+	}
+
+	header := string(data[0:4])
+	if header != "PACK" {
+		return nil, fmt.Errorf("invalid packfile header: %s", header)
+	}
+
+	version := int(binary.BigEndian.Uint32(data[4:8]))
+	if version != 2 {
+		return nil, fmt.Errorf("unsupported packfile version: %d", version)
+	}
+
+	count := int(binary.BigEndian.Uint32(data[8:12]))
+
+	objects := make(map[string]*ParsedObject)
+	pos := 12
+
+	for i := 0; i < count; i++ {
+		obj, newPos, err := parsePackObject(data, pos, objects)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing object %d/%d: %w", i+1, count, err)
+		}
+		objects[obj.Hash] = obj
+		pos = newPos
+	}
+
+	return &PackfileResult{
+		Version: version,
+		Count:   count,
+		Objects: objects,
+	}, nil
+}
+
+func parsePackObject(data []byte, startPos int, objects map[string]*ParsedObject) (*ParsedObject, int, error) {
+	pos := startPos
+	offset := startPos
+
+	b := data[pos]
+	pos++
+	objType := int((b >> 4) & 0x07)
+	size := int(b & 0x0f)
+	shift := 4
+
+	for b&0x80 != 0 {
+		b = data[pos]
+		pos++
+		size |= int(b&0x7f) << shift
+		shift += 7
+	}
+
+	var objData []byte
+	var err error
+
+	switch objType {
+	case ObjectTypeOfsDelta:
+		var actualType int
+		objData, pos, actualType, err = parseOfsDelta(data, pos, offset, objects)
+		if err != nil {
+			return nil, 0, err
+		}
+		objType = actualType
+	case ObjectTypeRefDelta:
+		var actualType int
+		objData, pos, actualType, err = parseRefDelta(data, pos, objects)
+		if err != nil {
+			return nil, 0, err
+		}
+		objType = actualType
+	case ObjectTypeCommit, ObjectTypeTree, ObjectTypeBlob, ObjectTypeTag:
+		objData, pos, err = zlibDecompress(data, pos)
+		if err != nil {
+			return nil, 0, err
+		}
+	default:
+		return nil, 0, fmt.Errorf("unknown object type: %d", objType)
+	}
+
+	hash, err := computeObjectHash(objType, objData)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return &ParsedObject{
+		Type:   objType,
+		Size:   size,
+		Data:   objData,
+		Offset: offset,
+		Hash:   hash,
+	}, pos, nil
+}
+
+func parseOfsDelta(data []byte, pos int, currentOffset int, objects map[string]*ParsedObject) ([]byte, int, int, error) {
+	b := data[pos]
+	pos++
+	offset := int(b & 0x7f)
+
+	for b&0x80 != 0 {
+		offset++
+		offset <<= 7
+		b = data[pos]
+		pos++
+		offset += int(b & 0x7f)
+	}
+
+	baseOffset := currentOffset - offset
+	baseObject, _, err := parsePackObject(data, baseOffset, objects)
+	if err != nil {
+		return nil, 0, 0, fmt.Errorf("failed to parse base object at offset %d: %w", baseOffset, err)
+	}
+
+	delta, newPos, err := zlibDecompress(data, pos)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	fullObj, err := applyDelta(delta, baseObject.Data)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	return fullObj, newPos, baseObject.Type, nil
+}
+
+func parseRefDelta(data []byte, pos int, objects map[string]*ParsedObject) ([]byte, int, int, error) {
+	baseName := hex.EncodeToString(data[pos : pos+20])
+	pos += 20
+
+	delta, newPos, err := zlibDecompress(data, pos)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	baseObject, ok := objects[baseName]
+	if !ok {
+		return nil, 0, 0, fmt.Errorf("base object not found with name %s", baseName)
+	}
+
+	fullObj, err := applyDelta(delta, baseObject.Data)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	return fullObj, newPos, baseObject.Type, nil
+}
+
+func computeObjectHash(objType int, data []byte) (string, error) {
+	var typeStr string
+	switch objType {
+	case ObjectTypeCommit:
+		typeStr = "commit"
+	case ObjectTypeTree:
+		typeStr = "tree"
+	case ObjectTypeBlob:
+		typeStr = "blob"
+	case ObjectTypeTag:
+		typeStr = "tag"
+	default:
+		return "", fmt.Errorf("unknown type when computing object hash: %d", objType)
+	}
+
+	header := fmt.Sprintf("%s %d\x00", typeStr, len(data))
+	h := sha1.New()
+	h.Write([]byte(header))
+	h.Write(data)
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+func applyDelta(delta []byte, base []byte) ([]byte, error) {
+	pos := 0
+
+	_, bytesRead := readVariableInt(delta, pos)
+	pos += bytesRead
+
+	resultSize, bytesRead := readVariableInt(delta, pos)
+	pos += bytesRead
+
+	result := make([]byte, resultSize)
+	resultOffset := 0
+
+	for pos < len(delta) {
+		cmd := delta[pos]
+		pos++
+
+		if cmd&0x80 != 0 {
+			var copyOffset, copySize int
+
+			if cmd&0x01 != 0 {
+				copyOffset = int(delta[pos])
+				pos++
+			}
+			if cmd&0x02 != 0 {
+				copyOffset |= int(delta[pos]) << 8
+				pos++
+			}
+			if cmd&0x04 != 0 {
+				copyOffset |= int(delta[pos]) << 16
+				pos++
+			}
+			if cmd&0x08 != 0 {
+				copyOffset |= int(delta[pos]) << 24
+				pos++
+			}
+
+			if cmd&0x10 != 0 {
+				copySize = int(delta[pos])
+				pos++
+			}
+			if cmd&0x20 != 0 {
+				copySize |= int(delta[pos]) << 8
+				pos++
+			}
+			if cmd&0x40 != 0 {
+				copySize |= int(delta[pos]) << 16
+				pos++
+			}
+
+			if copySize == 0 {
+				copySize = 0x10000
+			}
+
+			copy(result[resultOffset:], base[copyOffset:copyOffset+copySize])
+			resultOffset += copySize
+		} else if cmd > 0 {
+			copy(result[resultOffset:], delta[pos:pos+int(cmd)])
+			pos += int(cmd)
+			resultOffset += int(cmd)
+		} else {
+			return nil, fmt.Errorf("invalid delta command")
+		}
+	}
+
+	return result, nil
+}
+
+func zlibDecompress(data []byte, pos int) ([]byte, int, error) {
+	br := bytes.NewReader(data[pos:])
+	r, err := zlib.NewReader(br)
+	if err != nil {
+		return nil, 0, fmt.Errorf("zlib init error: %w", err)
+	}
+
+	decompressed, err := io.ReadAll(r)
+	r.Close()
+	if err != nil {
+		return nil, 0, fmt.Errorf("zlib decompress error: %w", err)
+	}
+
+	newPos := len(data) - br.Len()
+	return decompressed, newPos, nil
+}
+
+func readVariableInt(data []byte, pos int) (int, int) {
+	value := 0
+	shift := 0
+	bytesRead := 0
+
+	for {
+		b := data[pos]
+		pos++
+		bytesRead++
+		value |= int(b&0x7f) << shift
+		shift += 7
+		if b&0x80 == 0 {
+			break
+		}
+	}
+
+	return value, bytesRead
+}

nip34/gitnaturalapi/refs.go

@@ -0,0 +1,120 @@
+package gitnaturalapi
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type InfoRefsUploadPackResponse struct {
+	Refs         map[string]string
+	Capabilities []string
+	Symrefs      map[string]string
+}
+
+var capabilitiesCache sync.Map
+
+func GetCapabilities(url string, existingInfo *InfoRefsUploadPackResponse) ([]string, error) {
+	if existingInfo != nil {
+		capabilitiesCache.Store(url, existingInfo.Capabilities)
+		return existingInfo.Capabilities, nil
+	}
+
+	if cached, ok := capabilitiesCache.Load(url); ok {
+		return cached.([]string), nil
+	}
+
+	info, err := GetInfoRefs(url)
+	if err != nil {
+		return nil, err
+	}
+	capabilitiesCache.Store(url, info.Capabilities)
+	return info.Capabilities, nil
+}
+
+func GetInfoRefs(url string) (*InfoRefsUploadPackResponse, error) {
+	resp, err := http.Get(url + "/info/refs?service=git-upload-pack")
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch info/refs: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read info/refs response: %w", err)
+	}
+	response := string(body)
+
+	result := &InfoRefsUploadPackResponse{
+		Refs:    make(map[string]string),
+		Symrefs: make(map[string]string),
+	}
+
+	lines := strings.Split(response, "\n")
+	firstRef := true
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+
+		if strings.HasPrefix(line, "0000") {
+			line = line[4:]
+		}
+		if len(line) < 4 {
+			continue
+		}
+
+		length, err := strconv.ParseInt(line[:4], 16, 32)
+		if err != nil {
+			continue
+		}
+
+		endIdx := int(length)
+		if endIdx > len(line) {
+			endIdx = len(line)
+		}
+		if endIdx <= 4 {
+			continue
+		}
+		content := line[4:endIdx]
+
+		if firstRef && strings.HasPrefix(content, "# service=") {
+			firstRef = false
+			continue
+		}
+
+		if !strings.Contains(content, " ") {
+			continue
+		}
+
+		parts := strings.SplitN(content, " ", 2)
+		hash := parts[0]
+		refAndCaps := parts[1]
+
+		if strings.Contains(refAndCaps, "\x00") {
+			nulParts := strings.SplitN(refAndCaps, "\x00", 2)
+			ref := strings.TrimSpace(nulParts[0])
+			result.Refs[ref] = hash
+
+			caps := strings.Fields(nulParts[1])
+			result.Capabilities = caps
+
+			for _, cap := range caps {
+				if strings.HasPrefix(cap, "symref=") {
+					symrefData := cap[7:]
+					colonIdx := strings.Index(symrefData, ":")
+					if colonIdx != -1 {
+						result.Symrefs[symrefData[:colonIdx]] = symrefData[colonIdx+1:]
+					}
+				}
+			}
+		} else {
+			result.Refs[strings.TrimSpace(refAndCaps)] = hash
+		}
+	}
+
+	return result, nil
+}

nip34/gitnaturalapi/tree.go

@@ -0,0 +1,101 @@
+package gitnaturalapi
+
+import "encoding/hex"
+
+type TreeEntry struct {
+	Path  string
+	Mode  string
+	IsDir bool
+	Hash  string
+}
+
+type TreeFile struct {
+	Name    string
+	Hash    string
+	Content []byte
+}
+
+type TreeDirectory struct {
+	Name    string
+	Hash    string
+	Content *Tree
+}
+
+type Tree struct {
+	Directories []TreeDirectory
+	Files       []TreeFile
+}
+
+func LoadTree(obj *ParsedObject, objects map[string]*ParsedObject, depth *int) *Tree {
+	directories := make([]TreeDirectory, 0)
+	files := make([]TreeFile, 0)
+	entries := ParseTree(obj.Data)
+
+	for _, entry := range entries {
+		child := objects[entry.Hash]
+
+		if entry.IsDir {
+			var content *Tree
+			if child != nil && (depth == nil || *depth > 0) {
+				var newDepth *int
+				if depth != nil {
+					d := *depth - 1
+					newDepth = &d
+				}
+				content = LoadTree(child, objects, newDepth)
+			}
+			directories = append(directories, TreeDirectory{
+				Name:    entry.Path,
+				Hash:    entry.Hash,
+				Content: content,
+			})
+		} else {
+			var content []byte
+			if child != nil {
+				content = child.Data
+			}
+			files = append(files, TreeFile{
+				Name:    entry.Path,
+				Hash:    entry.Hash,
+				Content: content,
+			})
+		}
+	}
+
+	return &Tree{Directories: directories, Files: files}
+}
+
+func ParseTree(treeData []byte) []TreeEntry {
+	entries := make([]TreeEntry, 0)
+	offset := 0
+
+	for offset < len(treeData) {
+		modeEnd := offset
+		for treeData[modeEnd] != 0x20 {
+			modeEnd++
+		}
+		mode := string(treeData[offset:modeEnd])
+		offset = modeEnd + 1
+
+		filenameEnd := offset
+		for treeData[filenameEnd] != 0x00 {
+			filenameEnd++
+		}
+		path := string(treeData[offset:filenameEnd])
+		offset = filenameEnd + 1
+
+		hash := hex.EncodeToString(treeData[offset : offset+20])
+		offset += 20
+
+		isDir := mode == "40000" || mode == "040000"
+
+		entries = append(entries, TreeEntry{
+			Mode:  mode,
+			Path:  path,
+			Hash:  hash,
+			IsDir: isDir,
+		})
+	}
+
+	return entries
+}

nip34/grasp/helpers.go

@@ -0,0 +1,25 @@
+package grasp
+
+import (
+	"net/url"
+	"strings"
+
+	"fiatjaf.com/nostr/nip19"
+)
+
+func IsGraspURL(u string) bool {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return false
+	}
+
+	if strings.Count(parsed.Path, "/") != 2 || len(parsed.Path) < 65 {
+		return false
+	}
+
+	if prefix, _, err := nip19.Decode(parsed.Path[1:64]); err != nil || prefix != "npub" {
+		return false
+	}
+
+	return true
+}

nip46/client.go

@@ -6,7 +6,9 @@ import (
 	"math/rand"
 	"net/url"
 	"strconv"
+	"sync"
 	"sync/atomic"
+	"time"
 	"unsafe"
 
 	"fiatjaf.com/nostr"
@@ -15,6 +17,12 @@ import (
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
+var bunkerClientCtxKey = &struct{}{}
+
+func IsBunkerClientOperation(ctx context.Context) bool {
+	return ctx.Value(bunkerClientCtxKey) == true
+}
+
 type BunkerClient struct {
 	Relays []string
 
@@ -56,6 +64,7 @@ func ConnectBunker(
 		pool,
 		onAuth,
 	)
+
 	_, err = bunker.RPC(ctx, "connect", []string{nostr.HexEncodeToString(parsed.HostPubKey[:]), parsed.Secret})
 	return bunker, err
 }
@@ -112,7 +121,7 @@ func NewBunker(
 	onAuth func(string),
 ) *BunkerClient {
 	if pool == nil {
-		pool = nostr.NewPool(nostr.PoolOptions{})
+		pool = nostr.NewPool()
 	}
 
 	clientPublicKey := nostr.GetPublicKey(clientSecretKey)
@@ -131,10 +140,10 @@ func NewBunker(
 	}
 
 	cancellableCtx, cancel := context.WithCancel(ctx)
+	bunkerClientCtx := context.WithValue(cancellableCtx, bunkerClientCtxKey, true)
 	_ = cancel
 
-	go func() {
+	events, eosed := pool.SubscribeManyNotifyEOSE(bunkerClientCtx, relays, nostr.Filter{
-		events := pool.SubscribeMany(cancellableCtx, relays, nostr.Filter{
 		Tags:      nostr.TagMap{"p": []string{clientPublicKey.Hex()}},
 		Kinds:     []nostr.Kind{nostr.KindNostrConnect},
 		Since:     now,
@@ -142,6 +151,8 @@ func NewBunker(
 	}, nostr.SubscriptionOptions{
 		Label: "bunker46client",
 	})
+
+	go func() {
 		for ie := range events {
 			if ie.Kind != nostr.KindNostrConnect {
 				continue
@@ -174,12 +185,15 @@ func NewBunker(
 
 	// attempt switch_relays once every 10 times
 	if now%10 == 0 {
-		if newRelays, _ := bunker.SwitchRelays(ctx); newRelays != nil {
+		swctx, cancel := context.WithTimeout(ctx, time.Second*3)
-			cancel()
+		if newRelays, _ := bunker.SwitchRelays(swctx); newRelays != nil {
 			bunker = NewBunker(ctx, clientSecretKey, targetPublicKey, newRelays, pool, func(string) {})
 		}
+		cancel()
 	}
 
+	<-eosed
+
 	return bunker
 }
 
@@ -274,7 +288,7 @@ func (bunker *BunkerClient) NIP04Decrypt(
 }
 
 func (bunker *BunkerClient) RPC(ctx context.Context, method string, params []string) (string, error) {
-	id := bunker.idPrefix + "-" + strconv.FormatUint(bunker.serial.Add(1), 10)
+	id := bunker.idPrefix + "-" + strconv.FormatUint(bunker.serial.Add(1), 10) + "-" + method
 	req, err := json.Marshal(Request{
 		ID:     id,
 		Method: method,
@@ -303,21 +317,23 @@ func (bunker *BunkerClient) RPC(ctx context.Context, method string, params []str
 	bunker.listeners.Store(id, dispatcher)
 	defer bunker.listeners.Delete(id)
 	relayConnectionWorked := make(chan struct{})
+	relayConnectionWorkedO := sync.OnceFunc(func() {
+		close(relayConnectionWorked)
+	})
 	bunkerConnectionWorked := make(chan struct{})
+	bunkerConnectionWorkedO := sync.OnceFunc(func() {
+		close(bunkerConnectionWorked)
+	})
+
+	bunkerClientCtx := context.WithValue(ctx, bunkerClientCtxKey, true)
 
 	for _, url := range bunker.Relays {
 		go func(url string) {
 			relay, err := bunker.pool.EnsureRelay(url)
 			if err == nil {
-				select {
+				relayConnectionWorkedO()
-				case relayConnectionWorked <- struct{}{}:
+				if err := relay.Publish(bunkerClientCtx, evt); err == nil {
-				default:
+					bunkerConnectionWorkedO()
-				}
-				if err := relay.Publish(ctx, evt); err == nil {
-					select {
-					case bunkerConnectionWorked <- struct{}{}:
-					default:
-					}
 				}
 			}
 		}(url)

nip46/dynamic-signer.go

@@ -3,6 +3,8 @@ package nip46
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"strconv"
 	"sync"
 
 	"fiatjaf.com/nostr"
@@ -50,6 +52,50 @@ func (p *DynamicSigner) Init() {
 	p.sessions = make(map[nostr.PubKey]map[nostr.PubKey]*Session)
 }
 
+// HandleNostrConnectURI works like HandleRequest, but takes a nostrconnect:// URI as input, as scanned/pasted
+// by the user, produced by the client. Since DynamicSigner can serve multiple handler keys, the caller must
+// specify which handlerPubkey should respond to this connection.
+func (p *DynamicSigner) HandleNostrConnectURI(ctx context.Context, handlerPubkey nostr.PubKey, uri *url.URL) (
+	resp Response,
+	eventResponse nostr.Event,
+	err error,
+) {
+	clientPublicKey, err := nostr.PubKeyFromHex(uri.Host)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+
+	secret := uri.Query().Get("secret")
+
+	_, handlerSecret, err := p.GetHandlerSecretKey(ctx, handlerPubkey)
+	if err != nil {
+		return resp, eventResponse, fmt.Errorf("no private key for %s: %w", handlerPubkey, err)
+	}
+
+	// pretend they started with a request
+	conversationKey, err := nip44.GenerateConversationKey(clientPublicKey, handlerSecret)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+	reqj, _ := json.Marshal(Request{
+		ID:     "nostrconnect-" + strconv.FormatInt(int64(nostr.Now()), 10),
+		Method: "imagined-nostrconnect",
+		Params: []string{clientPublicKey.Hex(), secret},
+	})
+	ciphertext, err := nip44.Encrypt(string(reqj), conversationKey)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+
+	_, resp, eventResponse, err = p.HandleRequest(ctx, nostr.Event{
+		PubKey:  clientPublicKey,
+		Kind:    nostr.KindNostrConnect,
+		Content: ciphertext,
+		Tags:    nostr.Tags{nostr.Tag{"p", handlerPubkey.Hex()}},
+	})
+	return resp, eventResponse, err
+}
+
 func (p *DynamicSigner) HandleRequest(ctx context.Context, event nostr.Event) (
 	req Request,
 	resp Response,
@@ -118,6 +164,19 @@ func (p *DynamicSigner) HandleRequest(ctx context.Context, event nostr.Event) (
 	var resultErr error
 
 	switch req.Method {
+	case "imagined-nostrconnect":
+		// this is a fake request we pretend has existed, but was actually just we reading the nostrconnect:// uri
+		if len(req.Params) < 2 || req.Params[1] == "" {
+			resultErr = fmt.Errorf("needs a second argument 'secret'")
+			break
+		}
+		if p.OnConnect != nil {
+			if err := p.OnConnect(ctx, event.PubKey, req.Params[1]); err != nil {
+				resultErr = err
+				break
+			}
+		}
+		result = req.Params[1]
 	case "connect":
 		var secret string
 		if len(req.Params) >= 2 {

nip46/nostrconnect.go

@@ -67,7 +67,7 @@ func NewBunkerFromNostrConnect(
 	pool *nostr.Pool,
 ) (*BunkerClient, error) {
 	if pool == nil {
-		pool = nostr.NewPool(nostr.PoolOptions{})
+		pool = nostr.NewPool()
 	}
 
 	if len(relayURLs) == 0 {

nip54/nip54.go

@@ -11,20 +11,24 @@ import (
 )
 
 func NormalizeIdentifier(name string) string {
-	name = strings.TrimSpace(strings.ToLower(name))
 	res, _, _ := transform.Bytes(norm.NFKC, []byte(name))
-	runes := []rune(string(res))
+	runes := []rune(strings.ToLower(string(res)))
 
-	b := make([]rune, len(runes))
+	words := make([]string, 0, 3)
-	for i, letter := range runes {
+	word := make([]rune, 0, 12)
+	for _, letter := range runes {
 		if unicode.IsLetter(letter) || unicode.IsNumber(letter) {
-			b[i] = letter
+			word = append(word, letter)
-		} else {
+		} else if len(word) > 0 {
-			b[i] = '-'
+			words = append(words, string(word))
+			word = make([]rune, 0, 12)
 		}
 	}
+	if len(word) > 0 {
+		words = append(words, string(word))
+	}
 
-	return string(b)
+	return strings.Join(words, "-")
 }
 
 func ArticleAsHTML(content string) string {

nip54/nip54_test.go

@@ -13,7 +13,7 @@ func TestNormalization(t *testing.T) {
 	}{
 		{" hello  ", "hello"},
 		{"Goodbye", "goodbye"},
-		{"the long and winding road / that leads to your door", "the-long-and-winding-road---that-leads-to-your-door"},
+		{"the long and winding road / that leads to your door", "the-long-and-winding-road-that-leads-to-your-door"},
 		{"it's 平仮名", "it-s-平仮名"},
 	} {
 		if norm := NormalizeIdentifier(vector.before); norm != vector.after {