Add support for roles in nip 86

fixes nostr:nevent1qvzqqqqqqypzqtrucc4xjl4r57px2g0nl560pje8x6fuhe0fxy8n23ylgd3z5hxuqythwumn8ghj7un9d3shjtnswf5k6ctv9ehx2ap0qy88wumn8ghj7mn0wvhxcmmv9uqzpctrpfqmk9nqzdlvemhgrskz6yyer4j93gudlrlf7jn2qafys9pzyw9u5s

event_easyjson.go

@@ -74,6 +74,8 @@ func easyjsonDecodeEvent(in *jlexer.Lexer, out *Event) {
 			if len(b) == 128 {
 				xhex.Decode(out.Sig[:], b)
 			}
+		default:
+			in.SkipRecursive()
 		}
 		in.WantComma()
 	}

eventstore/bleve/bleve_test.go

@@ -2,10 +2,13 @@ package bleve
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/eventstore/lmdb"
+	"fiatjaf.com/nostr/eventstore/slicestore"
+	"github.com/pemistahl/lingua-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -21,6 +24,7 @@ func TestBleveFlow(t *testing.T) {
 	bl := BleveBackend{
 		Path:          "/tmp/blevetest-bleve",
 		RawEventStore: bb,
+		Languages:     []lingua.Language{lingua.English},
 	}
 	err := bl.Init()
 	require.NoError(t, err, "init")
@@ -74,3 +78,184 @@ func TestBleveFlow(t *testing.T) {
 		assert.Equal(t, 1, n)
 	}
 }
+
+func TestSearch(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "test_search_pyramid")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	db := &slicestore.SliceStore{}
+	db.Init()
+
+	index := &BleveBackend{
+		Path:          filepath.Join(tempDir, "test_index"),
+		RawEventStore: db,
+		Languages: []lingua.Language{
+			lingua.English,
+			lingua.Portuguese,
+			lingua.Italian,
+		},
+	}
+	err = index.Init()
+	require.NoError(t, err)
+	defer index.Close()
+
+	pirateEvents := []nostr.Event{
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			CreatedAt: nostr.Timestamp(1609459200),
+			Kind:      1,
+			Content:   "Ahoy mateys! I've discovered a treasure chest filled with gold doubloons and silver pieces buried beneath the old palm tree on Skull Island. The secret map shows an X marks the spot where the legendary pirate Blackbeard hid his most valuable plunder. The chest contains rubies, emeralds, and ancient coins from sunken Spanish galleons. https://www.youtube.com/watch?v=enTAromEeHo&t=88s",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000002"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001"),
+			CreatedAt: nostr.Timestamp(1609545600),
+			Kind:      1111,
+			Content:   "The treasure map I found reveals the location of Captain Morgan's lost gold mine deep in the Caribbean waters. Following the ancient compass directions leads to a hidden cave filled with golden artifacts, jeweled swords, and the crown jewels of forgotten kingdoms. The secret passage is guarded by mysterious symbols only known to the brotherhood of the sea. https://www.youtube.com/watch?v=yBtyNIqZios",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000003"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000003"),
+			CreatedAt: nostr.Timestamp(1609632000),
+			Kind:      1,
+			Content:   "Legends speak of the Emerald City of the Lost Pirates, a mythical place where streets are paved with gold and buildings adorned with precious gems. The secret entrance can only be found during a full moon when the tides reveal a hidden path across the coral reefs. Ancient scrolls tell of guardians protecting treasure vaults containing the world's most valuable gems.",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000004"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000004"),
+			CreatedAt: nostr.Timestamp(1609545601),
+			Kind:      1111,
+			Content:   "Bom dia seus piratas melequentos, onde está esse bendito tesouro? nostr:nprofile1qqsv6jemsnaq925ddfqjhwm3du3k0zk7dnj2ksk2k4hcfkf80mzf56spz9mhxue69uhkzcnpvdshgefwvdhk6tmjzyj",
+			Tags:      nil,
+		},
+		{
+			ID:        nostr.MustIDFromHex("0000000000000000000000000000000000000000000000000000000000000005"),
+			PubKey:    nostr.MustPubKeyFromHex("0000000000000000000000000000000000000000000000000000000000000005"),
+			CreatedAt: nostr.Timestamp(1609545602),
+			Kind:      30023,
+			Content:   "I pirati dei Caraibi del XVII e XVIII secolo sono diventati leggendari per la loro ricerca di tesori. Questi avventurieri del mare saccheggiavano navi cariche d'oro, argento e pietre preziose provenienti dalle colonie spagnole del Nuovo Mondo.\n\nSecondo la leggenda, molti pirati seppellivano i loro tesori su isole remote, creando mappe segrete con la famosa \"X\" che segnava il punto. Capitani famosi come Barbanera, Capitan Kidd e Henry Morgan sono entrati nell'immaginario collettivo come custodi di ricchezze nascoste.\n\nAnche se la maggior parte dei tesori dei pirati sono probabilmente solo miti, alcuni sono stati davvero ritrovati. Il fascino di questi bottini nascosti continua ad ispirare storie, film e cacciatori di tesori ancora oggi.",
+			Tags:      nil,
+		},
+	}
+
+	for _, event := range pirateEvents {
+		err := db.SaveEvent(event)
+		require.NoError(t, err)
+		err = index.SaveEvent(event)
+		require.NoError(t, err)
+	}
+
+	testCases := []struct {
+		name     string
+		filter   nostr.Filter
+		expected int
+	}{
+		{
+			name: "search for 'gold'",
+			filter: nostr.Filter{
+				Search: "gold",
+			},
+			expected: 3, // all events mention gold
+		},
+		{
+			name: "search for 'treasure'",
+			filter: nostr.Filter{
+				Search: "treasure",
+			},
+			expected: 3, // all events mention treasure
+		},
+		{
+			name: "search for 'emerald' together with 'astronomical'",
+			filter: nostr.Filter{
+				Search: "astronomical emeralds",
+			},
+			expected: 0, // no events mention emeralds together with astronomical
+		},
+		{
+			name: "search for 'secret map'",
+			filter: nostr.Filter{
+				Search: "\"secret map\"",
+			},
+			expected: 1, // only one event mentions secret map
+		},
+		{
+			name: "search with kind filter",
+			filter: nostr.Filter{
+				Search: "gold",
+				Kinds:  []nostr.Kind{1},
+			},
+			expected: 2, // only two events are kind 1
+		},
+		{
+			name: "search in portuguese",
+			filter: nostr.Filter{
+				Search: "melequento",
+			},
+			expected: 1,
+		},
+		{
+			name: "search with exact match",
+			filter: nostr.Filter{
+				Search: "\"the secret entrance can only be found during a full moon\"",
+			},
+			expected: 1,
+		},
+		{
+			name: "search with OR across languages",
+			filter: nostr.Filter{
+				Search: "melequento OR matey",
+			},
+			expected: 2,
+		},
+		{
+			name: "search with exact reference found in the text",
+			filter: nostr.Filter{
+				Search: "tesouro nostr:nprofile1qqsv6jemsnaq925ddfqjhwm3du3k0zk7dnj2ksk2k4hcfkf80mzf56spzpmhxue69uhkyctwv9hxztnrdaksmfp5mw", // this is the same pubkey from above, but it's a different nprofile
+			},
+			expected: 1,
+		},
+		{
+			name: "search for URL",
+			filter: nostr.Filter{
+				Search: "https://www.youtube.com/watch?v=yBtyNIqZios treasure",
+			},
+			expected: 1,
+		},
+		{
+			name: "search for host/domain of URL",
+			filter: nostr.Filter{
+				Search: "www.youtube.com",
+			},
+			expected: 2,
+		},
+		{
+			name: "mentioning the author should include their notes in the result",
+			filter: nostr.Filter{
+				Search: " nostr:npub1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqshp52w2",
+			},
+			expected: 2,
+		},
+		{
+			name: "mentioning the author should include their notes in the result",
+			filter: nostr.Filter{
+				Search: "found gold? nostr:npub1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqshp52w2",
+			},
+			expected: 1,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var count int
+			for range index.QueryEvents(tc.filter, 10) {
+				count++
+			}
+			require.Equal(t, tc.expected, count)
+		})
+	}
+}

eventstore/cmd/eventstore/main.go

@@ -19,7 +19,7 @@ import (
 
 var (
 	db  eventstore.Store
-	end func()
+	end = func() {}
 )
 
 var app = &cli.Command{

eventstore/mmm/freeranges.go

@@ -2,14 +2,32 @@ package mmm
 
 import (
 	"cmp"
+	"encoding/binary"
 	"fmt"
+	"iter"
+	"runtime"
 	"slices"
+	"syscall"
+	"unsafe"
 
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore/codec/betterbinary"
 	"github.com/PowerDNS/lmdb-go/lmdb"
 )
 
 const LARGE_FREERANGE = 142
 
+// AllFreeRanges returns an iterator of (start_pos, size) of all free ranges, in positional order.
+func (b *MultiMmapManager) AllFreeRanges() iter.Seq2[uint64, uint32] {
+	return func(yield func(uint64, uint32) bool) {
+		for _, pos := range b.freeRangesAll {
+			if !yield(pos.start, pos.size) {
+				return
+			}
+		}
+	}
+}
+
 func (b *MultiMmapManager) gatherFreeRanges(txn *lmdb.Txn) error {
 	cursor, err := txn.OpenCursor(b.indexId)
 	if err != nil {
@@ -150,3 +168,184 @@ func (b *MultiMmapManager) mergeNewFreeRange(newFreeRange position) {
 		}
 	}
 }
+
+func (b *MultiMmapManager) Defragment(n int) error {
+	for range min(n, len(b.freeRangesAll)-1) {
+		if err := b.DefragmentOne(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Defragment a single free range
+func (b *MultiMmapManager) DefragmentOne() error {
+	if b.ReadOnly {
+		return ReadOnly
+	}
+
+	b.writeMutex.Lock()
+	defer b.writeMutex.Unlock()
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	if len(b.freeRangesAll) < 2 {
+		return nil
+	}
+
+	mmmtxn, err := b.lmdbEnv.BeginTxn(nil, 0)
+	if err != nil {
+		return fmt.Errorf("failed to begin mmm transaction: %w", err)
+	}
+	defer mmmtxn.Abort()
+
+	type layerTxn struct {
+		il  *IndexingLayer
+		txn *lmdb.Txn
+	}
+	layerTxns := make(map[uint16]*layerTxn)
+	defer func() {
+		for _, lt := range layerTxns {
+			lt.txn.Abort()
+		}
+	}()
+
+	// will put stuff into the first free range
+	fr := b.freeRangesAll[0]
+
+	// where the free range ends, the events start (any number of them)
+	eventsStart := fr.start + uint64(fr.size)
+	eventsEnd := b.freeRangesAll[1].start // and they end when the next free range starts
+
+	fmt.Println("# defrag", fr, eventsStart, eventsEnd)
+
+	c := uint64(0) // this tracks our relative position inside the events section
+	for (eventsStart + c) < eventsEnd {
+		var evt nostr.Event
+		if err := betterbinary.Unmarshal(b.mmapf[(eventsStart+c):eventsEnd], &evt); err != nil {
+			id := betterbinary.GetID(b.mmapf[(eventsStart + c):eventsEnd])
+			return fmt.Errorf("failed to read event (%x) from mmap: %w", id[:], err)
+		}
+
+		// now that we have an event we'll update its pos on the id index and on every layer:
+		oldVal, err := mmmtxn.Get(b.indexId, evt.ID[0:8])
+		if err != nil {
+			return fmt.Errorf("failed to read val (%x) from index: %w", evt.ID[:], err)
+		}
+
+		// current position
+		pos := positionFromBytes(oldVal[0:12])
+
+		// new position (from the beginning of the free range before + relative position)
+		fmt.Println("  moving event", evt.ID, "from", pos)
+		pos.start = fr.start + uint64(c)
+
+		// update this cursor
+		c += uint64(pos.size)
+		fmt.Println("  to", pos, "...", c, "layers:", oldVal[12:])
+
+		// prepare and save id index
+		newVal := make([]byte, len(oldVal))
+		writeBytesFromPosition(newVal, pos)
+		copy(newVal[12:], oldVal[12:])
+		if err := mmmtxn.Put(b.indexId, evt.ID[0:8], newVal, 0); err != nil {
+			return fmt.Errorf("failed to write new pos to id index: %w", err)
+		}
+
+		for s := 12; s < len(oldVal); s += 2 {
+			layer := binary.BigEndian.Uint16(oldVal[s : s+2])
+			lt, ok := layerTxns[layer]
+			if !ok {
+				il := b.layers.ByID(layer)
+				if il == nil {
+					fmt.Println(b.layers)
+					panic(fmt.Errorf("missing layer %d", layer))
+				}
+				txn, err := il.lmdbEnv.BeginTxn(nil, 0)
+				if err != nil {
+					return fmt.Errorf("failed to begin layer txn for layer %d: %w", il.id, err)
+				}
+				txn.RawRead = true
+				lt = &layerTxn{il: il, txn: txn}
+				layerTxns[il.id] = lt
+			}
+
+			fmt.Println("    layer", lt.il.id)
+
+			for k := range lt.il.getIndexKeysForEvent(evt) {
+				fmt.Println("      index", k.dbi, k.key)
+				if err := lt.txn.Del(k.dbi, k.key, oldVal[0:12]); err != nil {
+					return fmt.Errorf("failed to delete old index entry for %x: %w", evt.ID[:], err)
+				}
+				if err := lt.txn.Put(k.dbi, k.key, newVal[0:12], 0); err != nil {
+					return fmt.Errorf("failed to insert new index entry for %x: %w", evt.ID[:], err)
+				}
+			}
+		}
+	}
+
+	// now that we have updated all the pointers, just copy all the bytes between the two free ranges
+	copy(b.mmapf[fr.start:], b.mmapf[fr.start+uint64(fr.size):eventsEnd])
+
+	// delete this free range if it's one of the big ones
+	if fr.isLarge() {
+		for l, lfr := range b.freeRangesLarge {
+			if lfr.start == fr.start {
+				fmt.Println("  deleting large fr", l, lfr)
+				b.freeRangesLarge[l] = b.freeRangesLarge[len(b.freeRangesLarge)-1]
+				b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
+				break
+			}
+		}
+	}
+
+	// now we have some space left at the end of this events section that is a free range
+	remainingSpaceStart := fr.start + c
+
+	// it must be merged with the next free range
+	updated := position{
+		start: remainingSpaceStart,
+		size:  b.freeRangesAll[1].size + uint32(eventsEnd) - uint32(remainingSpaceStart),
+	}
+	nextWasLarge := b.freeRangesAll[1].isLarge()
+	fmt.Println("  updating next", updated)
+	b.freeRangesAll[1] = updated
+
+	if nextWasLarge {
+		for l, lfr := range b.freeRangesLarge {
+			if lfr.start == eventsEnd {
+				fmt.Println("it is large:", l, lfr, "(now", updated, ")")
+				b.freeRangesLarge[l] = updated
+				break
+			}
+		}
+	} else if updated.isLarge() {
+		// if it wasn't large but now is, add it to the list of large free ranges
+		fmt.Println("  a new large fr was created", updated)
+		b.freeRangesLarge = append(b.freeRangesLarge, updated)
+	}
+
+	// msync
+	_, _, errno := syscall.Syscall(syscall.SYS_MSYNC,
+		uintptr(unsafe.Pointer(&b.mmapf[0])), uintptr(len(b.mmapf)), syscall.MS_SYNC)
+	if errno != 0 {
+		return fmt.Errorf("msync failed: %w", syscall.Errno(errno))
+	}
+
+	// commit transactions
+	if err := mmmtxn.Commit(); err != nil {
+		return fmt.Errorf("failed to commit mmm transaction: %w", err)
+	}
+	for lid, lt := range layerTxns {
+		if err := lt.txn.Commit(); err != nil {
+			return fmt.Errorf("failed to commit layer %d transaction: %w", lid, err)
+		}
+	}
+
+	// delete the first free range
+	b.freeRangesAll = slices.Delete(b.freeRangesAll, 0, 1)
+
+	return nil
+}

eventstore/mmm/freeranges_test.go

@@ -1,8 +1,11 @@
 package mmm
 
 import (
+	"cmp"
+	"fmt"
 	"math/rand/v2"
 	"os"
+	"slices"
 	"strings"
 	"testing"
 
@@ -125,6 +128,100 @@ func FuzzFreeRanges(f *testing.F) {
 	})
 }
 
+func TestDefragment(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "mmm-defrag-test-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	mmmm := &MultiMmapManager{Dir: tmpDir}
+	err = mmmm.Init()
+	require.NoError(t, err)
+	defer mmmm.Close()
+
+	il, err := mmmm.EnsureLayer("a")
+	require.NoError(t, err)
+	defer il.Close()
+
+	sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
+
+	const nevents = 30
+	var stored [nevents]nostr.Event
+	for i := range nevents {
+		evt := nostr.Event{
+			CreatedAt: nostr.Timestamp(i),
+			Kind:      nostr.KindTextNote,
+			Tags:      nostr.Tags{},
+			Content:   fmt.Sprintf("============= event %d ============= "+strings.Repeat("+", 23), i),
+		}
+		evt.Sign(sk)
+		err := il.SaveEvent(evt)
+		require.NoError(t, err)
+		stored[i] = evt
+	}
+
+	toDelete := []int{0, 5, 10, 15, 20}
+	var remaining []nostr.Event
+	for i, evt := range stored {
+		if slices.Contains(toDelete, i) {
+			err := il.DeleteEvent(evt.ID)
+			require.NoError(t, err)
+		} else {
+			remaining = append(remaining, evt)
+		}
+	}
+
+	require.Len(t, toDelete, len(mmmm.freeRangesAll))
+
+	err = mmmm.Defragment(2)
+	require.NoError(t, err)
+
+	require.Len(t, mmmm.freeRangesAll, 3)
+	require.Len(t, remaining, nevents-len(toDelete))
+
+	// all remaining events still accessible with correct content via GetByID
+	for _, evt := range remaining {
+		gotEvt, layers := mmmm.GetByID(evt.ID)
+		require.NotNil(t, gotEvt, "event %s should exist after defrag", evt.ID)
+		require.NotEmpty(t, layers, "event %s should have layers after defrag", evt.ID)
+		require.Equal(t, evt.Content, gotEvt.Content, "event %s content should match after defrag", evt.ID)
+
+		// also accessible via a query
+		require.Equal(t, il, layers[0])
+	}
+
+	evts := slices.Collect(il.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{nostr.KindTextNote}}, 100))
+	require.Len(t, evts, nevents-len(toDelete))
+
+	// free range invariants hold after defrag
+	verifyFreeRangesInvariants(t, mmmm)
+
+	// no overlapping positions after defrag
+	mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+		cursor, err := txn.OpenCursor(mmmm.indexId)
+		require.NoError(t, err)
+		defer cursor.Close()
+
+		var allPositions []position
+		for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
+			pos := positionFromBytes(val[0:12])
+			allPositions = append(allPositions, pos)
+		}
+
+		slices.SortFunc(allPositions, func(a, b position) int {
+			return cmp.Compare(a.start, b.start)
+		})
+
+		var lastEnd uint64
+		for _, pos := range allPositions {
+			if pos.start < lastEnd {
+				t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
+			}
+			lastEnd = pos.start + uint64(pos.size)
+		}
+		return nil
+	})
+}
+
 func countUsableFreeRanges(t *testing.T, mmmm *MultiMmapManager) (count int, space int) {
 	for _, fr := range mmmm.freeRangesAll {
 		if fr.size >= LARGE_FREERANGE {
@@ -142,7 +239,13 @@ func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
 	all := mmmm.freeRangesAll
 	large := mmmm.freeRangesLarge
 
+	require.True(t, slices.IsSortedFunc(all, func(a, b position) int {
+		return cmp.Compare(a.start, b.start)
+	}), "free ranges aren't sorted by start position")
+
 	for _, l := range large {
+		require.True(t, l.isLarge())
+
 		found := false
 		for _, a := range all {
 			if l.start == a.start && l.size == a.size {
@@ -157,7 +260,7 @@ func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
 		require.Greater(t, all[i].start, all[i-1].start, "all ranges should be sorted by start")
 	}
 
-	for i := range all {
+	for i, fr := range all {
 		for j := i + 1; j < len(all); j++ {
 			end1 := all[i].start + uint64(all[i].size)
 			end2 := all[j].start + uint64(all[j].size)
@@ -165,6 +268,17 @@ func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
 				(all[j].start >= all[i].start && all[j].start < end1),
 				"ranges %v and %v overlap", all[i], all[j])
 		}
+
+		foundInLarge := false
+		for _, l := range large {
+			if l.start == fr.start {
+				foundInLarge = true
+				break
+			}
+		}
+		if !foundInLarge {
+			require.False(t, fr.isLarge())
+		}
 	}
 
 	mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
@@ -176,3 +290,191 @@ func verifyFreeRangesInvariants(t *testing.T, mmmm *MultiMmapManager) {
 		return nil
 	})
 }
+
+func FuzzDefragment(f *testing.F) {
+	f.Add(0)
+	f.Fuzz(func(t *testing.T, seed int) {
+		tmpDir, err := os.MkdirTemp("", "mmm-defrag-fuzz-*")
+		require.NoError(t, err)
+		defer os.RemoveAll(tmpDir)
+
+		logger := zerolog.Nop()
+		rnd := rand.New(rand.NewPCG(uint64(seed), 0))
+
+		mmmm := &MultiMmapManager{
+			Dir:    tmpDir,
+			Logger: &logger,
+		}
+		err = mmmm.Init()
+		require.NoError(t, err)
+		defer mmmm.Close()
+
+		layerNames := []string{"a", "b", "c"}
+		var layers []*IndexingLayer
+		for _, name := range layerNames {
+			il, err := mmmm.EnsureLayer(name)
+			require.NoError(t, err)
+			defer il.Close()
+			layers = append(layers, il)
+		}
+
+		type indexedEvent struct {
+			evt nostr.Event
+			tag string
+		}
+		layerEvents := make([][]indexedEvent, len(layers))
+
+		sk := nostr.MustSecretKeyFromHex("945e01e37662430162121b804d3645a86d97df9d256917d86735d0eb219393eb")
+		pk := sk.Public()
+
+		totalEvents := rnd.IntN(500)
+		tChoices := []string{"foo", "bar", "banana"}
+		var written int
+
+		for written < totalEvents {
+			n := rnd.IntN(50) + 1
+			if n > totalEvents-written {
+				n = totalEvents - written
+			}
+
+			for i := 0; i < n; i++ {
+				sizeParam := rnd.IntN(2000)
+				content := strings.Repeat("z", sizeParam)
+
+				chosenTag := tChoices[rnd.IntN(3)]
+				evt := nostr.Event{
+					CreatedAt: nostr.Timestamp(rnd.Uint32()),
+					Kind:      nostr.KindTextNote,
+					Content:   content,
+					Tags:      nostr.Tags{{"t", chosenTag}},
+				}
+				evt.Sign(sk)
+
+				nLayers := rnd.IntN(len(layers)) + 1
+				perm := rnd.Perm(len(layers))
+				for pi := 0; pi < nLayers; pi++ {
+					li := perm[pi]
+					err := layers[li].SaveEvent(evt)
+					require.NoError(t, err)
+					layerEvents[li] = append(layerEvents[li], indexedEvent{evt, chosenTag})
+				}
+				written++
+			}
+
+			if n > 0 {
+				totalRemaining := 0
+				for _, levts := range layerEvents {
+					totalRemaining += len(levts)
+				}
+				if totalRemaining > 0 {
+					m := rnd.IntN(n)
+					if m > totalRemaining {
+						m = totalRemaining
+					}
+					for i := 0; i < m; i++ {
+						var nonEmpty []int
+						for li, levts := range layerEvents {
+							if len(levts) > 0 {
+								nonEmpty = append(nonEmpty, li)
+							}
+						}
+						if len(nonEmpty) == 0 {
+							break
+						}
+						li := nonEmpty[rnd.IntN(len(nonEmpty))]
+						idx := rnd.IntN(len(layerEvents[li]))
+						evtInfo := layerEvents[li][idx]
+						err := layers[li].DeleteEvent(evtInfo.evt.ID)
+						require.NoError(t, err)
+						layerEvents[li] = append(layerEvents[li][:idx], layerEvents[li][idx+1:]...)
+					}
+				}
+			}
+
+			if n > 0 {
+				o := rnd.IntN(n)
+				for i := 0; i < o; i++ {
+					if len(mmmm.freeRangesAll) > 1 {
+						param := rnd.IntN(len(mmmm.freeRangesAll))
+						err := mmmm.Defragment(param)
+						require.NoError(t, err)
+					}
+
+					verifyFreeRangesInvariants(t, mmmm)
+				}
+			}
+		}
+
+		// query each layer
+		for li, il := range layers {
+			levts := layerEvents[li]
+
+			// query by author
+			evts := slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}}, 10000))
+			require.Equal(t, len(levts), len(evts))
+
+			// query by author and kind
+			evts = slices.Collect(il.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{pk}, Kinds: []nostr.Kind{nostr.KindTextNote}}, 10000))
+			require.Equal(t, len(levts), len(evts))
+
+			// query by "t" tag
+			for _, tagVal := range tChoices {
+				expected := 0
+				for _, ie := range levts {
+					if ie.tag == tagVal {
+						expected++
+					}
+				}
+				evts = slices.Collect(il.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"t": []string{tagVal}}}, 10000))
+				require.Equal(t, expected, len(evts))
+			}
+
+			// query with no parameters
+			allEvts := slices.Collect(il.QueryEvents(nostr.Filter{}, 10000))
+			require.Equal(t, len(levts), len(allEvts))
+		}
+
+		// build union of all events across all layers
+		allEventSet := make(map[string]nostr.Event)
+		for _, levts := range layerEvents {
+			for _, ie := range levts {
+				allEventSet[ie.evt.ID.String()] = ie.evt
+			}
+		}
+
+		// all events still accessible via GetByID
+		for _, evt := range allEventSet {
+			gotEvt, eventLayers := mmmm.GetByID(evt.ID)
+			require.NotNil(t, gotEvt)
+			require.NotEmpty(t, eventLayers)
+			require.Equal(t, evt.Content, gotEvt.Content)
+		}
+
+		verifyFreeRangesInvariants(t, mmmm)
+
+		mmmm.lmdbEnv.View(func(txn *lmdb.Txn) error {
+			cursor, err := txn.OpenCursor(mmmm.indexId)
+			require.NoError(t, err)
+			defer cursor.Close()
+
+			var allPositions []position
+			for _, val, err := cursor.Get(nil, nil, lmdb.First); err == nil; _, val, err = cursor.Get(nil, val, lmdb.Next) {
+				pos := positionFromBytes(val[0:12])
+				allPositions = append(allPositions, pos)
+			}
+
+			slices.SortFunc(allPositions, func(a, b position) int {
+				return cmp.Compare(a.start, b.start)
+			})
+
+			var lastEnd uint64
+			for _, pos := range allPositions {
+				if pos.start < lastEnd {
+					t.Fatalf("event overlap after defrag: %d < %d", pos.start, lastEnd)
+				}
+				lastEnd = pos.start + uint64(pos.size)
+			}
+			return nil
+		})
+	})
+}

eventstore/mmm/fuzz_test.go

@@ -142,6 +142,11 @@ func FuzzTest(f *testing.F) {
 			mmmm.Rescan()
 		}
 
+		// perform random defrags -- shouldn't break the database
+		if rnd.UintN(3) == 1 {
+			mmmm.Defragment(len(deleted) / 3)
+		}
+
 		for id, deletedlayers := range deleted {
 			evt, foundlayers := mmmm.GetByID(id)
 

eventstore/mmm/mmmm.go

@@ -341,6 +341,7 @@ func (b *MultiMmapManager) removeAllReferencesFromLayer(txn *lmdb.Txn, layerId u
 	return nil
 }
 
+//go:inline
 func (b *MultiMmapManager) loadEvent(pos position, eventReceiver *nostr.Event) error {
 	return betterbinary.Unmarshal(b.mmapf[pos.start:pos.start+uint64(pos.size)], eventReceiver)
 }

eventstore/mmm/position.go

@@ -17,11 +17,6 @@ func (poss positions) find(start uint64) (idx int) {
 	return idx
 }
 
-func (poss positions) del(start uint64) positions {
-	idx := poss.find(start)
-	return slices.Delete(poss, idx, idx+1)
-}
-
 func (poss positions) String() string {
 	str := strings.Builder{}
 	str.Grow(10 + 20*len(poss))
@@ -46,6 +41,7 @@ func (pos position) isLarge() bool {
 	return pos.size >= LARGE_FREERANGE
 }
 
+//go:inline
 func positionFromBytes(posb []byte) position {
 	return position{
 		size:  binary.BigEndian.Uint32(posb[0:4]),
@@ -53,6 +49,7 @@ func positionFromBytes(posb []byte) position {
 	}
 }
 
+//go:inline
 func writeBytesFromPosition(out []byte, pos position) {
 	binary.BigEndian.PutUint32(out[0:4], pos.size)
 	binary.BigEndian.PutUint64(out[4:12], pos.start)

eventstore/mmm/save.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"slices"
 	"syscall"
 	"unsafe"
 
@@ -119,7 +120,8 @@ func (b *MultiMmapManager) storeOn(
 					b.freeRangesLarge = b.freeRangesLarge[0 : len(b.freeRangesLarge)-1]
 
 					// also delete it from b.freeRangesAll
-					b.freeRangesAll = b.freeRangesAll.del(fr.start)
+					idx := b.freeRangesAll.find(fr.start)
+					b.freeRangesAll = slices.Delete(b.freeRangesAll, idx, idx+1)
 				} else {
 					// otherwise modify it in place
 					newFreeRange := position{
@@ -155,7 +157,7 @@ func (b *MultiMmapManager) storeOn(
 		}
 
 		// write to the mmap
-		if err := betterbinary.Marshal(evt, b.mmapf[pos.start:]); err != nil {
+		if err := betterbinary.Marshal(evt, b.mmapf[pos.start:pos.start+uint64(pos.size)]); err != nil {
 			return false, fmt.Errorf("error marshaling to %d: %w", pos.start, err)
 		}
 

eventstore/mmm/testdata/fuzz/FuzzDefragment/2ce076b09281c580

@@ -0,0 +1,2 @@
+go test fuzz v1
+int(-360)

eventstore/mmm/testdata/fuzz/FuzzDefragment/d5845b29b7a801f7

@@ -0,0 +1,2 @@
+go test fuzz v1
+int(-17)

eventstore/mmm/testdata/fuzz/FuzzTest/ffd0b6bc2949a534

@@ -0,0 +1,5 @@
+go test fuzz v1
+int(46)
+uint(84)
+uint(55)
+uint(5)

khatru/blossom/blob.go

@@ -13,6 +13,9 @@ type BlobIndex interface {
 	List(ctx context.Context, pubkey nostr.PubKey) iter.Seq[blossom.BlobDescriptor]
 	Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error)
 	Delete(ctx context.Context, sha256 string, pubkey nostr.PubKey) error
+
+	ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor]
+	OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey
 }
 
 var (

khatru/blossom/eventstorewrapper.go

@@ -59,6 +59,30 @@ func (es EventStoreBlobIndexWrapper) List(ctx context.Context, pubkey nostr.PubK
 	}
 }
 
+func (es EventStoreBlobIndexWrapper) ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor] {
+	return func(yield func(nostr.PubKey, blossom.BlobDescriptor) bool) {
+		for evt := range es.Store.QueryEvents(nostr.Filter{
+			Kinds: []nostr.Kind{24242},
+		}, 1000) {
+			bd := es.parseEvent(evt)
+			if !yield(evt.PubKey, bd) {
+				return
+			}
+		}
+	}
+}
+
+func (es EventStoreBlobIndexWrapper) OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey {
+	var owners []nostr.PubKey
+	for evt := range es.Store.QueryEvents(nostr.Filter{
+		Tags:  nostr.TagMap{"x": []string{sha256}},
+		Kinds: []nostr.Kind{24242},
+	}, 1000) {
+		owners = append(owners, evt.PubKey)
+	}
+	return owners
+}
+
 func (es EventStoreBlobIndexWrapper) Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error) {
 	next, stop := iter.Pull(
 		es.Store.QueryEvents(nostr.Filter{Tags: nostr.TagMap{"x": []string{sha256}}, Kinds: []nostr.Kind{24242}, Limit: 1}, 1),

khatru/blossom/handlers.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -129,7 +128,7 @@ func (bs BlossomServer) handleUpload(w http.ResponseWriter, r *http.Request) {
 
 	hash := sha256.Sum256(b)
 	hhash := nostr.HexEncodeToString(hash[:])
-	mimeType := mime.TypeByExtension(ext)
+	mimeType := blossom.GetMIMEType(ext)
 	if mimeType == "" {
 		mimeType = "application/octet-stream"
 	}

khatru/blossom/memoryindex.go

@@ -59,6 +59,25 @@ func (x MemoryBlobIndex) List(ctx context.Context, pubkey nostr.PubKey) iter.Seq
 	}
 }
 
+func (x MemoryBlobIndex) ListAllBlobs(ctx context.Context) iter.Seq2[nostr.PubKey, blossom.BlobDescriptor] {
+	return func(yield func(nostr.PubKey, blossom.BlobDescriptor) bool) {
+		for _, v := range x.m.Range {
+			for _, owner := range v.owners {
+				if !yield(owner, v.blob) {
+					return
+				}
+			}
+		}
+	}
+}
+
+func (x MemoryBlobIndex) OwnersForBlob(ctx context.Context, sha256 string) []nostr.PubKey {
+	if val, ok := x.m.Load(sha256); ok {
+		return val.owners
+	}
+	return nil
+}
+
 func (x MemoryBlobIndex) Get(ctx context.Context, sha256 string) (*blossom.BlobDescriptor, error) {
 	if val, ok := x.m.Load(sha256); ok {
 		return &val.blob, nil

khatru/deleting.go

@@ -65,7 +65,7 @@ func (rl *Relay) handleDeleteRequest(ctx context.Context, evt nostr.Event) error
 			errg, ctx := errgroup.WithContext(ctx)
 			for target := range rl.QueryStored(ctx, f) {
 				// got the event, now check if the user can delete it
-				if target.PubKey == evt.PubKey {
+				if rl.AllowDeleting == nil && target.PubKey == evt.PubKey || rl.AllowDeleting != nil && rl.AllowDeleting(ctx, target, evt) {
 					// delete it
 					errg.Go(func() error {
 						if err := rl.DeleteEvent(ctx, target.ID); err != nil {

khatru/expiration.go

@@ -167,14 +167,14 @@ func (rl *Relay) StartExpirationManager(
 	}
 
 	go rl.expirationManager.start(rl.ctx)
-	rl.Info.AddSupportedNIP(40)
+	rl.Info.AddSupportedNIP("40")
 }
 
 func (rl *Relay) DisableExpirationManager() {
 	rl.expirationManager.stop()
 	rl.expirationManager = nil
 
-	idx := slices.Index(rl.Info.SupportedNIPs, 40)
+	idx := slices.Index(rl.Info.SupportedNIPs, "40")
 	if idx != -1 {
 		rl.Info.SupportedNIPs[idx] = rl.Info.SupportedNIPs[len(rl.Info.SupportedNIPs)-1]
 		rl.Info.SupportedNIPs = rl.Info.SupportedNIPs[0 : len(rl.Info.SupportedNIPs)-1]

khatru/grasp/server.go

@@ -31,7 +31,7 @@ func New(rl *khatru.Relay, repositoryDir string) *GraspServer {
 		},
 	}
 
-	rl.Info.AddSupportedNIP(34)
+	rl.Info.AddSupportedNIP("34")
 	rl.Info.SupportedGrasps = append(rl.Info.SupportedGrasps, "GRASP-01")
 
 	base := rl.Router()

khatru/handlers.go

@@ -227,6 +227,9 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						if writeErr == nil {
 							// this always returns "blocked: " whenever it returns an error
 							writeErr = rl.handleDeleteRequest(ctx, env.Event)
+							if writeErr == ErrNothingToDelete {
+								writeErr = nil
+							}
 						}
 					} else if env.Event.Kind.IsEphemeral() {
 						// this will also always return a prefixed reason
@@ -292,9 +295,6 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 				case *nostr.ReqEnvelope:
 					rl.removeListenerId(ws, env.SubscriptionID)
 
-					eose := sync.WaitGroup{}
-					eose.Add(len(env.Filters))
-
 					// a context just for the "stored events" request handler
 					reqCtx, cancelReqCtx := context.WithCancelCause(ctx)
 
@@ -303,7 +303,7 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 
 					// handle each filter separately -- dispatching events as they're loaded from databases
 					for _, filter := range env.Filters {
-						err := rl.handleRequest(reqCtx, env.SubscriptionID, &eose, ws, filter)
+						err := rl.handleRequest(reqCtx, env.SubscriptionID, ws, filter)
 						if err != nil {
 							// fail everything if any filter is rejected
 							reason := err.Error()
@@ -321,11 +321,7 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						}
 					}
 
-					go func() {
+					ws.WriteJSON(nostr.EOSEEnvelope(env.SubscriptionID))
-						// when all events have been loaded from databases and dispatched we can fire the EOSE message
-						eose.Wait()
-						ws.WriteJSON(nostr.EOSEEnvelope(env.SubscriptionID))
-					}()
 				case *nostr.CloseEnvelope:
 					id := string(*env)
 					rl.removeListenerId(ws, id)
@@ -350,6 +346,9 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						}
 						ws.authLock.Unlock()
 						ws.WriteJSON(nostr.OKEnvelope{EventID: env.Event.ID, OK: true})
+						if rl.OnAuth != nil {
+							rl.OnAuth(ctx, pubkey)
+						}
 					} else {
 						ws.WriteJSON(nostr.OKEnvelope{EventID: env.Event.ID, OK: false, Reason: "error: failed to authenticate: " + err.Error()})
 					}

khatru/nip11.go

@@ -12,13 +12,13 @@ func (rl *Relay) HandleNIP11(w http.ResponseWriter, r *http.Request) {
 	info := *rl.Info
 
 	if nil != rl.DeleteEvent {
-		info.AddSupportedNIP(9)
+		info.AddSupportedNIP("9")
 	}
 	if nil != rl.Count {
-		info.AddSupportedNIP(45)
+		info.AddSupportedNIP("45")
 	}
 	if rl.Negentropy {
-		info.AddSupportedNIP(77)
+		info.AddSupportedNIP("77")
 	}
 
 	// resolve relative icon and banner URLs against base URL

khatru/nip86.go

@@ -21,8 +21,10 @@ type RelayManagementAPI struct {
 
 	BanPubKey                   func(ctx context.Context, pubkey nostr.PubKey, reason string) error
 	ListBannedPubKeys           func(ctx context.Context) ([]nip86.PubKeyReason, error)
+	UnbanPubKey                 func(ctx context.Context, pubkey nostr.PubKey, reason string) error
 	AllowPubKey                 func(ctx context.Context, pubkey nostr.PubKey, reason string) error
 	ListAllowedPubKeys          func(ctx context.Context) ([]nip86.PubKeyReason, error)
+	UnallowPubKey               func(ctx context.Context, pubkey nostr.PubKey, reason string) error
 	ListEventsNeedingModeration func(ctx context.Context) ([]nip86.IDReason, error)
 	AllowEvent                  func(ctx context.Context, id nostr.ID, reason string) error
 	BanEvent                    func(ctx context.Context, id nostr.ID, reason string) error
@@ -41,6 +43,11 @@ type RelayManagementAPI struct {
 	Stats                       func(ctx context.Context) (nip86.Response, error)
 	GrantAdmin                  func(ctx context.Context, pubkey nostr.PubKey, methods []string) error
 	RevokeAdmin                 func(ctx context.Context, pubkey nostr.PubKey, methods []string) error
+	CreateRole                  func(ctx context.Context, id string, label string, description string, color int, order int) error
+	EditRole                    func(ctx context.Context, id string, label string, description string, color int, order int) error
+	DeleteRole                  func(ctx context.Context, id string) error
+	AssignRole                  func(ctx context.Context, pubkey nostr.PubKey, roleID string) error
+	UnassignRole                func(ctx context.Context, pubkey nostr.PubKey, roleID string) error
 	Generic                     func(ctx context.Context, request nip86.Request) (nip86.Response, error)
 }
 
@@ -168,6 +175,14 @@ func (rl *Relay) HandleNIP86(w http.ResponseWriter, r *http.Request) {
 			} else {
 				resp.Result = result
 			}
+		case nip86.UnbanPubKey:
+			if rl.ManagementAPI.UnbanPubKey == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.UnbanPubKey(ctx, thing.PubKey, thing.Reason); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
 		case nip86.AllowPubKey:
 			if rl.ManagementAPI.AllowPubKey == nil {
 				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
@@ -184,6 +199,14 @@ func (rl *Relay) HandleNIP86(w http.ResponseWriter, r *http.Request) {
 			} else {
 				resp.Result = result
 			}
+		case nip86.UnallowPubKey:
+			if rl.ManagementAPI.UnallowPubKey == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.UnallowPubKey(ctx, thing.PubKey, thing.Reason); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
 		case nip86.BanEvent:
 			if rl.ManagementAPI.BanEvent == nil {
 				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
@@ -312,6 +335,46 @@ func (rl *Relay) HandleNIP86(w http.ResponseWriter, r *http.Request) {
 			} else {
 				resp.Result = true
 			}
+		case nip86.CreateRole:
+			if rl.ManagementAPI.CreateRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.CreateRole(ctx, thing.ID, thing.Label, thing.Description, thing.Color, thing.Order); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.EditRole:
+			if rl.ManagementAPI.EditRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.EditRole(ctx, thing.ID, thing.Label, thing.Description, thing.Color, thing.Order); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.DeleteRole:
+			if rl.ManagementAPI.DeleteRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.DeleteRole(ctx, thing.ID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.AssignRole:
+			if rl.ManagementAPI.AssignRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.AssignRole(ctx, thing.PubKey, thing.RoleID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
+		case nip86.UnassignRole:
+			if rl.ManagementAPI.UnassignRole == nil {
+				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())
+			} else if err := rl.ManagementAPI.UnassignRole(ctx, thing.PubKey, thing.RoleID); err != nil {
+				resp.Error = err.Error()
+			} else {
+				resp.Result = true
+			}
 		case nip86.ListDisallowedKinds:
 			if rl.ManagementAPI.ListDisallowedKinds == nil {
 				resp.Error = fmt.Sprintf("method %s not supported", thing.MethodName())

khatru/relay.go

@@ -33,7 +33,7 @@ func NewRelay() *Relay {
 		Info: &nip11.RelayInformationDocument{
 			Software:      "https://pkg.go.dev/fiatjaf.com/nostr/khatru",
 			Version:       "n/a",
-			SupportedNIPs: []any{1, 11, 42, 70, 86},
+			SupportedNIPs: []string{"1", "11", "42", "70", "86"},
 		},
 
 		upgrader: websocket.Upgrader{
@@ -75,6 +75,7 @@ type Relay struct {
 	DeleteEvent               func(ctx context.Context, id nostr.ID) error
 	OnEventSaved              func(ctx context.Context, event nostr.Event)
 	OnEventDeleted            func(ctx context.Context, deleted nostr.Event)
+	AllowDeleting             func(ctx context.Context, target, deletion nostr.Event) bool
 	OnEphemeralEvent          func(ctx context.Context, event nostr.Event)
 	OnRequest                 func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
 	OnCount                   func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
@@ -84,6 +85,7 @@ type Relay struct {
 	RejectConnection          func(r *http.Request) bool
 	OnConnect                 func(ctx context.Context)
 	OnDisconnect              func(ctx context.Context)
+	OnAuth                    func(ctx context.Context, pubkey nostr.PubKey)
 	OnListenerAdded           func(ws *WebSocket, ssid int, id string, filter nostr.Filter)
 	OnListenerRemoved         func(ws *WebSocket, ssid int, id string, filter nostr.Filter)
 	OverwriteRelayInformation func(ctx context.Context, r *http.Request, info nip11.RelayInformationDocument) nip11.RelayInformationDocument

khatru/responding.go

@@ -3,15 +3,12 @@ package khatru
 import (
 	"context"
 	"errors"
-	"sync"
 
 	"fiatjaf.com/nostr"
 	"fiatjaf.com/nostr/nip45/hyperloglog"
 )
 
-func (rl *Relay) handleRequest(ctx context.Context, id string, eose *sync.WaitGroup, ws *WebSocket, filter nostr.Filter) error {
+func (rl *Relay) handleRequest(ctx context.Context, id string, ws *WebSocket, filter nostr.Filter) error {
-	defer eose.Done()
-
 	// then check if we'll reject this filter (we apply this after overwriting
 	// because we may, for example, remove some things from the incoming filters
 	// that we know we don't support, and then if the end result is an empty

kinds.go

@@ -38,10 +38,20 @@ func (kind Kind) Name() string {
 		return "Seal"
 	case KindDirectMessage:
 		return "DirectMessage"
+	case KindFileMessage:
+		return "FileMessage"
 	case KindGenericRepost:
 		return "GenericRepost"
 	case KindReactionToWebsite:
 		return "ReactionToWebsite"
+	case KindPhoto:
+		return "Photo"
+	case KindNormalVideoEvent:
+		return "NormalVideoEvent"
+	case KindShortVideoEvent:
+		return "ShortVideoEvent"
+	case KindPublicMessage:
+		return "PublicMessage"
 	case KindChannelCreation:
 		return "ChannelCreation"
 	case KindChannelMetadata:
@@ -52,12 +62,14 @@ func (kind Kind) Name() string {
 		return "ChannelHideMessage"
 	case KindChannelMuteUser:
 		return "ChannelMuteUser"
+	case KindPodcastEpisode:
+		return "PodcastEpisode"
 	case KindChess:
 		return "Chess"
 	case KindMergeRequests:
 		return "MergeRequests"
-	case KindComment:
+	case KindPollResponse:
-		return "Comment"
+		return "PollResponse"
 	case KindBid:
 		return "Bid"
 	case KindBidConfirmation:
@@ -68,10 +80,26 @@ func (kind Kind) Name() string {
 		return "GiftWrap"
 	case KindFileMetadata:
 		return "FileMetadata"
+	case KindPoll:
+		return "Poll"
+	case KindComment:
+		return "Comment"
+	case KindVoiceMessage:
+		return "VoiceMessage"
+	case KindScroll:
+		return "Scroll"
+	case KindVoiceMessageComment:
+		return "VoiceMessageComment"
 	case KindLiveChatMessage:
 		return "LiveChatMessage"
+	case KindCodeSnippet:
+		return "CodeSnippet"
 	case KindPatch:
 		return "Patch"
+	case KindGitPullRequest:
+		return "GitPullRequest"
+	case KindGitPullRequestUpdate:
+		return "GitPullRequestUpdate"
 	case KindIssue:
 		return "Issue"
 	case KindReply:
@@ -100,10 +128,24 @@ func (kind Kind) Name() string {
 		return "TorrentComment"
 	case KindCoinjoinPool:
 		return "CoinjoinPool"
+	case KindDecoupledKeyClientAnnouncement:
+		return "DecoupledKeyClientAnnouncement"
+	case KindDecoupledEncryptionKeyDistribution:
+		return "DecoupledEncryptionKeyDistribution"
 	case KindCommunityPostApproval:
 		return "CommunityPostApproval"
 	case KindJobFeedback:
 		return "JobFeedback"
+	case KindReservedCashuWalletTokens:
+		return "ReservedCashuWalletTokens"
+	case KindCashuWalletTokens:
+		return "CashuWalletTokens"
+	case KindCashuWalletHistory:
+		return "CashuWalletHistory"
+	case KindGeocacheLog:
+		return "GeocacheLog"
+	case KindGeocacheProofOfFind:
+		return "GeocacheProofOfFind"
 	case KindSimpleGroupPutUser:
 		return "SimpleGroupPutUser"
 	case KindSimpleGroupRemoveUser:
@@ -152,14 +194,24 @@ func (kind Kind) Name() string {
 		return "SearchRelayList"
 	case KindSimpleGroupList:
 		return "SimpleGroupList"
+	case KindFavoriteRelaysList:
+		return "FavoriteRelaysList"
+	case KindPrivateEventRelayList:
+		return "PrivateEventRelayList"
 	case KindInterestList:
 		return "InterestList"
 	case KindNutZapInfo:
 		return "NutZapInfo"
+	case KindMediaFollows:
+		return "MediaFollows"
 	case KindEmojiList:
 		return "EmojiList"
+	case KindDecoupledKeyAnnouncement:
+		return "DecoupledKeyAnnouncement"
 	case KindDMRelayList:
 		return "DMRelayList"
+	case KindFavoritePodcasts:
+		return "FavoritePodcasts"
 	case KindUserServerList:
 		return "UserServerList"
 	case KindFileStorageServerList:
@@ -168,8 +220,26 @@ func (kind Kind) Name() string {
 		return "GoodWikiAuthorList"
 	case KindGoodWikiRelayList:
 		return "GoodWikiRelayList"
+	case KindPodcastMetadata:
+		return "PodcastMetadata"
+	case KindAuthoredPodcasts:
+		return "AuthoredPodcasts"
+	case KindRelayMonitorAnnouncement:
+		return "RelayMonitorAnnouncement"
+	case KindRoomPresence:
+		return "RoomPresence"
+	case KindUserGraspList:
+		return "UserGraspList"
+	case KindProxyAnnouncement:
+		return "ProxyAnnouncement"
+	case KindTransportMethodAnnouncement:
+		return "TransportMethodAnnouncement"
 	case KindNWCWalletInfo:
 		return "NWCWalletInfo"
+	case KindNsiteRoot:
+		return "NsiteRoot"
+	case KindCashuWalletEvent:
+		return "CashuWalletEvent"
 	case KindLightningPubRPC:
 		return "LightningPubRPC"
 	case KindClientAuthentication:
@@ -226,10 +296,20 @@ func (kind Kind) Name() string {
 		return "ReleaseArtifactSets"
 	case KindApplicationSpecificData:
 		return "ApplicationSpecificData"
+	case KindRelayDiscovery:
+		return "RelayDiscovery"
+	case KindAppCurationSet:
+		return "AppCurationSet"
 	case KindLiveEvent:
 		return "LiveEvent"
+	case KindInteractiveRoom:
+		return "InteractiveRoom"
+	case KindConferenceEvent:
+		return "ConferenceEvent"
 	case KindUserStatuses:
 		return "UserStatuses"
+	case KindSlideSet:
+		return "SlideSet"
 	case KindClassifiedListing:
 		return "ClassifiedListing"
 	case KindDraftClassifiedListing:
@@ -238,20 +318,14 @@ func (kind Kind) Name() string {
 		return "RepositoryAnnouncement"
 	case KindRepositoryState:
 		return "RepositoryState"
-	case KindSimpleGroupMetadata:
-		return "SimpleGroupMetadata"
-	case KindSimpleGroupAdmins:
-		return "SimpleGroupAdmins"
-	case KindSimpleGroupMembers:
-		return "SimpleGroupMembers"
-	case KindSimpleGroupRoles:
-		return "SimpleGroupRoles"
-	case KindSimpleGroupLiveKitParticipants:
-		return "SimpleGroupLiveKitParticipants"
 	case KindWikiArticle:
 		return "WikiArticle"
 	case KindRedirects:
 		return "Redirects"
+	case KindDraftEvent:
+		return "DraftEvent"
+	case KindLinkSet:
+		return "LinkSet"
 	case KindFeed:
 		return "Feed"
 	case KindDateCalendarEvent:
@@ -266,158 +340,232 @@ func (kind Kind) Name() string {
 		return "HandlerRecommendation"
 	case KindHandlerInformation:
 		return "HandlerInformation"
-	case KindVideoEvent:
+	case KindSoftwareApplication:
-		return "VideoEvent"
+		return "SoftwareApplication"
-	case KindShortVideoEvent:
+	case KindLegacyNsiteFile:
-		return "ShortVideoEvent"
+		return "LegacyNsiteFile"
 	case KindVideoViewEvent:
 		return "VideoViewEvent"
 	case KindCommunityDefinition:
 		return "CommunityDefinition"
-	case KindNsiteRoot:
-		return "NsiteRoot"
 	case KindNsiteNamed:
 		return "NsiteNamed"
+	case KindGeocacheListing:
+		return "GeocacheListing"
+	case KindGeocacheLogEntry:
+		return "GeocacheLogEntry"
+	case KindCashuMintAnnouncement:
+		return "CashuMintAnnouncement"
+	case KindFedimintAnnouncement:
+		return "FedimintAnnouncement"
+	case KindPeerToPeerOrderEvents:
+		return "PeerToPeerOrderEvents"
+	case KindSimpleGroupMetadata:
+		return "SimpleGroupMetadata"
+	case KindSimpleGroupAdmins:
+		return "SimpleGroupAdmins"
+	case KindSimpleGroupMembers:
+		return "SimpleGroupMembers"
+	case KindSimpleGroupRoles:
+		return "SimpleGroupRoles"
+	case KindSimpleGroupLiveKitParticipants:
+		return "SimpleGroupLiveKitParticipants"
+	case KindStarterPacks:
+		return "StarterPacks"
+	case KindMediaStarterPacks:
+		return "MediaStarterPacks"
+	case KindWebBookmarks:
+		return "WebBookmarks"
+
 	}
 	return "unknown"
 }
 
 const (
-	KindProfileMetadata                Kind = 0
+	KindProfileMetadata                    Kind = 0
-	KindTextNote                       Kind = 1
+	KindTextNote                           Kind = 1
-	KindRecommendServer                Kind = 2
+	KindRecommendServer                    Kind = 2
-	KindFollowList                     Kind = 3
+	KindFollowList                         Kind = 3
-	KindEncryptedDirectMessage         Kind = 4
+	KindEncryptedDirectMessage             Kind = 4
-	KindDeletion                       Kind = 5
+	KindDeletion                           Kind = 5
-	KindRepost                         Kind = 6
+	KindRepost                             Kind = 6
-	KindReaction                       Kind = 7
+	KindReaction                           Kind = 7
-	KindBadgeAward                     Kind = 8
+	KindBadgeAward                         Kind = 8
-	KindSimpleGroupChatMessage         Kind = 9
+	KindSimpleGroupChatMessage             Kind = 9
-	KindSimpleGroupThreadedReply       Kind = 10
+	KindSimpleGroupThreadedReply           Kind = 10
-	KindSimpleGroupThread              Kind = 11
+	KindSimpleGroupThread                  Kind = 11
-	KindSimpleGroupReply               Kind = 12
+	KindSimpleGroupReply                   Kind = 12
-	KindSeal                           Kind = 13
+	KindSeal                               Kind = 13
-	KindDirectMessage                  Kind = 14
+	KindDirectMessage                      Kind = 14
-	KindGenericRepost                  Kind = 16
+	KindFileMessage                        Kind = 15
-	KindReactionToWebsite              Kind = 17
+	KindGenericRepost                      Kind = 16
-	KindChannelCreation                Kind = 40
+	KindReactionToWebsite                  Kind = 17
-	KindChannelMetadata                Kind = 41
+	KindPhoto                              Kind = 20
-	KindChannelMessage                 Kind = 42
+	KindNormalVideoEvent                   Kind = 21
-	KindChannelHideMessage             Kind = 43
+	KindShortVideoEvent                    Kind = 22
-	KindChannelMuteUser                Kind = 44
+	KindPublicMessage                      Kind = 24
-	KindChess                          Kind = 64
+	KindChannelCreation                    Kind = 40
-	KindMergeRequests                  Kind = 818
+	KindChannelMetadata                    Kind = 41
-	KindComment                        Kind = 1111
+	KindChannelMessage                     Kind = 42
-	KindBid                            Kind = 1021
+	KindChannelHideMessage                 Kind = 43
-	KindBidConfirmation                Kind = 1022
+	KindChannelMuteUser                    Kind = 44
-	KindOpenTimestamps                 Kind = 1040
+	KindPodcastEpisode                     Kind = 54
-	KindGiftWrap                       Kind = 1059
+	KindChess                              Kind = 64
-	KindFileMetadata                   Kind = 1063
+	KindMergeRequests                      Kind = 818
-	KindLiveChatMessage                Kind = 1311
+	KindPollResponse                       Kind = 1018
-	KindPatch                          Kind = 1617
+	KindBid                                Kind = 1021
-	KindIssue                          Kind = 1621
+	KindBidConfirmation                    Kind = 1022
-	KindReply                          Kind = 1622
+	KindOpenTimestamps                     Kind = 1040
-	KindStatusOpen                     Kind = 1630
+	KindGiftWrap                           Kind = 1059
-	KindStatusApplied                  Kind = 1631
+	KindFileMetadata                       Kind = 1063
-	KindStatusClosed                   Kind = 1632
+	KindPoll                               Kind = 1068
-	KindStatusDraft                    Kind = 1633
+	KindComment                            Kind = 1111
-	KindProblemTracker                 Kind = 1971
+	KindVoiceMessage                       Kind = 1222
-	KindReporting                      Kind = 1984
+	KindScroll                             Kind = 1227
-	KindLabel                          Kind = 1985
+	KindVoiceMessageComment                Kind = 1244
-	KindRelayReviews                   Kind = 1986
+	KindLiveChatMessage                    Kind = 1311
-	KindAIEmbeddings                   Kind = 1987
+	KindCodeSnippet                        Kind = 1337
-	KindTorrent                        Kind = 2003
+	KindPatch                              Kind = 1617
-	KindTorrentComment                 Kind = 2004
+	KindGitPullRequest                     Kind = 1618
-	KindCoinjoinPool                   Kind = 2022
+	KindGitPullRequestUpdate               Kind = 1619
-	KindCommunityPostApproval          Kind = 4550
+	KindIssue                              Kind = 1621
-	KindJobFeedback                    Kind = 7000
+	KindReply                              Kind = 1622
-	KindSimpleGroupPutUser             Kind = 9000
+	KindStatusOpen                         Kind = 1630
-	KindSimpleGroupRemoveUser          Kind = 9001
+	KindStatusApplied                      Kind = 1631
-	KindSimpleGroupEditMetadata        Kind = 9002
+	KindStatusClosed                       Kind = 1632
-	KindSimpleGroupDeleteEvent         Kind = 9005
+	KindStatusDraft                        Kind = 1633
-	KindSimpleGroupCreateGroup         Kind = 9007
+	KindProblemTracker                     Kind = 1971
-	KindSimpleGroupDeleteGroup         Kind = 9008
+	KindReporting                          Kind = 1984
-	KindSimpleGroupCreateInvite        Kind = 9009
+	KindLabel                              Kind = 1985
-	KindSimpleGroupJoinRequest         Kind = 9021
+	KindRelayReviews                       Kind = 1986
-	KindSimpleGroupLeaveRequest        Kind = 9022
+	KindAIEmbeddings                       Kind = 1987
-	KindZapGoal                        Kind = 9041
+	KindTorrent                            Kind = 2003
-	KindNutZap                         Kind = 9321
+	KindTorrentComment                     Kind = 2004
-	KindTidalLogin                     Kind = 9467
+	KindCoinjoinPool                       Kind = 2022
-	KindZapRequest                     Kind = 9734
+	KindDecoupledKeyClientAnnouncement     Kind = 4454
-	KindZap                            Kind = 9735
+	KindDecoupledEncryptionKeyDistribution Kind = 4455
-	KindHighlights                     Kind = 9802
+	KindCommunityPostApproval              Kind = 4550
-	KindMuteList                       Kind = 10000
+	KindJobFeedback                        Kind = 7000
-	KindPinList                        Kind = 10001
+	KindReservedCashuWalletTokens          Kind = 7374
-	KindRelayListMetadata              Kind = 10002
+	KindCashuWalletTokens                  Kind = 7375
-	KindBookmarkList                   Kind = 10003
+	KindCashuWalletHistory                 Kind = 7376
-	KindCommunityList                  Kind = 10004
+	KindGeocacheLog                        Kind = 7516
-	KindPublicChatList                 Kind = 10005
+	KindGeocacheProofOfFind                Kind = 7517
-	KindBlockedRelayList               Kind = 10006
+	KindSimpleGroupPutUser                 Kind = 9000
-	KindSearchRelayList                Kind = 10007
+	KindSimpleGroupRemoveUser              Kind = 9001
-	KindSimpleGroupList                Kind = 10009
+	KindSimpleGroupEditMetadata            Kind = 9002
-	KindInterestList                   Kind = 10015
+	KindSimpleGroupDeleteEvent             Kind = 9005
-	KindNutZapInfo                     Kind = 10019
+	KindSimpleGroupCreateGroup             Kind = 9007
-	KindEmojiList                      Kind = 10030
+	KindSimpleGroupDeleteGroup             Kind = 9008
-	KindDMRelayList                    Kind = 10050
+	KindSimpleGroupCreateInvite            Kind = 9009
-	KindUserServerList                 Kind = 10063
+	KindSimpleGroupJoinRequest             Kind = 9021
-	KindFileStorageServerList          Kind = 10096
+	KindSimpleGroupLeaveRequest            Kind = 9022
-	KindGoodWikiAuthorList             Kind = 10101
+	KindZapGoal                            Kind = 9041
-	KindGoodWikiRelayList              Kind = 10102
+	KindNutZap                             Kind = 9321
-	KindNWCWalletInfo                  Kind = 13194
+	KindTidalLogin                         Kind = 9467
-	KindNsiteRoot                      Kind = 15128
+	KindZapRequest                         Kind = 9734
-	KindLightningPubRPC                Kind = 21000
+	KindZap                                Kind = 9735
-	KindClientAuthentication           Kind = 22242
+	KindHighlights                         Kind = 9802
-	KindNWCWalletRequest               Kind = 23194
+	KindMuteList                           Kind = 10000
-	KindNWCWalletResponse              Kind = 23195
+	KindPinList                            Kind = 10001
-	KindNostrConnect                   Kind = 24133
+	KindRelayListMetadata                  Kind = 10002
-	KindBlobs                          Kind = 24242
+	KindBookmarkList                       Kind = 10003
-	KindHTTPAuth                       Kind = 27235
+	KindCommunityList                      Kind = 10004
-	KindCategorizedPeopleList          Kind = 30000
+	KindPublicChatList                     Kind = 10005
-	KindCategorizedBookmarksList       Kind = 30001
+	KindBlockedRelayList                   Kind = 10006
-	KindRelaySets                      Kind = 30002
+	KindSearchRelayList                    Kind = 10007
-	KindBookmarkSets                   Kind = 30003
+	KindSimpleGroupList                    Kind = 10009
-	KindCuratedSets                    Kind = 30004
+	KindFavoriteRelaysList                 Kind = 10012
-	KindCuratedVideoSets               Kind = 30005
+	KindPrivateEventRelayList              Kind = 10013
-	KindMuteSets                       Kind = 30007
+	KindInterestList                       Kind = 10015
-	KindProfileBadges                  Kind = 30008
+	KindNutZapInfo                         Kind = 10019
-	KindBadgeDefinition                Kind = 30009
+	KindMediaFollows                       Kind = 10020
-	KindInterestSets                   Kind = 30015
+	KindEmojiList                          Kind = 10030
-	KindStallDefinition                Kind = 30017
+	KindDecoupledKeyAnnouncement           Kind = 10044
-	KindProductDefinition              Kind = 30018
+	KindDMRelayList                        Kind = 10050
-	KindMarketplaceUI                  Kind = 30019
+	KindFavoritePodcasts                   Kind = 10054
-	KindProductSoldAsAuction           Kind = 30020
+	KindUserServerList                     Kind = 10063
-	KindArticle                        Kind = 30023
+	KindFileStorageServerList              Kind = 10096
-	KindDraftArticle                   Kind = 30024
+	KindGoodWikiAuthorList                 Kind = 10101
-	KindEmojiSets                      Kind = 30030
+	KindGoodWikiRelayList                  Kind = 10102
-	KindModularArticleHeader           Kind = 30040
+	KindPodcastMetadata                    Kind = 10154
-	KindModularArticleContent          Kind = 30041
+	KindAuthoredPodcasts                   Kind = 10164
-	KindReleaseArtifactSets            Kind = 30063
+	KindRelayMonitorAnnouncement           Kind = 10166
-	KindApplicationSpecificData        Kind = 30078
+	KindRoomPresence                       Kind = 10312
-	KindLiveEvent                      Kind = 30311
+	KindUserGraspList                      Kind = 10317
-	KindUserStatuses                   Kind = 30315
+	KindProxyAnnouncement                  Kind = 10377
-	KindClassifiedListing              Kind = 30402
+	KindTransportMethodAnnouncement        Kind = 11111
-	KindDraftClassifiedListing         Kind = 30403
+	KindNWCWalletInfo                      Kind = 13194
-	KindRepositoryAnnouncement         Kind = 30617
+	KindNsiteRoot                          Kind = 15128
-	KindRepositoryState                Kind = 30618
+	KindCashuWalletEvent                   Kind = 17375
-	KindNsiteNamed                     Kind = 35128
+	KindLightningPubRPC                    Kind = 21000
-	KindSimpleGroupMetadata            Kind = 39000
+	KindClientAuthentication               Kind = 22242
-	KindSimpleGroupAdmins              Kind = 39001
+	KindNWCWalletRequest                   Kind = 23194
-	KindSimpleGroupMembers             Kind = 39002
+	KindNWCWalletResponse                  Kind = 23195
-	KindSimpleGroupRoles               Kind = 39003
+	KindNostrConnect                       Kind = 24133
-	KindSimpleGroupLiveKitParticipants Kind = 39004
+	KindBlobs                              Kind = 24242
-	KindWikiArticle                    Kind = 30818
+	KindHTTPAuth                           Kind = 27235
-	KindRedirects                      Kind = 30819
+	KindCategorizedPeopleList              Kind = 30000
-	KindFeed                           Kind = 31890
+	KindCategorizedBookmarksList           Kind = 30001
-	KindDateCalendarEvent              Kind = 31922
+	KindRelaySets                          Kind = 30002
-	KindTimeCalendarEvent              Kind = 31923
+	KindBookmarkSets                       Kind = 30003
-	KindCalendar                       Kind = 31924
+	KindCuratedSets                        Kind = 30004
-	KindCalendarEventRSVP              Kind = 31925
+	KindCuratedVideoSets                   Kind = 30005
-	KindHandlerRecommendation          Kind = 31989
+	KindMuteSets                           Kind = 30007
-	KindHandlerInformation             Kind = 31990
+	KindProfileBadges                      Kind = 30008
-	KindVideoEvent                     Kind = 34235
+	KindBadgeDefinition                    Kind = 30009
-	KindShortVideoEvent                Kind = 34236
+	KindInterestSets                       Kind = 30015
-	KindVideoViewEvent                 Kind = 34237
+	KindStallDefinition                    Kind = 30017
-	KindCommunityDefinition            Kind = 34550
+	KindProductDefinition                  Kind = 30018
+	KindMarketplaceUI                      Kind = 30019
+	KindProductSoldAsAuction               Kind = 30020
+	KindArticle                            Kind = 30023
+	KindDraftArticle                       Kind = 30024
+	KindEmojiSets                          Kind = 30030
+	KindModularArticleHeader               Kind = 30040
+	KindModularArticleContent              Kind = 30041
+	KindReleaseArtifactSets                Kind = 30063
+	KindApplicationSpecificData            Kind = 30078
+	KindRelayDiscovery                     Kind = 30166
+	KindAppCurationSet                     Kind = 30267
+	KindLiveEvent                          Kind = 30311
+	KindInteractiveRoom                    Kind = 30312
+	KindConferenceEvent                    Kind = 30313
+	KindUserStatuses                       Kind = 30315
+	KindSlideSet                           Kind = 30388
+	KindClassifiedListing                  Kind = 30402
+	KindDraftClassifiedListing             Kind = 30403
+	KindRepositoryAnnouncement             Kind = 30617
+	KindRepositoryState                    Kind = 30618
+	KindWikiArticle                        Kind = 30818
+	KindRedirects                          Kind = 30819
+	KindDraftEvent                         Kind = 31234
+	KindLinkSet                            Kind = 31388
+	KindFeed                               Kind = 31890
+	KindDateCalendarEvent                  Kind = 31922
+	KindTimeCalendarEvent                  Kind = 31923
+	KindCalendar                           Kind = 31924
+	KindCalendarEventRSVP                  Kind = 31925
+	KindHandlerRecommendation              Kind = 31989
+	KindHandlerInformation                 Kind = 31990
+	KindSoftwareApplication                Kind = 32267
+	KindLegacyNsiteFile                    Kind = 34128
+	KindVideoViewEvent                     Kind = 34237
+	KindCommunityDefinition                Kind = 34550
+	KindNsiteNamed                         Kind = 35128
+	KindGeocacheListing                    Kind = 37515
+	KindGeocacheLogEntry                   Kind = 37516
+	KindCashuMintAnnouncement              Kind = 38172
+	KindFedimintAnnouncement               Kind = 38173
+	KindPeerToPeerOrderEvents              Kind = 38383
+	KindSimpleGroupMetadata                Kind = 39000
+	KindSimpleGroupAdmins                  Kind = 39001
+	KindSimpleGroupMembers                 Kind = 39002
+	KindSimpleGroupRoles                   Kind = 39003
+	KindSimpleGroupLiveKitParticipants     Kind = 39004
+	KindStarterPacks                       Kind = 39089
+	KindMediaStarterPacks                  Kind = 39092
+	KindWebBookmarks                       Kind = 39701
 )
 
 func (kind Kind) IsRegular() bool {

nip11/nip11_test.go

@@ -9,30 +9,30 @@ import (
 
 func TestAddSupportedNIP(t *testing.T) {
 	info := RelayInformationDocument{}
-	info.AddSupportedNIP(12)
+	info.AddSupportedNIP("12")
-	info.AddSupportedNIP(12)
+	info.AddSupportedNIP("12")
-	info.AddSupportedNIP(13)
+	info.AddSupportedNIP("13")
-	info.AddSupportedNIP(1)
+	info.AddSupportedNIP("1")
-	info.AddSupportedNIP(12)
+	info.AddSupportedNIP("12")
-	info.AddSupportedNIP(44)
+	info.AddSupportedNIP("44")
-	info.AddSupportedNIP(2)
+	info.AddSupportedNIP("2")
-	info.AddSupportedNIP(13)
+	info.AddSupportedNIP("13")
-	info.AddSupportedNIP(2)
+	info.AddSupportedNIP("2")
-	info.AddSupportedNIP(13)
+	info.AddSupportedNIP("13")
-	info.AddSupportedNIP(0)
+	info.AddSupportedNIP("0")
-	info.AddSupportedNIP(17)
+	info.AddSupportedNIP("17")
-	info.AddSupportedNIP(19)
+	info.AddSupportedNIP("19")
-	info.AddSupportedNIP(1)
+	info.AddSupportedNIP("1")
-	info.AddSupportedNIP(18)
+	info.AddSupportedNIP("18")
 
-	assert.Contains(t, info.SupportedNIPs, 0, 1, 2, 12, 13, 17, 18, 19, 44)
+	assert.Contains(t, info.SupportedNIPs, "0", "1", "2", "12", "13", "17", "18", "19", "44")
 }
 
 func TestAddSupportedNIPs(t *testing.T) {
 	info := RelayInformationDocument{}
-	info.AddSupportedNIPs([]int{0, 1, 2, 12, 13, 17, 18, 19, 44})
+	info.AddSupportedNIPs([]int{"0", "1", "2", "12", "13", "17", "18", "19", "44"})
 
-	assert.Contains(t, info.SupportedNIPs, 0, 1, 2, 12, 13, 17, 18, 19, 44)
+	assert.Contains(t, info.SupportedNIPs, "0", "1", "2", "12", "13", "17", "18", "19", "44")
 }
 
 func TestFetch(t *testing.T) {

nip11/types.go

@@ -14,7 +14,7 @@ type RelayInformationDocument struct {
 	PubKey        *nostr.PubKey `json:"pubkey,omitempty"`
 	Self          *nostr.PubKey `json:"self,omitempty"`
 	Contact       string        `json:"contact,omitempty"`
-	SupportedNIPs []any         `json:"supported_nips,omitempty"`
+	SupportedNIPs []string      `json:"supported_nips,omitempty"`
 	Software      string        `json:"software,omitempty"`
 	Version       string        `json:"version,omitempty"`
 
@@ -33,16 +33,16 @@ type RelayInformationDocument struct {
 	SupportedGrasps []string `json:"supported_grasps,omitempty"`
 }
 
-func (info *RelayInformationDocument) AddSupportedNIP(number int) {
+func (info *RelayInformationDocument) AddSupportedNIP(nip string) {
-	idx := slices.IndexFunc(info.SupportedNIPs, func(n any) bool { return n == number })
+	idx := slices.IndexFunc(info.SupportedNIPs, func(n string) bool { return n == nip })
 	if idx != -1 {
 		return
 	}
 
-	info.SupportedNIPs = append(info.SupportedNIPs, number)
+	info.SupportedNIPs = append(info.SupportedNIPs, nip)
 }
 
-func (info *RelayInformationDocument) AddSupportedNIPs(numbers []int) {
+func (info *RelayInformationDocument) AddSupportedNIPs(numbers []string) {
 	for _, n := range numbers {
 		info.AddSupportedNIP(n)
 	}

nip29/group.go

@@ -8,11 +8,18 @@ import (
 	"strings"
 
 	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/nip19"
 )
 
 type GroupAddress struct {
+	// URL of the relay that is hosting the group
 	Relay string
-	ID    string
+
+	// Group identifier ("d"/"h" tag)
+	ID string
+
+	// Public key of the relay, used to publish kind:39000/etc events
+	Self nostr.PubKey
 }
 
 func (gid GroupAddress) String() string {
@@ -20,6 +27,10 @@ func (gid GroupAddress) String() string {
 	return fmt.Sprintf("%s'%s", p.Host, gid.ID)
 }
 
+func (gid GroupAddress) Code() string {
+	return nip19.EncodeNaddr(gid.Self, 39000, gid.ID, []string{gid.Relay})
+}
+
 func (gid GroupAddress) IsValid() bool {
 	return gid.Relay != "" && gid.ID != ""
 }
@@ -28,14 +39,6 @@ func (gid GroupAddress) Equals(gid2 GroupAddress) bool {
 	return gid.Relay == gid2.Relay && gid.ID == gid2.ID
 }
 
-func ParseGroupAddress(raw string) (GroupAddress, error) {
-	spl := strings.Split(raw, "'")
-	if len(spl) != 2 {
-		return GroupAddress{}, fmt.Errorf("invalid group id")
-	}
-	return GroupAddress{ID: spl[1], Relay: nostr.NormalizeURL(spl[0])}, nil
-}
-
 type Group struct {
 	Address GroupAddress
 
@@ -63,6 +66,12 @@ type Group struct {
 	// indicates which event kinds this group supports
 	SupportedKinds []nostr.Kind
 
+	// arbitrary string indicating the parent group
+	Parent string
+
+	// ordered list of identifiers of child groups
+	Children []string
+
 	Roles       []*Role
 	InviteCodes []string
 
@@ -130,15 +139,15 @@ func (group Group) String() string {
 }
 
 // NewGroup takes a group address in the form "<id>'<relay-hostname>"
-func NewGroup(gadstr string) (Group, error) {
+func NewGroup(relayHost, groupId string) (Group, error) {
-	gad, err := ParseGroupAddress(gadstr)
+	relayHost = nostr.NormalizeURL(relayHost)
-	if err != nil {
-		return Group{}, fmt.Errorf("invalid group id '%s': %w", gadstr, err)
-	}
 
 	return Group{
-		Address:             gad,
+		Address: GroupAddress{
-		Name:                gad.ID,
+			Relay: relayHost,
+			ID:    groupId,
+		},
+		Name:                groupId,
 		Members:             make(map[nostr.PubKey][]*Role),
 		LiveKitParticipants: make([]nostr.PubKey, 0),
 	}, nil
@@ -203,6 +212,14 @@ func (group Group) ToMetadataEvent() nostr.Event {
 		evt.Tags = append(evt.Tags, tag)
 	}
 
+	if group.Parent != "" {
+		evt.Tags = append(evt.Tags, nostr.Tag{"parent", group.Parent})
+	}
+
+	for _, child := range group.Children {
+		evt.Tags = append(evt.Tags, nostr.Tag{"child", child})
+	}
+
 	return evt
 }
 
@@ -324,6 +341,10 @@ func (group *Group) MergeInMetadataEvent(evt *nostr.Event) error {
 						group.About = tag[1]
 					case "picture":
 						group.Picture = tag[1]
+					case "parent":
+						group.Parent = tag[1]
+					case "child":
+						group.Children = append(group.Children, tag[1])
 					}
 				}
 			}

nip29/moderation_actions.go

@@ -79,40 +79,23 @@ var moderationActionFactories = map[nostr.Kind]func(nostr.Event) (Action, error)
 	nostr.KindSimpleGroupEditMetadata: func(evt nostr.Event) (Action, error) {
 		ok := false
 		edit := EditMetadata{When: evt.CreatedAt}
-		y := true
-		n := false
-
-		hasName := false
-
-		// DEPRECATED: remove all the fields not tagged with Replace = true eventually
-		// edit-metadata to become a PUT rather than a PATCH
 
 		for _, tag := range evt.Tags {
 			if len(tag) >= 1 {
 				switch tag[0] {
 				case "name":
 					if len(tag) >= 2 {
-						edit.NameValue = &tag[1]
+						edit.Group.Name = tag[1]
-						if ok {
-							edit.Replace = true
-						}
 						ok = true
-						hasName = true
 					}
 				case "picture":
 					if len(tag) >= 2 {
-						edit.PictureValue = &tag[1]
+						edit.Group.Picture = tag[1]
-						if hasName {
-							edit.Replace = true
-						}
 						ok = true
 					}
 				case "about":
 					if len(tag) >= 2 {
-						edit.AboutValue = &tag[1]
+						edit.Group.About = tag[1]
-						if hasName {
-							edit.Replace = true
-						}
 						ok = true
 					}
 				case "supported_kinds":
@@ -124,54 +107,33 @@ var moderationActionFactories = map[nostr.Kind]func(nostr.Event) (Action, error)
 							kinds = append(kinds, nostr.Kind(kind))
 						}
 					}
-					edit.SupportedKindsValue = &kinds
+					edit.Group.SupportedKinds = kinds
-					edit.Replace = true
-				case "closed":
-					edit.ClosedValue = &y
-					if hasName {
-						edit.Replace = true
-					}
 					ok = true
-				case "open":
+				case "closed":
-					edit.ClosedValue = &n
+					edit.Group.Closed = true
 					ok = true
 				case "restricted":
-					edit.RestrictedValue = &y
+					edit.Group.Restricted = true
-					if hasName {
-						edit.Replace = true
-					}
-					ok = true
-				case "unrestricted":
-					edit.RestrictedValue = &n
 					ok = true
 				case "hidden":
-					edit.HiddenValue = &y
+					edit.Group.Hidden = true
-					if hasName {
-						edit.Replace = true
-					}
-					ok = true
-				case "visible":
-					edit.HiddenValue = &n
 					ok = true
 				case "private":
-					edit.PrivateValue = &y
+					edit.Group.Private = true
-					if hasName {
+					ok = true
-						edit.Replace = true
+				case "parent":
+					if len(tag) >= 2 {
+						edit.Group.Parent = tag[1]
+						ok = true
 					}
-					ok = true
-				case "public":
-					edit.PrivateValue = &n
-					ok = true
 				case "livekit":
-					edit.LiveKitValue = &y
+					edit.Group.LiveKit = true
-					edit.Replace = true
-					ok = true
-				case "no-livekit":
-					edit.LiveKitValue = &n
-					ok = true
-				case "no-text":
-					edit.SupportedKindsValue = nil
 					ok = true
+				case "child":
+					if len(tag) >= 2 {
+						edit.Group.Children = append(edit.Group.Children, tag[1])
+						ok = true
+					}
 				}
 			}
 		}
@@ -280,63 +242,26 @@ func (a RemoveUser) Apply(group *Group) {
 }
 
 type EditMetadata struct {
-	NameValue           *string
+	Group
-	PictureValue        *string
-	AboutValue          *string
-	RestrictedValue     *bool
-	ClosedValue         *bool
-	HiddenValue         *bool
-	PrivateValue        *bool
-	LiveKitValue        *bool
-	SupportedKindsValue *[]nostr.Kind
 
-	Replace bool
+	When nostr.Timestamp
-	When    nostr.Timestamp
 }
 
 func (_ EditMetadata) Name() string { return "edit-metadata" }
 func (a EditMetadata) Apply(group *Group) {
 	group.LastMetadataUpdate = a.When
 
-	if a.Replace {
+	group.Name = a.Group.Name
-		group.Name = ""
+	group.Picture = a.Group.Picture
-		group.Picture = ""
+	group.About = a.Group.About
-		group.About = ""
+	group.Restricted = a.Group.Restricted
-		group.Restricted = false
+	group.Closed = a.Group.Closed
-		group.Closed = false
+	group.Hidden = a.Group.Hidden
-		group.Hidden = false
+	group.Private = a.Group.Private
-		group.Private = false
+	group.LiveKit = a.Group.LiveKit
-		group.LiveKit = false
+	group.SupportedKinds = a.Group.SupportedKinds
-		group.SupportedKinds = nil
+	group.Parent = a.Group.Parent
-	}
+	group.Children = a.Group.Children
-
-	if a.NameValue != nil {
-		group.Name = *a.NameValue
-	}
-	if a.PictureValue != nil {
-		group.Picture = *a.PictureValue
-	}
-	if a.AboutValue != nil {
-		group.About = *a.AboutValue
-	}
-	if a.RestrictedValue != nil {
-		group.Restricted = *a.RestrictedValue
-	}
-	if a.ClosedValue != nil {
-		group.Closed = *a.ClosedValue
-	}
-	if a.HiddenValue != nil {
-		group.Hidden = *a.HiddenValue
-	}
-	if a.PrivateValue != nil {
-		group.Private = *a.PrivateValue
-	}
-	if a.LiveKitValue != nil {
-		group.LiveKit = *a.LiveKitValue
-	}
-	if a.SupportedKindsValue != nil {
-		group.SupportedKinds = *a.SupportedKindsValue
-	}
 }
 
 type CreateGroup struct {

nip29/nip29_test.go

@@ -15,7 +15,7 @@ const (
 )
 
 func TestGroupEventBackAndForth(t *testing.T) {
-	group1, _ := NewGroup("relay.com'xyz")
+	group1, _ := NewGroup("relay.com", "xyz")
 	group1.Name = "banana"
 	group1.Private = true
 	meta1 := group1.ToMetadataEvent()
@@ -31,7 +31,7 @@ func TestGroupEventBackAndForth(t *testing.T) {
 	}
 	require.True(t, hasPrivate, "translation of group1 to metadata event failed: %s", meta1)
 
-	group2, _ := NewGroup("groups.com'abc")
+	group2, _ := NewGroup("groups.com", "abc")
 	alicePub, _ := nostr.PubKeyFromHex(ALICE)
 	group2.Members[alicePub] = []*Role{{Name: "nada"}}
 	admins2 := group2.ToAdminsEvent()

nip46/client.go

@@ -6,7 +6,9 @@ import (
 	"math/rand"
 	"net/url"
 	"strconv"
+	"sync"
 	"sync/atomic"
+	"time"
 	"unsafe"
 
 	"fiatjaf.com/nostr"
@@ -15,6 +17,12 @@ import (
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
+var bunkerClientCtxKey = &struct{}{}
+
+func IsBunkerClientOperation(ctx context.Context) bool {
+	return ctx.Value(bunkerClientCtxKey) == true
+}
+
 type BunkerClient struct {
 	Relays []string
 
@@ -56,6 +64,7 @@ func ConnectBunker(
 		pool,
 		onAuth,
 	)
+
 	_, err = bunker.RPC(ctx, "connect", []string{nostr.HexEncodeToString(parsed.HostPubKey[:]), parsed.Secret})
 	return bunker, err
 }
@@ -131,17 +140,19 @@ func NewBunker(
 	}
 
 	cancellableCtx, cancel := context.WithCancel(ctx)
+	bunkerClientCtx := context.WithValue(cancellableCtx, bunkerClientCtxKey, true)
 	_ = cancel
 
+	events, eosed := pool.SubscribeManyNotifyEOSE(bunkerClientCtx, relays, nostr.Filter{
+		Tags:      nostr.TagMap{"p": []string{clientPublicKey.Hex()}},
+		Kinds:     []nostr.Kind{nostr.KindNostrConnect},
+		Since:     now,
+		LimitZero: true,
+	}, nostr.SubscriptionOptions{
+		Label: "bunker46client",
+	})
+
 	go func() {
-		events := pool.SubscribeMany(cancellableCtx, relays, nostr.Filter{
-			Tags:      nostr.TagMap{"p": []string{clientPublicKey.Hex()}},
-			Kinds:     []nostr.Kind{nostr.KindNostrConnect},
-			Since:     now,
-			LimitZero: true,
-		}, nostr.SubscriptionOptions{
-			Label: "bunker46client",
-		})
 		for ie := range events {
 			if ie.Kind != nostr.KindNostrConnect {
 				continue
@@ -174,12 +185,15 @@ func NewBunker(
 
 	// attempt switch_relays once every 10 times
 	if now%10 == 0 {
-		if newRelays, _ := bunker.SwitchRelays(ctx); newRelays != nil {
+		swctx, cancel := context.WithTimeout(ctx, time.Second*3)
-			cancel()
+		if newRelays, _ := bunker.SwitchRelays(swctx); newRelays != nil {
 			bunker = NewBunker(ctx, clientSecretKey, targetPublicKey, newRelays, pool, func(string) {})
 		}
+		cancel()
 	}
 
+	<-eosed
+
 	return bunker
 }
 
@@ -274,7 +288,7 @@ func (bunker *BunkerClient) NIP04Decrypt(
 }
 
 func (bunker *BunkerClient) RPC(ctx context.Context, method string, params []string) (string, error) {
-	id := bunker.idPrefix + "-" + strconv.FormatUint(bunker.serial.Add(1), 10)
+	id := bunker.idPrefix + "-" + strconv.FormatUint(bunker.serial.Add(1), 10) + "-" + method
 	req, err := json.Marshal(Request{
 		ID:     id,
 		Method: method,
@@ -303,21 +317,23 @@ func (bunker *BunkerClient) RPC(ctx context.Context, method string, params []str
 	bunker.listeners.Store(id, dispatcher)
 	defer bunker.listeners.Delete(id)
 	relayConnectionWorked := make(chan struct{})
+	relayConnectionWorkedO := sync.OnceFunc(func() {
+		close(relayConnectionWorked)
+	})
 	bunkerConnectionWorked := make(chan struct{})
+	bunkerConnectionWorkedO := sync.OnceFunc(func() {
+		close(bunkerConnectionWorked)
+	})
+
+	bunkerClientCtx := context.WithValue(ctx, bunkerClientCtxKey, true)
 
 	for _, url := range bunker.Relays {
 		go func(url string) {
 			relay, err := bunker.pool.EnsureRelay(url)
 			if err == nil {
-				select {
+				relayConnectionWorkedO()
-				case relayConnectionWorked <- struct{}{}:
+				if err := relay.Publish(bunkerClientCtx, evt); err == nil {
-				default:
+					bunkerConnectionWorkedO()
-				}
-				if err := relay.Publish(ctx, evt); err == nil {
-					select {
-					case bunkerConnectionWorked <- struct{}{}:
-					default:
-					}
 				}
 			}
 		}(url)

nip46/dynamic-signer.go

@@ -3,6 +3,8 @@ package nip46
 import (
 	"context"
 	"fmt"
+	"net/url"
+	"strconv"
 	"sync"
 
 	"fiatjaf.com/nostr"
@@ -50,6 +52,50 @@ func (p *DynamicSigner) Init() {
 	p.sessions = make(map[nostr.PubKey]map[nostr.PubKey]*Session)
 }
 
+// HandleNostrConnectURI works like HandleRequest, but takes a nostrconnect:// URI as input, as scanned/pasted
+// by the user, produced by the client. Since DynamicSigner can serve multiple handler keys, the caller must
+// specify which handlerPubkey should respond to this connection.
+func (p *DynamicSigner) HandleNostrConnectURI(ctx context.Context, handlerPubkey nostr.PubKey, uri *url.URL) (
+	resp Response,
+	eventResponse nostr.Event,
+	err error,
+) {
+	clientPublicKey, err := nostr.PubKeyFromHex(uri.Host)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+
+	secret := uri.Query().Get("secret")
+
+	_, handlerSecret, err := p.GetHandlerSecretKey(ctx, handlerPubkey)
+	if err != nil {
+		return resp, eventResponse, fmt.Errorf("no private key for %s: %w", handlerPubkey, err)
+	}
+
+	// pretend they started with a request
+	conversationKey, err := nip44.GenerateConversationKey(clientPublicKey, handlerSecret)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+	reqj, _ := json.Marshal(Request{
+		ID:     "nostrconnect-" + strconv.FormatInt(int64(nostr.Now()), 10),
+		Method: "imagined-nostrconnect",
+		Params: []string{clientPublicKey.Hex(), secret},
+	})
+	ciphertext, err := nip44.Encrypt(string(reqj), conversationKey)
+	if err != nil {
+		return resp, eventResponse, err
+	}
+
+	_, resp, eventResponse, err = p.HandleRequest(ctx, nostr.Event{
+		PubKey:  clientPublicKey,
+		Kind:    nostr.KindNostrConnect,
+		Content: ciphertext,
+		Tags:    nostr.Tags{nostr.Tag{"p", handlerPubkey.Hex()}},
+	})
+	return resp, eventResponse, err
+}
+
 func (p *DynamicSigner) HandleRequest(ctx context.Context, event nostr.Event) (
 	req Request,
 	resp Response,
@@ -118,6 +164,19 @@ func (p *DynamicSigner) HandleRequest(ctx context.Context, event nostr.Event) (
 	var resultErr error
 
 	switch req.Method {
+	case "imagined-nostrconnect":
+		// this is a fake request we pretend has existed, but was actually just we reading the nostrconnect:// uri
+		if len(req.Params) < 2 || req.Params[1] == "" {
+			resultErr = fmt.Errorf("needs a second argument 'secret'")
+			break
+		}
+		if p.OnConnect != nil {
+			if err := p.OnConnect(ctx, event.PubKey, req.Params[1]); err != nil {
+				resultErr = err
+				break
+			}
+		}
+		result = req.Params[1]
 	case "connect":
 		var secret string
 		if len(req.Params) >= 2 {

nip5a/nip5a.go

@@ -97,7 +97,9 @@ func (sm SiteManifest) ToEvent() nostr.Event {
 		event.Tags = append(event.Tags, nostr.Tag{"path", NormalizePath(path), hex.EncodeToString(hash[:])})
 	}
 	for _, s := range sm.Servers {
-		event.Tags = append(event.Tags, nostr.Tag{"server", s})
+		if ns, err := nostr.NormalizeHTTPURL(s); err == nil {
+			event.Tags = append(event.Tags, nostr.Tag{"server", ns})
+		}
 	}
 	if sm.Title != "" {
 		event.Tags = append(event.Tags, nostr.Tag{"title", sm.Title})

nip5a/nip5a_test.go

@@ -234,4 +234,3 @@ func mustHash(s string) [32]byte {
 	copy(h[:], b)
 	return h
 }
-

nip86/methods.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math"
 	"net"
+	"strconv"
 
 	"fiatjaf.com/nostr"
 )
@@ -32,6 +33,24 @@ func DecodeRequest(req Request) (MethodParams, error) {
 		return BanPubKey{pk, reason}, nil
 	case "listbannedpubkeys":
 		return ListBannedPubKeys{}, nil
+	case "unbanpubkey":
+		if len(req.Params) == 0 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		pkh, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing pubkey param for '%s'", req.Method)
+		}
+		pk, err := nostr.PubKeyFromHex(pkh)
+		if err != nil {
+			return nil, fmt.Errorf("invalid pubkey param for '%s'", req.Method)
+		}
+
+		var reason string
+		if len(req.Params) >= 2 {
+			reason, _ = req.Params[1].(string)
+		}
+		return UnbanPubKey{pk, reason}, nil
 	case "allowpubkey":
 		if len(req.Params) == 0 {
 			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
@@ -52,6 +71,24 @@ func DecodeRequest(req Request) (MethodParams, error) {
 		return AllowPubKey{pk, reason}, nil
 	case "listallowedpubkeys":
 		return ListAllowedPubKeys{}, nil
+	case "unallowpubkey":
+		if len(req.Params) == 0 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		pkh, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing pubkey param for '%s'", req.Method)
+		}
+		pk, err := nostr.PubKeyFromHex(pkh)
+		if err != nil {
+			return nil, fmt.Errorf("invalid pubkey param for '%s'", req.Method)
+		}
+
+		var reason string
+		if len(req.Params) >= 2 {
+			reason, _ = req.Params[1].(string)
+		}
+		return UnallowPubKey{pk, reason}, nil
 	case "listeventsneedingmoderation":
 		return ListEventsNeedingModeration{}, nil
 	case "allowevent":
@@ -204,6 +241,95 @@ func DecodeRequest(req Request) (MethodParams, error) {
 			Pubkey:          pk,
 			DisallowMethods: disallowedMethods,
 		}, nil
+	case "createrole":
+		if len(req.Params) == 0 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		id, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing id param for '%s'", req.Method)
+		}
+		var label, description string
+		if len(req.Params) >= 2 {
+			label, _ = req.Params[1].(string)
+		}
+		if len(req.Params) >= 3 {
+			description, _ = req.Params[2].(string)
+		}
+		var color, order int
+		if len(req.Params) >= 4 {
+			color = coerceInt(req.Params[3])
+		}
+		if len(req.Params) >= 5 {
+			order = coerceInt(req.Params[4])
+		}
+		return CreateRole{id, label, description, color, order}, nil
+	case "editrole":
+		if len(req.Params) == 0 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		id, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing id param for '%s'", req.Method)
+		}
+		var label, description string
+		if len(req.Params) >= 2 {
+			label, _ = req.Params[1].(string)
+		}
+		if len(req.Params) >= 3 {
+			description, _ = req.Params[2].(string)
+		}
+		var color, order int
+		if len(req.Params) >= 4 {
+			color = coerceInt(req.Params[3])
+		}
+		if len(req.Params) >= 5 {
+			order = coerceInt(req.Params[4])
+		}
+		return EditRole{id, label, description, color, order}, nil
+	case "deleterole":
+		if len(req.Params) == 0 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		id, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing id param for '%s'", req.Method)
+		}
+		return DeleteRole{id}, nil
+	case "assignrole":
+		if len(req.Params) < 2 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		pkh, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing pubkey param for '%s'", req.Method)
+		}
+		pk, err := nostr.PubKeyFromHex(pkh)
+		if err != nil {
+			return nil, fmt.Errorf("invalid pubkey param for '%s'", req.Method)
+		}
+		roleID, ok := req.Params[1].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing role id param for '%s'", req.Method)
+		}
+		return AssignRole{pk, roleID}, nil
+	case "unassignrole":
+		if len(req.Params) < 2 {
+			return nil, fmt.Errorf("invalid number of params for '%s'", req.Method)
+		}
+		pkh, ok := req.Params[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing pubkey param for '%s'", req.Method)
+		}
+		pk, err := nostr.PubKeyFromHex(pkh)
+		if err != nil {
+			return nil, fmt.Errorf("invalid pubkey param for '%s'", req.Method)
+		}
+		roleID, ok := req.Params[1].(string)
+		if !ok {
+			return nil, fmt.Errorf("missing role id param for '%s'", req.Method)
+		}
+		return UnassignRole{pk, roleID}, nil
 	case "stats":
 		return Stats{}, nil
 	default:
@@ -211,6 +337,19 @@ func DecodeRequest(req Request) (MethodParams, error) {
 	}
 }
 
+// coerceInt converts a decoded JSON param (a float64 number or a numeric
+// string) into an int, returning 0 when the value is neither.
+func coerceInt(v any) int {
+	switch n := v.(type) {
+	case float64:
+		return int(n)
+	case string:
+		i, _ := strconv.Atoi(n)
+		return i
+	}
+	return 0
+}
+
 type MethodParams interface {
 	MethodName() string
 }
@@ -219,8 +358,10 @@ var (
 	_ MethodParams = (*SupportedMethods)(nil)
 	_ MethodParams = (*BanPubKey)(nil)
 	_ MethodParams = (*ListBannedPubKeys)(nil)
+	_ MethodParams = (*UnbanPubKey)(nil)
 	_ MethodParams = (*AllowPubKey)(nil)
 	_ MethodParams = (*ListAllowedPubKeys)(nil)
+	_ MethodParams = (*UnallowPubKey)(nil)
 	_ MethodParams = (*ListEventsNeedingModeration)(nil)
 	_ MethodParams = (*AllowEvent)(nil)
 	_ MethodParams = (*BanEvent)(nil)
@@ -238,6 +379,11 @@ var (
 	_ MethodParams = (*ListDisallowedKinds)(nil)
 	_ MethodParams = (*GrantAdmin)(nil)
 	_ MethodParams = (*RevokeAdmin)(nil)
+	_ MethodParams = (*CreateRole)(nil)
+	_ MethodParams = (*EditRole)(nil)
+	_ MethodParams = (*DeleteRole)(nil)
+	_ MethodParams = (*AssignRole)(nil)
+	_ MethodParams = (*UnassignRole)(nil)
 	_ MethodParams = (*Stats)(nil)
 )
 
@@ -256,6 +402,13 @@ type ListBannedPubKeys struct{}
 
 func (ListBannedPubKeys) MethodName() string { return "listbannedpubkeys" }
 
+type UnbanPubKey struct {
+	PubKey nostr.PubKey
+	Reason string
+}
+
+func (UnbanPubKey) MethodName() string { return "unbanpubkey" }
+
 type AllowPubKey struct {
 	PubKey nostr.PubKey
 	Reason string
@@ -267,6 +420,13 @@ type ListAllowedPubKeys struct{}
 
 func (ListAllowedPubKeys) MethodName() string { return "listallowedpubkeys" }
 
+type UnallowPubKey struct {
+	PubKey nostr.PubKey
+	Reason string
+}
+
+func (UnallowPubKey) MethodName() string { return "unallowpubkey" }
+
 type ListEventsNeedingModeration struct{}
 
 func (ListEventsNeedingModeration) MethodName() string { return "listeventsneedingmoderation" }
@@ -363,6 +523,46 @@ type RevokeAdmin struct {
 
 func (RevokeAdmin) MethodName() string { return "revokeadmin" }
 
+type CreateRole struct {
+	ID          string
+	Label       string
+	Description string
+	Color       int
+	Order       int
+}
+
+func (CreateRole) MethodName() string { return "createrole" }
+
+type EditRole struct {
+	ID          string
+	Label       string
+	Description string
+	Color       int
+	Order       int
+}
+
+func (EditRole) MethodName() string { return "editrole" }
+
+type DeleteRole struct {
+	ID string
+}
+
+func (DeleteRole) MethodName() string { return "deleterole" }
+
+type AssignRole struct {
+	PubKey nostr.PubKey
+	RoleID string
+}
+
+func (AssignRole) MethodName() string { return "assignrole" }
+
+type UnassignRole struct {
+	PubKey nostr.PubKey
+	RoleID string
+}
+
+func (UnassignRole) MethodName() string { return "unassignrole" }
+
 type Stats struct{}
 
 func (Stats) MethodName() string { return "stats" }

nipb0/blossom/blossom_test.go

@@ -3,6 +3,8 @@ package blossom_test
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"net/http/httptest"
@@ -18,6 +20,18 @@ import (
 	blossomclient "fiatjaf.com/nostr/nipb0/blossom"
 )
 
+func hexTo32(s string) (h [32]byte, err error) {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return h, err
+	}
+	if len(b) != 32 {
+		return h, errors.New("not 32 bytes")
+	}
+	copy(h[:], b)
+	return h, nil
+}
+
 func TestBlossomBasicOperations(t *testing.T) {
 	// setup two test servers
 	server1 := setupTestServer(t, ":38081")
@@ -78,7 +92,10 @@ func TestBlossomBasicOperations(t *testing.T) {
 			t.Fatalf("Expected 1 blob, got %d", len(blobs))
 		}
 
-		hash := blobs[0].SHA256
+		hash, err := hexTo32(blobs[0].SHA256)
+		if err != nil {
+			t.Fatalf("Failed to parse hash: %v", err)
+		}
 		downloaded, err := client1.Download(ctx, hash)
 		if err != nil {
 			t.Fatalf("Failed to download blob: %v", err)
@@ -173,8 +190,12 @@ func TestBlossomBasicOperations(t *testing.T) {
 			t.Errorf("Expected pubkey2 to still see 1 blob after pubkey1 delete, got %d", len(blobs2))
 		}
 
+		hash32, err := hexTo32(hash)
+		if err != nil {
+			t.Fatalf("Failed to parse hash: %v", err)
+		}
 		// download should still work
-		downloaded, err := client2.Download(ctx, hash)
+		downloaded, err := client2.Download(ctx, hash32)
 		if err != nil {
 			t.Fatalf("Failed to download blob after pubkey1 delete: %v", err)
 		}
@@ -203,8 +224,12 @@ func TestBlossomBasicOperations(t *testing.T) {
 			t.Errorf("Expected 1 blob on server2, got %d", len(blobs))
 		}
 
+		hash32, err := hexTo32(bd.SHA256)
+		if err != nil {
+			t.Fatalf("Failed to parse hash: %v", err)
+		}
 		// verify download
-		downloaded, err := client2Server.Download(ctx, bd.SHA256)
+		downloaded, err := client2Server.Download(ctx, hash32)
 		if err != nil {
 			t.Fatalf("Failed to download from server2: %v", err)
 		}
@@ -246,13 +271,21 @@ func TestBlossomBasicOperations(t *testing.T) {
 		}
 
 		// verify the mirrored blob can be downloaded
-		downloaded, err := client2Server.Download(ctx, bd.SHA256)
+		hash32, err := hexTo32(bd.SHA256)
+		if err != nil {
+			t.Fatalf("Failed to parse hash: %v", err)
+		}
+		downloaded, err := client2Server.Download(ctx, hash32)
 		if err != nil {
 			t.Fatalf("Failed to download mirrored blob: %v", err)
 		}
 
+		hash32, err = hexTo32(blobs1[0].SHA256)
+		if err != nil {
+			t.Fatalf("Failed to parse hash: %v", err)
+		}
 		// should match the original content
-		originalDownloaded, err := client2.Download(ctx, blobs1[0].SHA256)
+		originalDownloaded, err := client2.Download(ctx, hash32)
 		if err != nil {
 			t.Fatalf("Failed to download original blob for comparison: %v", err)
 		}

nipb0/blossom/client.go

@@ -6,6 +6,7 @@ import (
 
 	"fiatjaf.com/nostr"
 	"github.com/valyala/fasthttp"
+	"github.com/valyala/fasthttp/fasthttpproxy"
 )
 
 // Client represents a Blossom client for interacting with a media server
@@ -28,6 +29,18 @@ func NewClient(mediaserver string, signer nostr.Signer) *Client {
 
 // createHTTPClient creates a properly configured HTTP client
 func createHTTPClient() *fasthttp.Client {
+	d := fasthttpproxy.Dialer{
+		Timeout:        10 * time.Second,
+		ConnectTimeout: 10 * time.Second,
+		TCPDialer: fasthttp.TCPDialer{
+			// increase DNS cache time to an hour instead of default minute
+			Concurrency:      4096,
+			DNSCacheDuration: time.Hour,
+		},
+		DialDualStack: true,
+	}
+	dialFunc, _ := d.GetDialFunc(true)
+
 	return &fasthttp.Client{
 		MaxIdleConnDuration:           time.Hour,
 		DisableHeaderNamesNormalizing: true, // because our headers are properly constructed
@@ -35,11 +48,7 @@ func createHTTPClient() *fasthttp.Client {
 
 		Name: "nl-b", // user-agent
 
-		// increase DNS cache time to an hour instead of default minute
+		Dial: dialFunc,
-		Dial: (&fasthttp.TCPDialer{
-			Concurrency:      4096,
-			DNSCacheDuration: time.Hour,
-		}).Dial,
 	}
 }
 

nipb0/blossom/download.go

@@ -15,7 +15,7 @@ import (
 func (c *Client) Download(ctx context.Context, hash [32]byte) ([]byte, error) {
 	hhash := hex.EncodeToString(hash[:])
 
-	req, err := http.NewRequestWithContext(ctx, "GET", c.mediaserver+"/"+hhash, nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", c.mediaserver+hhash, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
@@ -43,7 +43,7 @@ func (c *Client) Download(ctx context.Context, hash [32]byte) ([]byte, error) {
 func (c *Client) DownloadToFile(ctx context.Context, hash [32]byte, filePath string) error {
 	hhash := hex.EncodeToString(hash[:])
 
-	req, err := http.NewRequestWithContext(ctx, "GET", c.mediaserver+"/"+hhash, nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", c.mediaserver+hhash, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create request: %w", err)
 	}

nipb0/blossom/utils.go

@@ -2,37 +2,109 @@ package blossom
 
 import (
 	"mime"
+	"strings"
 )
 
-func GetExtension(mimetype string) string {
+var commonMimeExtensions = map[string]string{
+	"application/json":                        ".json",
+	"application/pdf":                         ".pdf",
+	"application/vnd.android.package-archive": ".apk",
+	"application/vnd.sqlite3":                 ".sqlite3",
+	"application/xml":                         ".xml",
+	"audio/aac":                               ".aac",
+	"audio/flac":                              ".flac",
+	"audio/midi":                              ".midi",
+	"audio/mp3":                               ".mp3",
+	"audio/mpeg":                              ".mp3",
+	"audio/mp4":                               ".m4a",
+	"audio/ogg":                               ".ogg",
+	"audio/wav":                               ".wav",
+	"audio/webm":                              ".weba",
+	"audio/x-aiff":                            ".aiff",
+	"audio/x-m4a":                             ".m4a",
+	"image/avif":                              ".avif",
+	"image/gif":                               ".gif",
+	"image/jpeg":                              ".jpg",
+	"image/png":                               ".png",
+	"image/svg+xml":                           ".svg",
+	"image/webp":                              ".webp",
+	"text/css":                                ".css",
+	"text/csv":                                ".csv",
+	"text/html":                               ".html",
+	"text/javascript":                         ".js",
+	"text/markdown":                           ".md",
+	"text/plain":                              ".txt",
+	"text/xml":                                ".xml",
+	"video/mp2t":                              ".ts",
+	"video/mp4":                               ".mp4",
+	"video/ogg":                               ".ogv",
+	"video/quicktime":                         ".mov",
+	"video/webm":                              ".webm",
+	"video/x-matroska":                        ".mkv",
+}
+
+var commonExtensionMimes = map[string]string{
+	".aac":     "audio/aac",
+	".aiff":    "audio/x-aiff",
+	".apk":     "application/vnd.android.package-archive",
+	".avif":    "image/avif",
+	".css":     "text/css; charset=utf-8",
+	".csv":     "text/csv; charset=utf-8",
+	".flac":    "audio/flac",
+	".gif":     "image/gif",
+	".html":    "text/html; charset=utf-8",
+	".jpeg":    "image/jpeg",
+	".jpg":     "image/jpeg",
+	".js":      "text/javascript; charset=utf-8",
+	".json":    "application/json",
+	".m4a":     "audio/mp4",
+	".md":      "text/markdown; charset=utf-8",
+	".midi":    "audio/midi",
+	".mkv":     "video/x-matroska",
+	".mov":     "video/quicktime",
+	".mp3":     "audio/mpeg",
+	".mp4":     "video/mp4",
+	".oga":     "audio/ogg",
+	".ogg":     "audio/ogg",
+	".ogv":     "video/ogg",
+	".pdf":     "application/pdf",
+	".png":     "image/png",
+	".sqlite3": "application/vnd.sqlite3",
+	".svg":     "image/svg+xml",
+	".ts":      "video/mp2t",
+	".txt":     "text/plain; charset=utf-8",
+	".wav":     "audio/wav",
+	".weba":    "audio/webm",
+	".webm":    "video/webm",
+	".webp":    "image/webp",
+	".xml":     "application/xml",
+}
+
+func normalizeMIMEType(mimetype string) string {
 	if mimetype == "" {
 		return ""
 	}
 
-	// hardcode some common cases (abd jbiwb oribkenatuc cases kuje ,ogg/.oga or .mov/.moov)
+	base, _, err := mime.ParseMediaType(mimetype)
-	switch mimetype {
+	if err == nil {
-	case "image/jpeg":
+		return strings.ToLower(base)
-		return ".jpg"
+	}
-	case "image/gif":
+
-		return ".gif"
+	if idx := strings.IndexByte(mimetype, ';'); idx >= 0 {
-	case "image/png":
+		mimetype = mimetype[:idx]
-		return ".png"
+	}
-	case "image/webp":
+
-		return ".webp"
+	return strings.ToLower(strings.TrimSpace(mimetype))
-	case "video/mp4":
+}
-		return ".mp4"
+
-	case "application/vnd.android.package-archive":
+func GetExtension(mimetype string) string {
-		return ".apk"
+	mimetype = normalizeMIMEType(mimetype)
-	case "video/quicktime":
+	if mimetype == "" {
-		return ".mov"
+		return ""
-	case "application/vnd.sqlite3":
+	}
-		return "sqlite3"
+
-	case "text/markdown":
+	if ext, ok := commonMimeExtensions[mimetype]; ok {
-		return "md"
+		return ext
-	case "audio/midi":
-		return "midi"
-	case "audio/x-aiff":
-		return "aiff"
 	}
 
 	exts, _ := mime.ExtensionsByType(mimetype)
@@ -42,3 +114,20 @@ func GetExtension(mimetype string) string {
 
 	return ""
 }
+
+func GetMIMEType(ext string) string {
+	if ext == "" {
+		return ""
+	}
+
+	ext = strings.ToLower(strings.TrimSpace(ext))
+	if ext != "" && ext[0] != '.' {
+		ext = "." + ext
+	}
+
+	if mimetype, ok := commonExtensionMimes[ext]; ok {
+		return mimetype
+	}
+
+	return mime.TypeByExtension(ext)
+}

pointers_easyjson.go

@@ -313,7 +313,7 @@ func easyjson33014d6eEncodeFiatjafComNostr2(out *jwriter.Writer, in EntityPointe
 		out.RawString(prefix)
 		out.Uint(uint(in.Kind))
 	}
-	if in.Identifier == "" && in.Kind.IsReplaceable() {
+	if in.Identifier != "" || in.Kind.IsAddressable() {
 		// this is expected, no identifiers in replaceable events, so don't print
 		// but we do print in case there is an identifier, incorrectly, to assist debug
 		const prefix string = ",\"identifier\":"

pool.go

@@ -46,7 +46,8 @@ type Pool struct {
 	RelayOptions RelayOptions
 
 	// custom things not often used
-	penaltyBox *xsync.MapOf[string, [2]float64]
+	penaltyBox       *xsync.MapOf[string, [2]float64]
+	penaltyBoxCancel context.CancelFunc
 }
 
 // DirectedFilter combines a Filter with a specific relay URL.
@@ -74,13 +75,19 @@ func NewPool() *Pool {
 }
 
 func (pool *Pool) StartPenaltyBox() {
+	if pool.penaltyBoxCancel != nil {
+		pool.penaltyBoxCancel()
+	}
+
+	ctx, cancel := context.WithCancel(pool.Context)
+	pool.penaltyBoxCancel = cancel
 	pool.penaltyBox = xsync.NewMapOf[string, [2]float64]()
 
 	go func() {
 		sleep := 30.0
 		for {
 			select {
-			case <-pool.Context.Done():
+			case <-ctx.Done():
 				return
 			case <-time.After(time.Duration(sleep) * time.Second):
 
@@ -106,6 +113,15 @@ func (pool *Pool) StartPenaltyBox() {
 	}()
 }
 
+func (pool *Pool) StopPenaltyBox() {
+	if pool.penaltyBoxCancel != nil {
+		pool.penaltyBoxCancel()
+		pool.penaltyBoxCancel = nil
+	}
+
+	pool.penaltyBox = nil
+}
+
 // AddToPenaltyBox manually adds a relay to the penalty box for the specified duration.
 // This prevents EnsureRelay from attempting to connect to the relay until the duration expires.
 func (pool *Pool) AddToPenaltyBox(url string, duration time.Duration) {
@@ -143,7 +159,7 @@ func (pool *Pool) EnsureRelay(url string) (*Relay, error) {
 		if pool.penaltyBox != nil {
 			// putting relay in penalty box
 			pool.penaltyBox.Compute(nm, func(v [2]float64, loaded bool) (newV [2]float64, delete bool) {
-				return [2]float64{v[0] + 1, 30.0 + math.Pow(2, v[0]+1)}, false
+				return [2]float64{v[0] + 1, math.Min(600.0, 30.0*math.Pow(1.5, v[0]+1))}, false
 			})
 			pool.Relays.Store(nm, nil) // this is important for penalty box detection on EnsureRelay
 		}

relay.go

@@ -494,11 +494,11 @@ func (r *Relay) Auth(ctx context.Context, sign func(context.Context, *Event) err
 			},
 			Content: "",
 		}
-		if err := sign(ctx, &authEvent); err != nil {
+		if err = sign(ctx, &authEvent); err != nil {
 			err = fmt.Errorf("error signing auth event: %w", err)
+		} else {
+			err = r.publish(ctx, authEvent.ID, &AuthEnvelope{Event: authEvent})
 		}
-
-		err = r.publish(ctx, authEvent.ID, &AuthEnvelope{Event: authEvent})
 	})
 
 	if err == nil {

schema/schema.go

@@ -18,7 +18,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-const DefaultSchemaURL = "https://raw.githubusercontent.com/nostr-protocol/registry-of-kinds/ffa18bf6fb5496d755b465b062e18c676df1a5d4/schema.yaml"
+const DefaultSchemaURL = "https://raw.githubusercontent.com/nostr-protocol/registry-of-kinds/refs/heads/master/schema.yaml"
 
 // this is used by hex.Decode in the "hex" validator -- we don't care about data races
 var hexdummydecoder = make([]byte, 128)
@@ -39,64 +39,74 @@ func FetchSchemaFromURL(schemaURL string) (Schema, error) {
 		return Schema{}, fmt.Errorf("failed to read schema response: %w", err)
 	}
 
-	var schema Schema
+	return NewSchemaFromBytes(body)
-	if err := yaml.Unmarshal(body, &schema); err != nil {
-		return Schema{}, fmt.Errorf("failed to parse schema: %w", err)
-	}
-
-	return schema, nil
 }
 
 type Schema struct {
-	GenericTags map[string]ContentSpec `yaml:"generic_tags"`
+	GenericTags map[string]ContentSpec `yaml:"generic_tags" json:"generic_tags,omitempty"`
-	Kinds       map[string]KindSchema  `yaml:"kinds"`
+	Kinds       map[string]*KindSchema `yaml:"kinds" json:"kinds,omitempty"`
 }
 
 type KindSchema struct {
-	Description string      `yaml:"description"`
+	Kind        nostr.Kind  `yaml:"-" json:"kind"`
-	InUse       bool        `yaml:"in_use"`
+	Description string      `yaml:"description" json:"description,omitempty"`
-	Content     ContentSpec `yaml:"content"`
+	InUse       bool        `yaml:"in_use" json:"in_use,omitempty"`
-	Required    []string    `yaml:"required"`
+	Content     ContentSpec `yaml:"content" json:"content,omitempty"`
-	Multiple    []string    `yaml:"multiple"`
+	Required    []string    `yaml:"required" json:"required,omitempty"`
-	Tags        []TagSpec   `yaml:"tags"`
+	Multiple    []string    `yaml:"multiple" json:"multiple,omitempty"`
+	Tags        []TagSpec   `yaml:"tags" json:"tags,omitempty"`
 }
 
 type TagSpec struct {
-	Name   string       `yaml:"name"`
+	Name   string       `yaml:"name" json:"name,omitempty"`
-	Prefix string       `yaml:"prefix"`
+	Prefix string       `yaml:"prefix" json:"prefix,omitempty"`
-	Next   *ContentSpec `yaml:"next"`
+	Next   *ContentSpec `yaml:"next" json:"next,omitempty"`
 }
 
 type ContentSpec struct {
-	Type     string       `yaml:"type"`
+	Type     string       `yaml:"type" json:"type,omitempty"`
-	Required bool         `yaml:"required"`
+	Required bool         `yaml:"required" json:"required,omitempty"`
-	Min      int          `yaml:"min"`
+	Min      int          `yaml:"min" json:"min,omitempty"`
-	Max      int          `yaml:"max"`
+	Max      int          `yaml:"max" json:"max,omitempty"`
-	Either   []string     `yaml:"either"`
+	Either   []string     `yaml:"either" json:"either,omitempty"`
-	Next     *ContentSpec `yaml:"next"`
+	Next     *ContentSpec `yaml:"next" json:"next,omitempty"`
-	Variadic bool         `yaml:"variadic"`
+	Variadic bool         `yaml:"variadic" json:"variadic,omitempty"`
 }
 
 type Validator struct {
-	Schema            Schema
+	Schema            Schema                                                 `json:"schema,omitempty"`
-	FailOnUnknownKind bool
+	FailOnUnknownKind bool                                                   `json:"fail_on_unknown_kind,omitempty"`
-	FailOnUnknownType bool
+	FailOnUnknownType bool                                                   `json:"fail_on_unknown_type,omitempty"`
-	TypeValidators    map[string]func(value string, spec *ContentSpec) error
+	TypeValidators    map[string]func(value string, spec *ContentSpec) error `json:"type_validators,omitempty"`
-	UnknownTypes      []string
+	UnknownTypes      []string                                               `json:"unknown_types,omitempty"`
 }
 
 func NewValidatorFromBytes(schemaData []byte) (Validator, error) {
-	schema := Schema{
+	schema, err := NewSchemaFromBytes(schemaData)
-		GenericTags: make(map[string]ContentSpec),
+	if err != nil {
-		Kinds:       make(map[string]KindSchema),
+		return Validator{}, err
-	}
-	if err := yaml.Unmarshal(schemaData, &schema); err != nil {
-		return Validator{}, fmt.Errorf("failed to parse schema: %w", err)
 	}
 
 	return NewValidatorFromSchema(schema), nil
 }
 
+func NewSchemaFromBytes(schemaData []byte) (Schema, error) {
+	schema := Schema{
+		GenericTags: make(map[string]ContentSpec),
+		Kinds:       make(map[string]*KindSchema),
+	}
+	if err := yaml.Unmarshal(schemaData, &schema); err != nil {
+		return Schema{}, fmt.Errorf("failed to parse schema: %w", err)
+	}
+
+	for k := range schema.Kinds {
+		kn, _ := strconv.ParseUint(k, 10, 16)
+		schema.Kinds[k].Kind = nostr.Kind(kn)
+	}
+
+	return schema, nil
+}
+
 func NewValidatorFromSchema(sch Schema) Validator {
 	validator := Validator{
 		Schema: sch,
@@ -185,11 +195,19 @@ func NewValidatorFromSchema(sch Schema) Validator {
 }
 
 func NewValidatorFromFile(filename string) (Validator, error) {
+	schema, err := NewSchemaFromFile(filename)
+	if err != nil {
+		return Validator{}, err
+	}
+	return NewValidatorFromSchema(schema), nil
+}
+
+func NewSchemaFromFile(filename string) (Schema, error) {
 	data, err := os.ReadFile(filename)
 	if err != nil {
-		return Validator{}, fmt.Errorf("failed to read schema file: %w", err)
+		return Schema{}, fmt.Errorf("failed to read schema file: %w", err)
 	}
-	return NewValidatorFromBytes(data)
+	return NewSchemaFromBytes(data)
 }
 
 func NewValidatorFromURL(schemaURL string) (Validator, error) {
@@ -212,7 +230,7 @@ var (
 )
 
 type UnknownTypes struct {
-	Types []string
+	Types []string `json:"types,omitempty"`
 }
 
 func (ut UnknownTypes) Error() string {
@@ -220,7 +238,7 @@ func (ut UnknownTypes) Error() string {
 }
 
 type ContentError struct {
-	Err error
+	Err error `json:"err,omitempty"`
 }
 
 func (ce ContentError) Error() string {
@@ -228,9 +246,9 @@ func (ce ContentError) Error() string {
 }
 
 type TagError struct {
-	Tag  int
+	Tag  int   `json:"tag,omitempty"`
-	Item int
+	Item int   `json:"item,omitempty"`
-	Err  error
+	Err  error `json:"err,omitempty"`
 }
 
 func (te TagError) Error() string {
@@ -238,7 +256,7 @@ func (te TagError) Error() string {
 }
 
 type RequiredTagError struct {
-	Missing []string
+	Missing []string `json:"missing,omitempty"`
 }
 
 func (rte RequiredTagError) Error() string {
@@ -385,7 +403,9 @@ func (v *Validator) validateNext(tag nostr.Tag, index int, this *ContentSpec) (f
 		return index, ErrDanglingSpace
 	}
 
-	if validator, ok := v.TypeValidators[this.Type]; ok {
+	if tag[index] == "" && !this.Required {
+		// empty string ok for non-required items
+	} else if validator, ok := v.TypeValidators[this.Type]; ok {
 		if err := validator(tag[index], this); err != nil {
 			return index, fmt.Errorf("invalid %s value '%s' at tag '%s', index %d: %w",
 				this.Type, tag[index], tag[0], index, err)

sdk/list.go

@@ -94,7 +94,7 @@ func fetchGenericList[V comparable, I TagItemWithValue[V]](
 		// we'll only save this if we got something which means we found at least one event
 		lastFetchKey := makeLastFetchKey(actualKind, pubkey)
 		sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
-		sys.Store.SaveEvent(*v.Event)
+		sys.Store.ReplaceEvent(*v.Event)
 	}
 
 	// save cache even if we didn't get anything

sdk/metadata.go

@@ -142,7 +142,7 @@ func (sys *System) FetchProfileMetadata(ctx context.Context, pubkey nostr.PubKey
 	if newM := sys.tryFetchMetadataFromNetwork(ctx, pubkey); newM != nil {
 		pm = *newM
 
-		sys.Store.SaveEvent(*pm.Event)
+		sys.Store.ReplaceEvent(*pm.Event)
 
 		// we'll only save this if we got something which means we found at least one event
 		lastFetchKey := makeLastFetchKey(0, pubkey)

sdk/utils.go

@@ -22,9 +22,7 @@ func IsVirtualRelay(url string) bool {
 		return true
 	}
 
-	if !testing.Testing() &&
+	if !testing.Testing() && (strings.HasPrefix(url, "ws://localhost") || strings.HasPrefix(url, "ws://127.0.0.1")) {
-		strings.HasPrefix(url, "ws://localhost") ||
-		strings.HasPrefix(url, "ws://127.0.0.1") {
 		return true
 	}
 

sdk/wot.go

@@ -92,7 +92,7 @@ func (sys *System) loadWoT(ctx context.Context, pubkey nostr.PubKey) chan nostr.
 
 	res := make(chan nostr.PubKey)
 
-	go func() {
+	g.Go(func() error {
 		for _, f := range sys.FetchFollowList(ctx, pubkey).Items {
 			g.Go(func() error {
 				res <- f.Pubkey
@@ -107,7 +107,9 @@ func (sys *System) loadWoT(ctx context.Context, pubkey nostr.PubKey) chan nostr.
 				return nil
 			})
 		}
-	}()
+
+		return nil
+	})
 
 	go func() {
 		g.Wait()

sdk/wot_test.go

@@ -14,8 +14,7 @@ func TestLoadWoT(t *testing.T) {
 	ctx := t.Context()
 
 	// test with fiatjaf's pubkey
-	wotch, err := sys.loadWoT(ctx, nostr.MustPubKeyFromHex("3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d"))
+	wotch := sys.loadWoT(ctx, nostr.MustPubKeyFromHex("3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d"))
-	require.NoError(t, err)
 
 	wot := make([]nostr.PubKey, 0, 100000)
 	wotch2 := make(chan nostr.PubKey)
@@ -60,8 +59,7 @@ func TestLoadWoTManyPeople(t *testing.T) {
 
 	// these are the same pubkey
 	go func() {
-		rabble, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
+		rabble := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
-		require.NoError(t, err)
 		diffs[0] = nostr.Now()
 		rabble1 = rabble
 		wg.Done()
@@ -69,8 +67,7 @@ func TestLoadWoTManyPeople(t *testing.T) {
 
 	time.Sleep(time.Millisecond * 20)
 	go func() {
-		rabble, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
+		rabble := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
-		require.NoError(t, err)
 		diffs[1] = nostr.Now()
 		rabble2 = rabble
 		wg.Done()
@@ -78,8 +75,7 @@ func TestLoadWoTManyPeople(t *testing.T) {
 
 	time.Sleep(time.Millisecond * 20)
 	go func() {
-		rabble, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
+		rabble := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa"))
-		require.NoError(t, err)
 		diffs[2] = nostr.Now()
 		rabble3 = rabble
 		wg.Done()
@@ -88,8 +84,7 @@ func TestLoadWoTManyPeople(t *testing.T) {
 	// these should map to the same pos
 	time.Sleep(time.Millisecond * 20)
 	go func() {
-		alex, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7"))
+		alex := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7"))
-		require.NoError(t, err)
 		diffs[3] = nostr.Now()
 		alex1 = alex
 		wg.Done()
@@ -97,8 +92,7 @@ func TestLoadWoTManyPeople(t *testing.T) {
 
 	time.Sleep(time.Millisecond * 20)
 	go func() {
-		alex, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7"))
+		alex := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("9ce71f1506ccf4b99f234af49bd6202be883a80f95a155c6e9a1c36fd7e780c7"))
-		require.NoError(t, err)
 		diffs[4] = nostr.Now()
 		alex2 = alex
 		wg.Done()
@@ -106,16 +100,14 @@ func TestLoadWoTManyPeople(t *testing.T) {
 
 	// these are independent
 	go func() {
-		hodlbod, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322"))
+		hodlbod := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322"))
-		require.NoError(t, err)
 		require.True(t, hodlbod.Contains(nostr.MustPubKeyFromHex("ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49")))
 		require.True(t, hodlbod.Contains(nostr.MustPubKeyFromHex("76c71aae3a491f1d9eec47cba17e229cda4113a0bbb6e6ae1776d7643e29cafa")))
 		require.True(t, hodlbod.Contains(nostr.MustPubKeyFromHex("3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d")))
 		wg.Done()
 	}()
 	go func() {
-		mikedilger, err := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49"))
+		mikedilger := sys.LoadWoTFilter(ctx, nostr.MustPubKeyFromHex("ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49"))
-		require.NoError(t, err)
 		require.True(t, mikedilger.Contains(nostr.MustPubKeyFromHex("97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322")))
 		require.True(t, mikedilger.Contains(nostr.MustPubKeyFromHex("3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d")))
 		wg.Done()