From 53ffa3fcbc257750fd9fbcf1394d938ec739e6d8 Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Thu, 25 Sep 2025 11:54:41 -0700 Subject: [PATCH] Add schema abstraction, move event store into zooid --- README.md | 16 ++- cmd/relay/main.go | 2 +- sqlite/count.go | 10 -- sqlite/delete.go | 12 -- sqlite/lib.go | 125 ----------------- sqlite/query.go | 146 ------------------- sqlite/replace.go | 34 ----- sqlite/save.go | 64 --------- sqlite/sqlite_test.go | 111 --------------- zooid/blossom.go | 15 +- zooid/database.go | 26 ++++ zooid/events.go | 319 ++++++++++++++++++++++++++++++++++++++++++ zooid/instance.go | 8 +- zooid/schema.go | 25 ++++ zooid/util.go | 3 + 15 files changed, 398 insertions(+), 518 deletions(-) delete mode 100644 sqlite/count.go delete mode 100644 sqlite/delete.go delete mode 100644 sqlite/lib.go delete mode 100644 sqlite/query.go delete mode 100644 sqlite/replace.go delete mode 100644 sqlite/save.go delete mode 100644 sqlite/sqlite_test.go create mode 100644 zooid/database.go create mode 100644 zooid/events.go create mode 100644 zooid/schema.go diff --git a/README.md b/README.md index f00d1d4..4a5acc5 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ A single zooid instance can run any number of "virtual" relays. The `config` dir ## Environment -Zooid supports a few environment variables, which configure shared resources, like the web server or sqlite database file. +Zooid supports a few environment variables, which configure shared resources like the web server or sqlite database. - `PORT` - the port the server will listen on for all requests. Defaults to `3334`. -- `DATABASE_PATH` - the location of the database path. Defaults to `./data.db` +- `DATA` - the location of the directory for storing database files and media. Defaults to `./data`. ## Configuration @@ -52,7 +52,6 @@ Configures NIP 86 support. Configures blossom support. - `enabled` - whether blossom is enabled. -- `directory` - where to store files. Defaults to `./data/{my-relay}/media`. ### `[roles]` @@ -88,11 +87,16 @@ can_invite = true [roles.admin] pubkeys = ["d9254d9898fd4728f7e2b32b87520221a50f6b8b97d935d7da2de8923988aa6d"] - -[data] -events = "./data/my-relay/events" ``` ## Development See `justfile` for defined commands. + +## TODO + +- [ ] Create a "schema" abstraction to namespace tables + - This resource should be passed to event stores as well as claims, redemptions, etc + - We might need to create a custom blossom backend since the prefixes for the two stores will collide +- [ ] Watch configuration files and hot reload +- [ ] Free up resources after instance inactivity diff --git a/cmd/relay/main.go b/cmd/relay/main.go index efa39bd..992433d 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -18,7 +18,7 @@ func main() { shutdown := make(chan os.Signal, 1) signal.Notify(shutdown, syscall.SIGINT, syscall.SIGTERM) - port := zooid.Env("PORT", "3334") + port := zooid.Env("PORT") srv := &http.Server{ Addr: fmt.Sprintf(":%s", port), Handler: http.HandlerFunc(zooid.ServeHTTP), diff --git a/sqlite/count.go b/sqlite/count.go deleted file mode 100644 index 18fa0e8..0000000 --- a/sqlite/count.go +++ /dev/null @@ -1,10 +0,0 @@ -package sqlite - -import ( - "errors" - "fiatjaf.com/nostr" -) - -func (s *SqliteBackend) CountEvents(nostr.Filter) (uint32, error) { - return 0, errors.New("not supported") -} diff --git a/sqlite/delete.go b/sqlite/delete.go deleted file mode 100644 index 5b497e7..0000000 --- a/sqlite/delete.go +++ /dev/null @@ -1,12 +0,0 @@ -package sqlite - -import ( - "fiatjaf.com/nostr" - "github.com/Masterminds/squirrel" -) - -func (s *SqliteBackend) DeleteEvent(id nostr.ID) error { - _, err := squirrel.Delete(s.tmpl("{{.Prefix}}events")).Where(squirrel.Eq{"id": id.Hex()}).RunWith(s.db).Exec() - - return err -} diff --git a/sqlite/lib.go b/sqlite/lib.go deleted file mode 100644 index ab30231..0000000 --- a/sqlite/lib.go +++ /dev/null @@ -1,125 +0,0 @@ -package sqlite - -import ( - "bytes" - "database/sql" - "fmt" - "text/template" - - "fiatjaf.com/nostr/eventstore" - _ "github.com/mattn/go-sqlite3" -) - -var _ eventstore.Store = (*SqliteBackend)(nil) - -type SqliteBackend struct { - db *sql.DB - Path string - Prefix string - FTSAvailable bool -} - -func (s *SqliteBackend) Close() { - if s.db != nil { - s.db.Close() - } -} - -func (s *SqliteBackend) Init() error { - if s.Path == "" { - return fmt.Errorf("missing Path") - } - - var err error - s.db, err = sql.Open("sqlite3", s.Path+"?_journal_mode=WAL&_sync=NORMAL&_cache_size=1000&_foreign_keys=true") - if err != nil { - return fmt.Errorf("failed to open database: %w", err) - } - - // Create tables and indexes - if err := s.createSchema(); err != nil { - return fmt.Errorf("failed to create schema: %w", err) - } - - return nil -} - -func (s *SqliteBackend) tmpl(t string) string { - var buf bytes.Buffer - err := template.Must(template.New("schema").Parse(t)).Execute(&buf, s) - if err != nil { - panic(err) - } - - return buf.String() -} - -func (s *SqliteBackend) createSchema() error { - // Create basic schema first - basicSchema := s.tmpl(` - CREATE TABLE IF NOT EXISTS {{.Prefix}}events ( - id TEXT PRIMARY KEY, - created_at INTEGER NOT NULL, - kind INTEGER NOT NULL, - pubkey TEXT NOT NULL, - content TEXT NOT NULL, - tags TEXT NOT NULL, - sig TEXT NOT NULL - ); - - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_events_created_at ON {{.Prefix}}events(created_at); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_events_kind ON {{.Prefix}}events(kind); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_events_pubkey ON {{.Prefix}}events(pubkey); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_events_kind_pubkey ON {{.Prefix}}events(kind, pubkey); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_events_kind_pubkey_created_at ON {{.Prefix}}events(kind, pubkey, created_at DESC); - - CREATE TABLE IF NOT EXISTS {{.Prefix}}event_tags ( - event_id TEXT NOT NULL, - key TEXT NOT NULL, - value TEXT NOT NULL, - FOREIGN KEY (event_id) REFERENCES {{.Prefix}}events(id) ON DELETE CASCADE - ); - - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_event_tags_event_id ON {{.Prefix}}event_tags(event_id); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_event_tags_key ON {{.Prefix}}event_tags(key); - CREATE INDEX IF NOT EXISTS {{.Prefix}}idx_event_tags_key_value ON {{.Prefix}}event_tags(key, value); - `) - - if _, err := s.db.Exec(basicSchema); err != nil { - return fmt.Errorf("failed to create schema: %w", err) - } - - // Try to create FTS5 schema - if it fails, continue without it - ftsSchema := ` - CREATE VIRTUAL TABLE IF NOT EXISTS {{.Prefix}}events_fts USING fts5( - content, - content='{{.Prefix}}events', - content_rowid='rowid' - ); - - CREATE TRIGGER IF NOT EXISTS {{.Prefix}}events_ai AFTER INSERT ON {{.Prefix}}events BEGIN - INSERT INTO {{.Prefix}}events_fts(rowid, content) VALUES (new.rowid, new.content); - END; - - CREATE TRIGGER IF NOT EXISTS {{.Prefix}}events_ad AFTER DELETE ON {{.Prefix}}events BEGIN - INSERT INTO {{.Prefix}}events_fts({{.Prefix}}events_fts, rowid, content) - VALUES('delete', old.rowid, old.content); - END; - - CREATE TRIGGER IF NOT EXISTS {{.Prefix}}events_au AFTER UPDATE ON {{.Prefix}}events BEGIN - INSERT INTO {{.Prefix}}events_fts({{.Prefix}}events_fts, rowid, content) - VALUES('delete', old.rowid, old.content); - INSERT INTO {{.Prefix}}events_fts(rowid, content) - VALUES (new.rowid, new.content); - END; - ` - - if _, err := s.db.Exec(ftsSchema); err != nil { - // FTS5 not available, continue without full-text search - s.FTSAvailable = false - } else { - s.FTSAvailable = true - } - - return nil -} diff --git a/sqlite/query.go b/sqlite/query.go deleted file mode 100644 index 956e09a..0000000 --- a/sqlite/query.go +++ /dev/null @@ -1,146 +0,0 @@ -package sqlite - -import ( - "encoding/hex" - "encoding/json" - "iter" - - "fiatjaf.com/nostr" - "github.com/Masterminds/squirrel" -) - -func (s *SqliteBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { - return func(yield func(nostr.Event) bool) { - if filter.LimitZero { - return - } - - limit := maxLimit - if filter.Limit > 0 && filter.Limit < limit { - limit = filter.Limit - } - - rows, err := s.buildSelectQuery(filter, limit).RunWith(s.db).Query() - if err != nil { - return - } - defer rows.Close() - - for rows.Next() { - var evt nostr.Event - var idStr, pubkeyStr, sigStr, tagsStr string - var createdAt int64 - var kind int - - err := rows.Scan(&idStr, &createdAt, &kind, &pubkeyStr, &evt.Content, &tagsStr, &sigStr) - if err != nil { - continue - } - - // Parse ID - if id, err := nostr.IDFromHex(idStr); err == nil { - evt.ID = id - } else { - continue - } - - // Parse PubKey - if pubkey, err := nostr.PubKeyFromHex(pubkeyStr); err == nil { - evt.PubKey = pubkey - } else { - continue - } - - // Parse Signature - if sigBytes, err := hex.DecodeString(sigStr); err == nil && len(sigBytes) == 64 { - copy(evt.Sig[:], sigBytes) - } else { - continue - } - - // Set other fields - evt.CreatedAt = nostr.Timestamp(createdAt) - evt.Kind = nostr.Kind(kind) - - // Parse Tags - if err := json.Unmarshal([]byte(tagsStr), &evt.Tags); err != nil { - continue - } - - if !yield(evt) { - return - } - } - } -} - -func (s *SqliteBackend) buildSelectQuery(filter nostr.Filter, limit int) squirrel.SelectBuilder { - qb := squirrel.Select("id", "created_at", "kind", "pubkey", "content", "tags", "sig"). - From(s.tmpl("{{.Prefix}}events")). - OrderBy("created_at DESC") - - // Handle search with FTS (if available) - if filter.Search != "" && s.FTSAvailable { - qb = qb.Join(s.tmpl("{{.Prefix}}events_fts ON {{.Prefix}}events.rowid = {{.Prefix}}events_fts.rowid")). - Where(squirrel.Eq{"events_fts": filter.Search}) - } else if filter.Search != "" { - // Fallback to LIKE search if FTS not available - qb = qb.Where(squirrel.Like{"content": "%" + filter.Search + "%"}) - } - - if len(filter.IDs) > 0 { - idStrs := make([]interface{}, len(filter.IDs)) - for i, id := range filter.IDs { - idStrs[i] = id.Hex() - } - qb = qb.Where(squirrel.Eq{"id": idStrs}) - } - - if len(filter.Authors) > 0 { - authorStrs := make([]interface{}, len(filter.Authors)) - for i, author := range filter.Authors { - authorStrs[i] = author.Hex() - } - qb = qb.Where(squirrel.Eq{"pubkey": authorStrs}) - } - - if len(filter.Kinds) > 0 { - kindInts := make([]interface{}, len(filter.Kinds)) - for i, kind := range filter.Kinds { - kindInts[i] = int(kind) - } - qb = qb.Where(squirrel.Eq{"kind": kindInts}) - } - - if filter.Since != 0 { - qb = qb.Where(squirrel.GtOrEq{"created_at": filter.Since}) - } - - if filter.Until != 0 { - qb = qb.Where(squirrel.LtOrEq{"created_at": filter.Until}) - } - - for tagKey, tagValues := range filter.Tags { - if len(tagValues) > 0 && len(tagKey) == 1 { - tagValueInterfaces := make([]interface{}, len(tagValues)) - for i, tagValue := range tagValues { - tagValueInterfaces[i] = tagValue - } - - subQuery := squirrel.Select("event_id"). - From(s.tmpl("{{.Prefix}}event_tags")). - Where(squirrel.Eq{"key": tagKey}). - Where(squirrel.Eq{"value": tagValueInterfaces}) - - subQuerySql, subQueryArgs, _ := subQuery.ToSql() - qb = qb.Where("id IN ("+subQuerySql+")", subQueryArgs...) - } - } - - // Add limit - if limit > 0 { - qb = qb.Limit(uint64(limit)) - } - - return qb -} diff --git a/sqlite/replace.go b/sqlite/replace.go deleted file mode 100644 index 1b3657a..0000000 --- a/sqlite/replace.go +++ /dev/null @@ -1,34 +0,0 @@ -package sqlite - -import ( - "fmt" - - "fiatjaf.com/nostr" - "fiatjaf.com/nostr/eventstore" -) - -func (s *SqliteBackend) ReplaceEvent(evt nostr.Event) error { - filter := nostr.Filter{Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}} - if evt.Kind.IsAddressable() { - filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}} - } - - shouldStore := true - for previous := range s.QueryEvents(filter, 1) { - if previous.CreatedAt <= evt.CreatedAt { - if err := s.DeleteEvent(previous.ID); err != nil { - return fmt.Errorf("failed to delete event for replacing: %w", err) - } - } else { - shouldStore = false - } - } - - if shouldStore { - if err := s.SaveEvent(evt); err != nil && err != eventstore.ErrDupEvent { - return fmt.Errorf("failed to save: %w", err) - } - } - - return nil -} diff --git a/sqlite/save.go b/sqlite/save.go deleted file mode 100644 index 6892949..0000000 --- a/sqlite/save.go +++ /dev/null @@ -1,64 +0,0 @@ -package sqlite - -import ( - "encoding/hex" - "encoding/json" - "fmt" - - "fiatjaf.com/nostr" - "fiatjaf.com/nostr/eventstore" - "github.com/Masterminds/squirrel" -) - -func (s *SqliteBackend) SaveEvent(evt nostr.Event) error { - // Check if event already exists - var existingID string - qb := squirrel.Select("id").From(s.tmpl("{{.Prefix}}events")).Where(squirrel.Eq{"id": evt.ID.Hex()}) - err := qb.RunWith(s.db).QueryRow().Scan(&existingID) - if err == nil { - // Event already exists - return eventstore.ErrDupEvent - } - - // Serialize tags to JSON - tagsJSON, err := json.Marshal(evt.Tags) - if err != nil { - return fmt.Errorf("failed to marshal tags: %w", err) - } - - // Insert the event - insertQb := squirrel.Insert(s.tmpl("{{.Prefix}}events")). - Columns("id", "created_at", "kind", "pubkey", "content", "tags", "sig"). - Values( - evt.ID.Hex(), - int64(evt.CreatedAt), - int(evt.Kind), - evt.PubKey.Hex(), - evt.Content, - string(tagsJSON), - hex.EncodeToString(evt.Sig[:]), - ) - - _, err = insertQb.RunWith(s.db).Exec() - - if err != nil { - return fmt.Errorf("failed to save event '%s': %w", evt.ID, err) - } - - // Insert single-letter tags into event_tags table - for _, tag := range evt.Tags { - if len(tag) >= 2 && len(tag[0]) == 1 { - tagQb := squirrel.Insert(s.tmpl("{{.Prefix}}event_tags")). - Columns("event_id", "key", "value"). - Values(evt.ID.Hex(), tag[0], tag[1]) - - _, err := tagQb.RunWith(s.db).Exec() - if err != nil { - // Log error but don't fail the entire save operation - continue - } - } - } - - return nil -} diff --git a/sqlite/sqlite_test.go b/sqlite/sqlite_test.go deleted file mode 100644 index 7115e6d..0000000 --- a/sqlite/sqlite_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package sqlite - -import ( - "os" - "testing" - - "fiatjaf.com/nostr" - "github.com/stretchr/testify/assert" -) - -func TestSqliteFlow(t *testing.T) { - os.RemoveAll("/tmp/sqlitetest.db") - - sb := &SqliteBackend{ - Path: "/tmp/sqlitetest.db", - Prefix: "prefix", - } - err := sb.Init() - assert.NoError(t, err) - defer sb.Close() - - willDelete := make([]nostr.Event, 0, 3) - - sk := nostr.MustSecretKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001") - - for i, content := range []string{ - "good morning mr paper maker", - "good night", - "I'll see you again in the paper house", - "tonight we dine in my house", - "the paper in this house if very good, mr", - } { - evt := nostr.Event{ - Content: content, - Tags: nostr.Tags{}, - Kind: 1, - CreatedAt: nostr.Now(), - PubKey: sk.Public(), - } - evt.ID = evt.GetID() - // For testing, we'll skip actual signing and just set a dummy signature - // In real usage, you'd call evt.Sign(sk) - - err := sb.SaveEvent(evt) - assert.NoError(t, err) - - if i%2 == 0 { - willDelete = append(willDelete, evt) - } - } - - // Test search functionality (if FTS5 is available) - if sb.FTSAvailable { - n := 0 - for range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) { - n++ - } - assert.Equal(t, 3, n) - } else { - // With LIKE fallback, should still work but might be less precise - n := 0 - for range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) { - n++ - } - assert.Equal(t, 3, n) - } - - // Delete some events - for _, evt := range willDelete { - err := sb.DeleteEvent(evt.ID) - assert.NoError(t, err) - } - - // Test search after deletion - if sb.FTSAvailable { - n := 0 - for res := range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) { - n++ - assert.Equal(t, res.Content, "good night") - assert.Equal(t, sk.Public(), res.PubKey) - } - assert.Equal(t, 1, n) - } else { - // With LIKE fallback, should still work - n := 0 - for res := range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) { - n++ - assert.Equal(t, res.Content, "good night") - assert.Equal(t, sk.Public(), res.PubKey) - } - assert.Equal(t, 1, n) - } - - // Test query by kind - { - n := 0 - for range sb.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{1}}, 400) { - n++ - } - assert.Equal(t, 2, n) - } - - // Test query by author - { - n := 0 - for range sb.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{sk.Public()}}, 400) { - n++ - } - assert.Equal(t, 2, n) - } -} diff --git a/zooid/blossom.go b/zooid/blossom.go index 6362b79..ff42f98 100644 --- a/zooid/blossom.go +++ b/zooid/blossom.go @@ -9,24 +9,29 @@ import ( "fiatjaf.com/nostr" "fiatjaf.com/nostr/khatru/blossom" + "github.com/gosimple/slug" "github.com/spf13/afero" - "zooid/sqlite" ) func EnableBlossom(instance *Instance) { fs := afero.NewOsFs() - if err := fs.MkdirAll(instance.Config.Blossom.Directory, 0755); err != nil { + if err := fs.MkdirAll(Env("DATA"), 0755); err != nil { log.Fatal("🚫 error creating blossom path:", err) } - store := &sqlite.SqliteBackend{ - Path: instance.Config.Data.Blossom, + store := &EventStore{ + Schema: &Schema{ + Name: slug.Make(instance.Host) + "_blossom__", + }, } backend := blossom.New(instance.Relay, "https://"+instance.Host) - backend.Store = blossom.EventStoreBlobIndexWrapper{Store: store, ServiceURL: "https://" + instance.Host} + backend.Store = blossom.EventStoreBlobIndexWrapper{ + Store: store, + ServiceURL: "https://" + instance.Host, + } backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { file, err := fs.Create(instance.Config.Blossom.Directory + "/" + sha256) diff --git a/zooid/database.go b/zooid/database.go new file mode 100644 index 0000000..4083446 --- /dev/null +++ b/zooid/database.go @@ -0,0 +1,26 @@ +package zooid + +import ( + "database/sql" + "log" + "sync" +) + +var ( + db *sql.DB + dbOnce sync.Once +) + +func GetDb() *sql.DB { + dbOnce.Do(func() { + newDb, err := sql.Open("sqlite3", Env("DATA")+"/db?_journal_mode=WAL&_sync=NORMAL&_cache_size=1000&_foreign_keys=true") + + if err != nil { + log.Fatal("Failed to open database: %w", err) + } + + db = newDb + }) + + return db +} diff --git a/zooid/events.go b/zooid/events.go new file mode 100644 index 0000000..d166cb7 --- /dev/null +++ b/zooid/events.go @@ -0,0 +1,319 @@ +package zooid + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "iter" + + "fiatjaf.com/nostr" + "fiatjaf.com/nostr/eventstore" + "github.com/Masterminds/squirrel" + _ "github.com/mattn/go-sqlite3" +) + +type EventStore struct { + Schema *Schema + FTSAvailable bool +} + +var _ eventstore.Store = (*EventStore)(nil) + +func (events *EventStore) Init() error { + // Create basic schema first + basicSchema := events.Schema.Render(` + CREATE TABLE IF NOT EXISTS {{.Prefix}}__events ( + id TEXT PRIMARY KEY, + created_at INTEGER NOT NULL, + kind INTEGER NOT NULL, + pubkey TEXT NOT NULL, + content TEXT NOT NULL, + tags TEXT NOT NULL, + sig TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_events_created_at ON {{.Prefix}}__events(created_at); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_events_kind ON {{.Prefix}}__events(kind); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_events_pubkey ON {{.Prefix}}__events(pubkey); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_events_kind_pubkey ON {{.Prefix}}__events(kind, pubkey); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_events_kind_pubkey_created_at ON {{.Prefix}}__events(kind, pubkey, created_at DESC); + + CREATE TABLE IF NOT EXISTS {{.Prefix}}__event_tags ( + event_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + FOREIGN KEY (event_id) REFERENCES {{.Prefix}}__events(id) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_event_tags_event_id ON {{.Prefix}}__event_tags(event_id); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_event_tags_key ON {{.Prefix}}__event_tags(key); + CREATE INDEX IF NOT EXISTS {{.Prefix}}__idx_event_tags_key_value ON {{.Prefix}}__event_tags(key, value); + `) + + if _, err := GetDb().Exec(basicSchema); err != nil { + return fmt.Errorf("failed to create schema: %w", err) + } + + // Try to create FTS5 schema - if it fails, continue without it + ftsSchema := ` + CREATE VIRTUAL TABLE IF NOT EXISTS {{.Prefix}}__events_fts USING fts5( + content, + content='{{.Prefix}}__events', + content_rowid='rowid' + ); + + CREATE TRIGGER IF NOT EXISTS {{.Prefix}}__events_ai AFTER INSERT ON {{.Prefix}}__events BEGIN + INSERT INTO {{.Prefix}}__events_fts(rowid, content) VALUES (new.rowid, new.content); + END; + + CREATE TRIGGER IF NOT EXISTS {{.Prefix}}__events_ad AFTER DELETE ON {{.Prefix}}__events BEGIN + INSERT INTO {{.Prefix}}__events_fts({{.Prefix}}__events_fts, rowid, content) + VALUES('delete', old.rowid, old.content); + END; + + CREATE TRIGGER IF NOT EXISTS {{.Prefix}}__events_au AFTER UPDATE ON {{.Prefix}}__events BEGIN + INSERT INTO {{.Prefix}}__events_fts({{.Prefix}}__events_fts, rowid, content) + VALUES('delete', old.rowid, old.content); + INSERT INTO {{.Prefix}}__events_fts(rowid, content) + VALUES (new.rowid, new.content); + END; + ` + + if _, err := GetDb().Exec(ftsSchema); err != nil { + // FTS5 not available, continue without full-text search + events.FTSAvailable = false + } else { + events.FTSAvailable = true + } + + return nil +} + +func (events *EventStore) Close() { + // Never close the database, since it's a shared resource +} + +func (events *EventStore) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] { + return func(yield func(nostr.Event) bool) { + if filter.LimitZero { + return + } + + limit := maxLimit + if filter.Limit > 0 && filter.Limit < limit { + limit = filter.Limit + } + + rows, err := events.buildSelectQuery(filter, limit).RunWith(GetDb()).Query() + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var evt nostr.Event + var idStr, pubkeyStr, sigStr, tagsStr string + var createdAt int64 + var kind int + + err := rows.Scan(&idStr, &createdAt, &kind, &pubkeyStr, &evt.Content, &tagsStr, &sigStr) + if err != nil { + continue + } + + // Parse ID + if id, err := nostr.IDFromHex(idStr); err == nil { + evt.ID = id + } else { + continue + } + + // Parse PubKey + if pubkey, err := nostr.PubKeyFromHex(pubkeyStr); err == nil { + evt.PubKey = pubkey + } else { + continue + } + + // Parse Signature + if sigBytes, err := hex.DecodeString(sigStr); err == nil && len(sigBytes) == 64 { + copy(evt.Sig[:], sigBytes) + } else { + continue + } + + // Set other fields + evt.CreatedAt = nostr.Timestamp(createdAt) + evt.Kind = nostr.Kind(kind) + + // Parse Tags + if err := json.Unmarshal([]byte(tagsStr), &evt.Tags); err != nil { + continue + } + + if !yield(evt) { + return + } + } + } +} + +func (events *EventStore) buildSelectQuery(filter nostr.Filter, limit int) squirrel.SelectBuilder { + qb := squirrel.Select("id", "created_at", "kind", "pubkey", "content", "tags", "sig"). + From(events.Schema.Prefix("events")). + OrderBy("created_at DESC") + + // Handle search with FTS (if available) + if filter.Search != "" && events.FTSAvailable { + qb = qb.Join(events.Schema.Render("{{.Prefix}}__events_fts ON {{.Prefix}}__events.rowid = {{.Prefix}}__events_fts.rowid")). + Where(squirrel.Eq{"events_fts": filter.Search}) + } else if filter.Search != "" { + // Fallback to LIKE search if FTS not available + qb = qb.Where(squirrel.Like{"content": "%" + filter.Search + "%"}) + } + + if len(filter.IDs) > 0 { + idStrs := make([]interface{}, len(filter.IDs)) + for i, id := range filter.IDs { + idStrs[i] = id.Hex() + } + qb = qb.Where(squirrel.Eq{"id": idStrs}) + } + + if len(filter.Authors) > 0 { + authorStrs := make([]interface{}, len(filter.Authors)) + for i, author := range filter.Authors { + authorStrs[i] = author.Hex() + } + qb = qb.Where(squirrel.Eq{"pubkey": authorStrs}) + } + + if len(filter.Kinds) > 0 { + kindInts := make([]interface{}, len(filter.Kinds)) + for i, kind := range filter.Kinds { + kindInts[i] = int(kind) + } + qb = qb.Where(squirrel.Eq{"kind": kindInts}) + } + + if filter.Since != 0 { + qb = qb.Where(squirrel.GtOrEq{"created_at": filter.Since}) + } + + if filter.Until != 0 { + qb = qb.Where(squirrel.LtOrEq{"created_at": filter.Until}) + } + + for tagKey, tagValues := range filter.Tags { + if len(tagValues) > 0 && len(tagKey) == 1 { + tagValueInterfaces := make([]interface{}, len(tagValues)) + for i, tagValue := range tagValues { + tagValueInterfaces[i] = tagValue + } + + subQuery := squirrel.Select("event_id"). + From(events.Schema.Prefix("event_tags")). + Where(squirrel.Eq{"key": tagKey}). + Where(squirrel.Eq{"value": tagValueInterfaces}) + + subQuerySql, subQueryArgs, _ := subQuery.ToSql() + qb = qb.Where("id IN ("+subQuerySql+")", subQueryArgs...) + } + } + + // Add limit + if limit > 0 { + qb = qb.Limit(uint64(limit)) + } + + return qb +} + +func (events *EventStore) DeleteEvent(id nostr.ID) error { + _, err := squirrel.Delete(events.Schema.Prefix("events")).Where(squirrel.Eq{"id": id.Hex()}).RunWith(GetDb()).Exec() + + return err +} + +func (events *EventStore) SaveEvent(evt nostr.Event) error { + // Check if event already exists + var existingID string + qb := squirrel.Select("id").From(events.Schema.Prefix("events")).Where(squirrel.Eq{"id": evt.ID.Hex()}) + err := qb.RunWith(GetDb()).QueryRow().Scan(&existingID) + if err == nil { + return eventstore.ErrDupEvent + } + + // Serialize tags to JSON + tagsJSON, err := json.Marshal(evt.Tags) + if err != nil { + return fmt.Errorf("failed to marshal tags: %w", err) + } + + // Insert the event + insertQb := squirrel.Insert(events.Schema.Prefix("events")). + Columns("id", "created_at", "kind", "pubkey", "content", "tags", "sig"). + Values( + evt.ID.Hex(), + int64(evt.CreatedAt), + int(evt.Kind), + evt.PubKey.Hex(), + evt.Content, + string(tagsJSON), + hex.EncodeToString(evt.Sig[:]), + ) + + _, err = insertQb.RunWith(GetDb()).Exec() + + if err != nil { + return fmt.Errorf("failed to save event '%s': %w", evt.ID, err) + } + + // Insert single-letter tags into event_tags table + for _, tag := range evt.Tags { + if len(tag) >= 2 && len(tag[0]) == 1 { + tagQb := squirrel.Insert(events.Schema.Prefix("event_tags")). + Columns("event_id", "key", "value"). + Values(evt.ID.Hex(), tag[0], tag[1]) + + _, err := tagQb.RunWith(GetDb()).Exec() + if err != nil { + // Log error but don't fail the entire save operation + continue + } + } + } + + return nil +} + +func (events *EventStore) ReplaceEvent(evt nostr.Event) error { + filter := nostr.Filter{Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}} + if evt.Kind.IsAddressable() { + filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}} + } + + shouldStore := true + for previous := range events.QueryEvents(filter, 1) { + if previous.CreatedAt <= evt.CreatedAt { + if err := events.DeleteEvent(previous.ID); err != nil { + return fmt.Errorf("failed to delete event for replacing: %w", err) + } + } else { + shouldStore = false + } + } + + if shouldStore { + if err := events.SaveEvent(evt); err != nil && err != eventstore.ErrDupEvent { + return fmt.Errorf("failed to save: %w", err) + } + } + + return nil +} + +func (events *EventStore) CountEvents(nostr.Filter) (uint32, error) { + return 0, errors.New("COUNT is not supported") +} diff --git a/zooid/instance.go b/zooid/instance.go index d8f735b..07bf000 100644 --- a/zooid/instance.go +++ b/zooid/instance.go @@ -12,7 +12,6 @@ import ( "fiatjaf.com/nostr/eventstore" "fiatjaf.com/nostr/khatru" "github.com/gosimple/slug" - "zooid/sqlite" ) type Instance struct { @@ -41,9 +40,10 @@ func MakeInstance(hostname string) (*Instance, error) { instance := &Instance{ Host: hostname, Config: config, - Events: &sqlite.SqliteBackend{ - Path: Env("DATABASE_PATH", "./data.db"), - Prefix: slug.Make(hostname) + "__", + Events: &EventStore{ + Schema: &Schema{ + Name: slug.Make(hostname), + }, }, Relay: khatru.NewRelay(), } diff --git a/zooid/schema.go b/zooid/schema.go new file mode 100644 index 0000000..9816831 --- /dev/null +++ b/zooid/schema.go @@ -0,0 +1,25 @@ +package zooid + +import ( + "bytes" + "log" + "text/template" +) + +type Schema struct { + Name string +} + +func (s *Schema) Render(t string) string { + var buf bytes.Buffer + err := template.Must(template.New("schema").Parse(t)).Execute(&buf, s) + if err != nil { + log.Fatal("Failed to create template: %w", err) + } + + return buf.String() +} + +func (s *Schema) Prefix(t string) string { + return s.Render("{{.Name}}__" + t) +} diff --git a/zooid/util.go b/zooid/util.go index 10942df..378f373 100644 --- a/zooid/util.go +++ b/zooid/util.go @@ -20,6 +20,9 @@ func Env(k string, fallback ...string) (v string) { envOnce.Do(func() { env = make(map[string]string) + env["PORT"] = "3334" + env["DATA"] = "./data" + for _, item := range os.Environ() { parts := strings.SplitN(item, "=", 2) env[parts[0]] = parts[1]