coracle/zooid
1
3
Fork 3
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Add sqlite backend

Browse Source
This commit is contained in:
Jon Staab
2025-09-24 10:32:10 -07:00
parent 9eedeceb6a
commit 91f23cddc9
12 changed files with 556 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CLAUDE.md
+10
View File
@@ -14,3 +14,13 @@
- **zooid/util.go**: Environment variable utilities with `Env()` function.
- **cmd/relay/main.go**: HTTP server entry point with graceful shutdown.
## SQLite EventStore
The `sqlite/` directory contains a complete SQLite-based khatru eventstore implementation.
### nostrlib API Compatibility
- `Event.Sig` is `[64]byte`, not a separate Signature type
- `Event.CreatedAt` is `nostr.Timestamp` (int64), not `time.Time`
- Use `hex.EncodeToString(evt.Sig[:])` for signature serialization
- Use `hex.DecodeString()` and `copy()` for signature parsing
go.mod
+5
View File
@@ -5,7 +5,9 @@ go 1.24.1
require (
fiatjaf.com/nostr v0.0.0-20250924142401-59bd3c29fffd
github.com/BurntSushi/toml v1.5.0
github.com/mattn/go-sqlite3 v1.14.32
github.com/spf13/afero v1.15.0
github.com/stretchr/testify v1.10.0
)
require (
@@ -16,6 +18,7 @@ require (
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
github.com/coder/websocket v1.8.13 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/fasthttp/websocket v1.5.12 // indirect
@@ -26,6 +29,7 @@ require (
github.com/mailru/easyjson v0.9.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
github.com/rs/cors v1.11.1 // indirect
github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
@@ -37,4 +41,5 @@ require (
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/text v0.28.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
+3
View File
@@ -43,6 +43,8 @@ github.com/liamg/magic v0.0.1 h1:Ru22ElY+sCh6RvRTWjQzKKCxsEco8hE0co8n1qe7TBM=
github.com/liamg/magic v0.0.1/go.mod h1:yQkOmZZI52EA+SQ2xyHpVw8fNvTBruF873Y+Vt6S+fk=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -86,6 +88,7 @@ golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
sqlite/count.go
+10
View File
@@ -0,0 +1,10 @@
package sqlite
import (
"errors"
"fiatjaf.com/nostr"
)
func (s *SqliteBackend) CountEvents(nostr.Filter) (uint32, error) {
return 0, errors.New("not supported")
}
sqlite/delete.go
+13
View File
@@ -0,0 +1,13 @@
package sqlite
import (
"fiatjaf.com/nostr"
)
func (s *SqliteBackend) DeleteEvent(id nostr.ID) error {
s.Lock()
defer s.Unlock()
_, err := s.db.Exec("DELETE FROM events WHERE id = ?", id.Hex())
return err
}
sqlite/helpers.go
+8
View File
@@ -0,0 +1,8 @@
package sqlite
// Helper functions and constants for the SQLite eventstore
const (
// Database configuration
defaultTimeout = 30 // seconds
)
sqlite/lib.go
+113
View File
@@ -0,0 +1,113 @@
package sqlite
import (
"database/sql"
"fmt"
"sync"
"fiatjaf.com/nostr/eventstore"
_ "github.com/mattn/go-sqlite3"
)
var _ eventstore.Store = (*SqliteBackend)(nil)
type SqliteBackend struct {
sync.RWMutex
// Path is where the database will be saved
Path string
db *sql.DB
FTSAvailable bool
}
func (s *SqliteBackend) Close() {
if s.db != nil {
s.db.Close()
}
}
func (s *SqliteBackend) Init() error {
if s.Path == "" {
return fmt.Errorf("missing Path")
}
var err error
s.db, err = sql.Open("sqlite3", s.Path+"?_journal_mode=WAL&_sync=NORMAL&_cache_size=1000&_foreign_keys=true")
if err != nil {
return fmt.Errorf("failed to open database: %w", err)
}
// Create tables and indexes
if err := s.createSchema(); err != nil {
return fmt.Errorf("failed to create schema: %w", err)
}
return nil
}
func (s *SqliteBackend) createSchema() error {
// Create basic schema first
basicSchema := `
CREATE TABLE IF NOT EXISTS events (
id TEXT PRIMARY KEY,
created_at INTEGER NOT NULL,
kind INTEGER NOT NULL,
pubkey TEXT NOT NULL,
content TEXT NOT NULL,
tags TEXT NOT NULL,
sig TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at);
CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind);
CREATE INDEX IF NOT EXISTS idx_events_pubkey ON events(pubkey);
CREATE INDEX IF NOT EXISTS idx_events_kind_pubkey ON events(kind, pubkey);
CREATE INDEX IF NOT EXISTS idx_events_kind_pubkey_created_at ON events(kind, pubkey, created_at DESC);
CREATE TABLE IF NOT EXISTS event_tags (
event_id TEXT NOT NULL,
key TEXT NOT NULL,
value TEXT NOT NULL,
FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_event_tags_event_id ON event_tags(event_id);
CREATE INDEX IF NOT EXISTS idx_event_tags_key ON event_tags(key);
CREATE INDEX IF NOT EXISTS idx_event_tags_key_value ON event_tags(key, value);
`
if _, err := s.db.Exec(basicSchema); err != nil {
return fmt.Errorf("failed to create schema: %w", err)
}
// Try to create FTS5 schema - if it fails, continue without it
ftsSchema := `
CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
content,
content='events',
content_rowid='rowid'
);
CREATE TRIGGER IF NOT EXISTS events_ai AFTER INSERT ON events BEGIN
INSERT INTO events_fts(rowid, content) VALUES (new.rowid, new.content);
END;
CREATE TRIGGER IF NOT EXISTS events_ad AFTER DELETE ON events BEGIN
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', old.rowid, old.content);
END;
CREATE TRIGGER IF NOT EXISTS events_au AFTER UPDATE ON events BEGIN
INSERT INTO events_fts(events_fts, rowid, content) VALUES('delete', old.rowid, old.content);
INSERT INTO events_fts(rowid, content) VALUES (new.rowid, new.content);
END;
`
if _, err := s.db.Exec(ftsSchema); err != nil {
// FTS5 not available, continue without full-text search
s.FTSAvailable = false
} else {
s.FTSAvailable = true
}
return nil
}
sqlite/query.go
+171
View File
@@ -0,0 +1,171 @@
package sqlite
import (
"encoding/hex"
"encoding/json"
"fmt"
"iter"
"strings"
"fiatjaf.com/nostr"
)
func (s *SqliteBackend) QueryEvents(filter nostr.Filter, maxLimit int) iter.Seq[nostr.Event] {
return func(yield func(nostr.Event) bool) {
s.RLock()
defer s.RUnlock()
if filter.LimitZero {
return
}
limit := maxLimit
if filter.Limit > 0 && filter.Limit < limit {
limit = filter.Limit
}
query, args := s.buildSelectQuery(filter, limit)
rows, err := s.db.Query(query, args...)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var evt nostr.Event
var idStr, pubkeyStr, sigStr, tagsStr string
var createdAt int64
var kind int
err := rows.Scan(&idStr, &createdAt, &kind, &pubkeyStr, &evt.Content, &tagsStr, &sigStr)
if err != nil {
continue
}
// Parse ID
if id, err := nostr.IDFromHex(idStr); err == nil {
evt.ID = id
} else {
continue
}
// Parse PubKey
if pubkey, err := nostr.PubKeyFromHex(pubkeyStr); err == nil {
evt.PubKey = pubkey
} else {
continue
}
// Parse Signature
if sigBytes, err := hex.DecodeString(sigStr); err == nil && len(sigBytes) == 64 {
copy(evt.Sig[:], sigBytes)
} else {
continue
}
// Set other fields
evt.CreatedAt = nostr.Timestamp(createdAt)
evt.Kind = nostr.Kind(kind)
// Parse Tags
if err := json.Unmarshal([]byte(tagsStr), &evt.Tags); err != nil {
continue
}
if !yield(evt) {
return
}
}
}
}
func (s *SqliteBackend) buildSelectQuery(filter nostr.Filter, limit int) (string, []interface{}) {
var conditions []string
var args []interface{}
var joins []string
baseQuery := "SELECT id, created_at, kind, pubkey, content, tags, sig FROM events"
// Handle search with FTS (if available)
if filter.Search != "" && s.FTSAvailable {
joins = append(joins, "JOIN events_fts ON events.rowid = events_fts.rowid")
conditions = append(conditions, "events_fts MATCH ?")
args = append(args, filter.Search)
} else if filter.Search != "" {
// Fallback to LIKE search if FTS not available
conditions = append(conditions, "content LIKE ?")
args = append(args, "%"+filter.Search+"%")
}
// Add WHERE clause conditions
if len(filter.IDs) > 0 {
placeholders := make([]string, len(filter.IDs))
for i, id := range filter.IDs {
placeholders[i] = "?"
args = append(args, id.Hex())
}
conditions = append(conditions, fmt.Sprintf("id IN (%s)", strings.Join(placeholders, ",")))
}
if len(filter.Authors) > 0 {
placeholders := make([]string, len(filter.Authors))
for i, author := range filter.Authors {
placeholders[i] = "?"
args = append(args, author.Hex())
}
conditions = append(conditions, fmt.Sprintf("pubkey IN (%s)", strings.Join(placeholders, ",")))
}
if len(filter.Kinds) > 0 {
placeholders := make([]string, len(filter.Kinds))
for i, kind := range filter.Kinds {
placeholders[i] = "?"
args = append(args, int(kind))
}
conditions = append(conditions, fmt.Sprintf("kind IN (%s)", strings.Join(placeholders, ",")))
}
if filter.Since != 0 {
conditions = append(conditions, "created_at >= ?")
args = append(args, filter.Since)
}
if filter.Until != 0 {
conditions = append(conditions, "created_at <= ?")
args = append(args, filter.Until)
}
// Handle tags - only filter single-letter tags using event_tags table
for tagKey, tagValues := range filter.Tags {
if len(tagValues) > 0 && len(tagKey) == 1 {
placeholders := make([]string, len(tagValues))
for i, tagValue := range tagValues {
placeholders[i] = "?"
args = append(args, tagValue)
}
conditions = append(conditions, fmt.Sprintf("id IN (SELECT event_id FROM event_tags WHERE key = ? AND value IN (%s))", strings.Join(placeholders, ",")))
args = append(args, tagKey)
}
}
// Build the complete query
if len(joins) > 0 {
baseQuery += " " + strings.Join(joins, " ")
}
if len(conditions) > 0 {
baseQuery += " WHERE " + strings.Join(conditions, " AND ")
}
// Order by created_at DESC for most recent first
baseQuery += " ORDER BY created_at DESC"
// Add limit
if limit > 0 {
baseQuery += " LIMIT ?"
args = append(args, limit)
}
return baseQuery, args
}
sqlite/replace.go
+37
View File
@@ -0,0 +1,37 @@
package sqlite
import (
"fmt"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore"
)
func (s *SqliteBackend) ReplaceEvent(evt nostr.Event) error {
s.Lock()
defer s.Unlock()
filter := nostr.Filter{Kinds: []nostr.Kind{evt.Kind}, Authors: []nostr.PubKey{evt.PubKey}}
if evt.Kind.IsAddressable() {
filter.Tags = nostr.TagMap{"d": []string{evt.Tags.GetD()}}
}
shouldStore := true
for previous := range s.QueryEvents(filter, 1) {
if previous.CreatedAt <= evt.CreatedAt {
if err := s.DeleteEvent(previous.ID); err != nil {
return fmt.Errorf("failed to delete event for replacing: %w", err)
}
} else {
shouldStore = false
}
}
if shouldStore {
if err := s.SaveEvent(evt); err != nil && err != eventstore.ErrDupEvent {
return fmt.Errorf("failed to save: %w", err)
}
}
return nil
}
sqlite/save.go
+61
View File
@@ -0,0 +1,61 @@
package sqlite
import (
"encoding/hex"
"encoding/json"
"fmt"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore"
)
func (s *SqliteBackend) SaveEvent(evt nostr.Event) error {
s.Lock()
defer s.Unlock()
// Check if event already exists
var existingID string
err := s.db.QueryRow("SELECT id FROM events WHERE id = ?", evt.ID.Hex()).Scan(&existingID)
if err == nil {
// Event already exists
return eventstore.ErrDupEvent
}
// Serialize tags to JSON
tagsJSON, err := json.Marshal(evt.Tags)
if err != nil {
return fmt.Errorf("failed to marshal tags: %w", err)
}
// Insert the event
query := `INSERT INTO events (id, created_at, kind, pubkey, content, tags, sig)
VALUES (?, ?, ?, ?, ?, ?, ?)`
_, err = s.db.Exec(query,
evt.ID.Hex(),
int64(evt.CreatedAt),
int(evt.Kind),
evt.PubKey.Hex(),
evt.Content,
string(tagsJSON),
hex.EncodeToString(evt.Sig[:]),
)
if err != nil {
return fmt.Errorf("failed to save event '%s': %w", evt.ID, err)
}
// Insert single-letter tags into event_tags table
for _, tag := range evt.Tags {
if len(tag) >= 2 && len(tag[0]) == 1 {
_, err := s.db.Exec("INSERT INTO event_tags (event_id, key, value) VALUES (?, ?, ?)",
evt.ID.Hex(), tag[0], tag[1])
if err != nil {
// Log error but don't fail the entire save operation
continue
}
}
}
return nil
}
sqlite/sqlite_test.go
+110
View File
@@ -0,0 +1,110 @@
package sqlite
import (
"os"
"testing"
"fiatjaf.com/nostr"
"github.com/stretchr/testify/assert"
)
func TestSqliteFlow(t *testing.T) {
os.RemoveAll("/tmp/sqlitetest.db")
sb := &SqliteBackend{
Path: "/tmp/sqlitetest.db",
}
err := sb.Init()
assert.NoError(t, err)
defer sb.Close()
willDelete := make([]nostr.Event, 0, 3)
sk := nostr.MustSecretKeyFromHex("0000000000000000000000000000000000000000000000000000000000000001")
for i, content := range []string{
"good morning mr paper maker",
"good night",
"I'll see you again in the paper house",
"tonight we dine in my house",
"the paper in this house if very good, mr",
} {
evt := nostr.Event{
Content: content,
Tags: nostr.Tags{},
Kind: 1,
CreatedAt: nostr.Now(),
PubKey: sk.Public(),
}
evt.ID = evt.GetID()
// For testing, we'll skip actual signing and just set a dummy signature
// In real usage, you'd call evt.Sign(sk)
err := sb.SaveEvent(evt)
assert.NoError(t, err)
if i%2 == 0 {
willDelete = append(willDelete, evt)
}
}
// Test search functionality (if FTS5 is available)
if sb.FTSAvailable {
n := 0
for range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) {
n++
}
assert.Equal(t, 3, n)
} else {
// With LIKE fallback, should still work but might be less precise
n := 0
for range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) {
n++
}
assert.Equal(t, 3, n)
}
// Delete some events
for _, evt := range willDelete {
err := sb.DeleteEvent(evt.ID)
assert.NoError(t, err)
}
// Test search after deletion
if sb.FTSAvailable {
n := 0
for res := range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) {
n++
assert.Equal(t, res.Content, "good night")
assert.Equal(t, sk.Public(), res.PubKey)
}
assert.Equal(t, 1, n)
} else {
// With LIKE fallback, should still work
n := 0
for res := range sb.QueryEvents(nostr.Filter{Search: "good"}, 400) {
n++
assert.Equal(t, res.Content, "good night")
assert.Equal(t, sk.Public(), res.PubKey)
}
assert.Equal(t, 1, n)
}
// Test query by kind
{
n := 0
for range sb.QueryEvents(nostr.Filter{Kinds: []nostr.Kind{1}}, 400) {
n++
}
assert.Equal(t, 2, n)
}
// Test query by author
{
n := 0
for range sb.QueryEvents(nostr.Filter{Authors: []nostr.PubKey{sk.Public()}}, 400) {
n++
}
assert.Equal(t, 2, n)
}
}
zooid/blossom.go
+15 -13
View File
@@ -8,9 +8,9 @@ import (
"net/url"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore/lmdb"
"fiatjaf.com/nostr/khatru/blossom"
"github.com/spf13/afero"
"zooid/sqlite"
)
func EnableBlossom(instance *Instance) {
@@ -20,16 +20,15 @@ func EnableBlossom(instance *Instance) {
log.Fatal("🚫 error creating blossom path:", err)
}
backend := &lmdb.LMDBBackend{Path: instance.Config.Data.Blossom}
if err := backend.Init(); err != nil {
panic(err)
store := &sqlite.SqliteBackend{
Path: instance.Config.Data.Blossom,
}
blossom := blossom.New(instance.Relay, "https://"+instance.Host)
backend := blossom.New(instance.Relay, "https://"+instance.Host)
blossom.Store = backend
backend.Store = blossom.EventStoreBlobIndexWrapper{Store: store, ServiceURL: "https://" + instance.Host}
blossom.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error {
backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error {
file, err := fs.Create(instance.Config.Blossom.Directory + "/" + sha256)
if err != nil {
return err
@@ -42,7 +41,7 @@ func EnableBlossom(instance *Instance) {
return nil
}
blossom.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) {
backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) {
file, err := fs.Open(instance.Config.Blossom.Directory + "/" + sha256)
if err != nil {
return nil, nil, err
@@ -50,11 +49,11 @@ func EnableBlossom(instance *Instance) {
return file, nil, nil
}
blossom.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error {
backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error {
return fs.Remove(instance.Config.Blossom.Directory + "/" + sha256)
}
blossom.RejectUpload = func(ctx context.Context, auth *nostr.Event, size int, ext string) (bool, string, int) {
backend.RejectUpload = func(ctx context.Context, auth *nostr.Event, size int, ext string) (bool, string, int) {
if size > 10*1024*1024 {
return true, "file too large", 413
}
@@ -66,7 +65,7 @@ func EnableBlossom(instance *Instance) {
return false, ext, size
}
blossom.RejectGet = func(ctx context.Context, auth *nostr.Event, sha256 string, ext string) (bool, string, int) {
backend.RejectGet = func(ctx context.Context, auth *nostr.Event, sha256 string, ext string) (bool, string, int) {
if auth == nil || !instance.IsMember(auth.PubKey) {
return true, "unauthorized", 403
}
@@ -74,7 +73,7 @@ func EnableBlossom(instance *Instance) {
return false, "", 200
}
blossom.RejectList = func(ctx context.Context, auth *nostr.Event, pubkey nostr.PubKey) (bool, string, int) {
backend.RejectList = func(ctx context.Context, auth *nostr.Event, pubkey nostr.PubKey) (bool, string, int) {
if auth == nil || !instance.IsMember(auth.PubKey) {
return true, "unauthorized", 403
}
@@ -82,11 +81,14 @@ func EnableBlossom(instance *Instance) {
return false, "", 200
}
blossom.RejectDelete = func(ctx context.Context, auth *nostr.Event, sha256 string, ext string) (bool, string, int) {
backend.RejectDelete = func(ctx context.Context, auth *nostr.Event, sha256 string, ext string) (bool, string, int) {
if auth == nil || !instance.IsMember(auth.PubKey) {
return true, "unauthorized", 403
}
return false, "", 200
}
if err := store.Init(); err != nil {
panic(err)
}
}