bring in khatru and eventstore.

eventstore/cmd/eventstore/.gitignore

@@ -0,0 +1 @@
+eventstore

eventstore/cmd/eventstore/README.md

@@ -0,0 +1,39 @@
+# eventstore command-line tool
+
+```
+go install github.com/fiatjaf/eventstore/cmd/eventstore@latest
+```
+
+## Usage
+
+This should be pretty straightforward. You pipe events or filters, as JSON, to the `eventstore` command, and they yield something. You can use [nak](https://github.com/fiatjaf/nak) to generate these events or filters easily.
+
+### Querying the last 100 events of kind 1
+
+```fish
+~> nak req -k 1 -l 100 --bare | eventstore -d /path/to/store query
+~> # or
+~> echo '{"kinds":[1],"limit":100}' | eventstore -d /path/to/store query
+```
+
+This will automatically determine the storage type being used at `/path/to/store`, but you can also specify it manually using the `-t` option (`-t lmdb`, `-t sqlite` etc).
+
+### Saving an event to the store
+
+```fish
+~> nak event -k 1 -c hello | eventstore -d /path/to/store save
+~> # or
+~> echo '{"id":"35369e6bae5f77c4e1745c2eb5db84c4493e87f6e449aee62a261bbc1fea2788","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1701193836,"kind":1,"tags":[],"content":"hello","sig":"ef08d559e042d9af4cdc3328a064f737603d86ec4f929f193d5a3ce9ea22a3fb8afc1923ee3c3742fd01856065352c5632e91f633528c80e9c5711fa1266824c"}' | eventstore -d /path/to/store save
+```
+
+You can also create a database from scratch if it's a disk database, but then you have to specify `-t` to `sqlite`, `badger` or `lmdb`.
+
+### Connecting to Postgres, MySQL and other remote databases
+
+You should be able to connect by just passing the database connection URI to `-d`:
+
+```bash
+~> eventstore -d 'postgres://myrelay:38yg4o83yf48a3s7g@localhost:5432/myrelay?sslmode=disable' <query|save|delete>
+```
+
+That should be prefixed with `postgres://` for Postgres, `mysql://` for MySQL and `https://` for ElasticSearch.

eventstore/cmd/eventstore/delete.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var delete_ = &cli.Command{
+	Name:        "delete",
+	ArgsUsage:   "[<id>]",
+	Usage:       "deletes an event by id and all its associated index entries",
+	Description: "takes an id either as an argument or reads a stream of ids from stdin and deletes them from the currently open eventstore.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			f := nostr.Filter{IDs: []string{line}}
+			ch, err := db.QueryEvents(ctx, f)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error querying for %s: %s\n", f, err)
+				hasError = true
+			}
+			for evt := range ch {
+				if err := db.DeleteEvent(ctx, evt); err != nil {
+					fmt.Fprintf(os.Stderr, "error deleting %s: %s\n", evt, err)
+					hasError = true
+				}
+			}
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}

eventstore/cmd/eventstore/helpers.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v3"
+)
+
+const (
+	LINE_PROCESSING_ERROR = iota
+)
+
+func detect(dir string) (string, error) {
+	mayBeMMM := false
+	if n := strings.Index(dir, "/"); n > 0 {
+		mayBeMMM = true
+		dir = filepath.Dir(dir)
+	}
+
+	f, err := os.Stat(dir)
+	if err != nil {
+		return "", err
+	}
+	if !f.IsDir() {
+		f, err := os.Open(dir)
+		if err != nil {
+			return "", err
+		}
+		buf := make([]byte, 15)
+		f.Read(buf)
+		if string(buf) == "SQLite format 3" {
+			return "sqlite", nil
+		}
+
+		return "", fmt.Errorf("unknown db format")
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return "", err
+	}
+
+	if mayBeMMM {
+		for _, entry := range entries {
+			if entry.Name() == "mmmm" {
+				if entries, err := os.ReadDir(filepath.Join(dir, "mmmm")); err == nil {
+					for _, e := range entries {
+						if strings.HasSuffix(e.Name(), ".mdb") {
+							return "mmm", nil
+						}
+					}
+				}
+			}
+		}
+	}
+	for _, entry := range entries {
+		if strings.HasSuffix(entry.Name(), ".mdb") {
+			return "lmdb", nil
+		}
+		if strings.HasSuffix(entry.Name(), ".vlog") {
+			return "badger", nil
+		}
+	}
+
+	return "", fmt.Errorf("undetected")
+}
+
+func getStdin() string {
+	stat, _ := os.Stdin.Stat()
+	if (stat.Mode() & os.ModeCharDevice) == 0 {
+		read := bytes.NewBuffer(make([]byte, 0, 1000))
+		_, err := io.Copy(read, os.Stdin)
+		if err == nil {
+			return read.String()
+		}
+	}
+	return ""
+}
+
+func isPiped() bool {
+	stat, _ := os.Stdin.Stat()
+	return stat.Mode()&os.ModeCharDevice == 0
+}
+
+func getStdinLinesOrFirstArgument(c *cli.Command) chan string {
+	// try the first argument
+	target := c.Args().First()
+	if target != "" {
+		single := make(chan string, 1)
+		single <- target
+		close(single)
+		return single
+	}
+
+	// try the stdin
+	multi := make(chan string)
+	writeStdinLinesOrNothing(multi)
+	return multi
+}
+
+func getStdinLinesOrBlank() chan string {
+	multi := make(chan string)
+	if hasStdinLines := writeStdinLinesOrNothing(multi); !hasStdinLines {
+		single := make(chan string, 1)
+		single <- ""
+		close(single)
+		return single
+	} else {
+		return multi
+	}
+}
+
+func writeStdinLinesOrNothing(ch chan string) (hasStdinLines bool) {
+	if isPiped() {
+		// piped
+		go func() {
+			scanner := bufio.NewScanner(os.Stdin)
+			for scanner.Scan() {
+				ch <- strings.TrimSpace(scanner.Text())
+			}
+			close(ch)
+		}()
+		return true
+	} else {
+		// not piped
+		return false
+	}
+}

eventstore/cmd/eventstore/main.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/badger"
+	"github.com/fiatjaf/eventstore/elasticsearch"
+	"github.com/fiatjaf/eventstore/lmdb"
+	"github.com/fiatjaf/eventstore/mysql"
+	"github.com/fiatjaf/eventstore/postgresql"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/fiatjaf/eventstore/sqlite3"
+	"github.com/fiatjaf/eventstore/strfry"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/urfave/cli/v3"
+)
+
+var db eventstore.Store
+
+var app = &cli.Command{
+	Name:      "eventstore",
+	Usage:     "a CLI for all the eventstore backends",
+	UsageText: "eventstore -d ./data/sqlite <query|save|delete> ...",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "store",
+			Aliases:  []string{"d"},
+			Usage:    "path to the database file or directory or database connection uri",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:    "type",
+			Aliases: []string{"t"},
+			Usage:   "store type ('sqlite', 'lmdb', 'badger', 'postgres', 'mysql', 'elasticsearch', 'mmm')",
+		},
+	},
+	Before: func(ctx context.Context, c *cli.Command) (context.Context, error) {
+		path := strings.Trim(c.String("store"), "/")
+		typ := c.String("type")
+		if typ != "" {
+			// bypass automatic detection
+			// this also works for creating disk databases from scratch
+		} else {
+			// try to detect based on url scheme
+			switch {
+			case strings.HasPrefix(path, "postgres://"), strings.HasPrefix(path, "postgresql://"):
+				typ = "postgres"
+			case strings.HasPrefix(path, "mysql://"):
+				typ = "mysql"
+			case strings.HasPrefix(path, "https://"):
+				// if we ever add something else that uses URLs we'll have to modify this
+				typ = "elasticsearch"
+			case strings.HasSuffix(path, ".conf"):
+				typ = "strfry"
+			case strings.HasSuffix(path, ".jsonl"):
+				typ = "file"
+			default:
+				// try to detect based on the form and names of disk files
+				dbname, err := detect(path)
+				if err != nil {
+					if os.IsNotExist(err) {
+						return ctx, fmt.Errorf(
+							"'%s' does not exist, to create a store there specify the --type argument", path)
+					}
+					return ctx, fmt.Errorf("failed to detect store type: %w", err)
+				}
+				typ = dbname
+			}
+		}
+
+		switch typ {
+		case "sqlite":
+			db = &sqlite3.SQLite3Backend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "lmdb":
+			db = &lmdb.LMDBBackend{Path: path, MaxLimit: 1_000_000}
+		case "badger":
+			db = &badger.BadgerBackend{Path: path, MaxLimit: 1_000_000}
+		case "mmm":
+			var err error
+			if db, err = doMmmInit(path); err != nil {
+				return ctx, err
+			}
+		case "postgres", "postgresql":
+			db = &postgresql.PostgresBackend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "mysql":
+			db = &mysql.MySQLBackend{
+				DatabaseURL:       path,
+				QueryLimit:        1_000_000,
+				QueryAuthorsLimit: 1_000_000,
+				QueryKindsLimit:   1_000_000,
+				QueryIDsLimit:     1_000_000,
+				QueryTagsLimit:    1_000_000,
+			}
+		case "elasticsearch":
+			db = &elasticsearch.ElasticsearchStorage{URL: path}
+		case "strfry":
+			db = &strfry.StrfryBackend{ConfigPath: path}
+		case "file":
+			db = &slicestore.SliceStore{}
+
+			// run this after we've called db.Init()
+			defer func() {
+				f, err := os.Open(path)
+				if err != nil {
+					log.Printf("failed to file at '%s': %s\n", path, err)
+					os.Exit(3)
+				}
+				scanner := bufio.NewScanner(f)
+				scanner.Buffer(make([]byte, 16*1024*1024), 256*1024*1024)
+				i := 0
+				for scanner.Scan() {
+					var evt nostr.Event
+					if err := json.Unmarshal(scanner.Bytes(), &evt); err != nil {
+						log.Printf("invalid event read at line %d: %s (`%s`)\n", i, err, scanner.Text())
+					}
+					db.SaveEvent(ctx, &evt)
+					i++
+				}
+			}()
+		case "":
+			return ctx, fmt.Errorf("couldn't determine store type, you can use --type to specify it manually")
+		default:
+			return ctx, fmt.Errorf("'%s' store type is not supported by this CLI", typ)
+		}
+
+		if err := db.Init(); err != nil {
+			return ctx, err
+		}
+
+		return ctx, nil
+	},
+	Commands: []*cli.Command{
+		queryOrSave,
+		query,
+		save,
+		delete_,
+		neg,
+	},
+	DefaultCommand: "query-or-save",
+}
+
+func main() {
+	if err := app.Run(context.Background(), os.Args); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

eventstore/cmd/eventstore/main_mmm.go

@@ -0,0 +1,34 @@
+//go:build !windows
+
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/fiatjaf/eventstore"
+	"github.com/fiatjaf/eventstore/mmm"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/rs/zerolog"
+)
+
+func doMmmInit(path string) (eventstore.Store, error) {
+	logger := zerolog.New(zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
+		w.Out = os.Stderr
+	}))
+	mmmm := mmm.MultiMmapManager{
+		Dir:    filepath.Dir(path),
+		Logger: &logger,
+	}
+	if err := mmmm.Init(); err != nil {
+		return nil, err
+	}
+	il := &mmm.IndexingLayer{
+		ShouldIndex: func(ctx context.Context, e *nostr.Event) bool { return false },
+	}
+	if err := mmmm.EnsureLayer(filepath.Base(path), il); err != nil {
+		return nil, err
+	}
+	return il, nil
+}

eventstore/cmd/eventstore/main_other.go

@@ -0,0 +1,14 @@
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/fiatjaf/eventstore"
+)
+
+func doMmmInit(path string) (eventstore.Store, error) {
+	return nil, fmt.Errorf("unsupported OSs (%v)", runtime.GOOS)
+}

eventstore/cmd/eventstore/neg.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"sync"
+
+	"github.com/urfave/cli/v3"
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy/storage/vector"
+)
+
+var neg = &cli.Command{
+	Name:        "neg",
+	ArgsUsage:   "<filter-json> [<negentropy-message-hex>]",
+	Usage:       "initiates a negentropy session with a filter or reconciles a received negentropy message",
+	Description: "applies the filter to the currently open eventstore. if no negentropy message was given it will initiate the process and emit one, if one was given either as an argument or via stdin, it will be reconciled against the current eventstore.\nthe next reconciliation message will be emitted on stdout.\na stream of need/have ids (or nothing) will be emitted to stderr.",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name: "frame-size-limit",
+		},
+	},
+	Action: func(ctx context.Context, c *cli.Command) error {
+		jfilter := c.Args().First()
+		if jfilter == "" {
+			return fmt.Errorf("missing filter argument")
+		}
+
+		filter := nostr.Filter{}
+		if err := easyjson.Unmarshal([]byte(jfilter), &filter); err != nil {
+			return fmt.Errorf("invalid filter %s: %s\n", jfilter, err)
+		}
+
+		frameSizeLimit := int(c.Uint("frame-size-limit"))
+		if frameSizeLimit == 0 {
+			frameSizeLimit = math.MaxInt
+		}
+
+		// create negentropy object and initialize it with events
+		vec := vector.New()
+		neg := negentropy.New(vec, frameSizeLimit)
+		ch, err := db.QueryEvents(ctx, filter)
+		if err != nil {
+			return fmt.Errorf("error querying: %s\n", err)
+		}
+		for evt := range ch {
+			vec.Insert(evt.CreatedAt, evt.ID)
+		}
+
+		wg := sync.WaitGroup{}
+		go func() {
+			defer wg.Done()
+			for item := range neg.Haves {
+				fmt.Fprintf(os.Stderr, "have %s", item)
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			for item := range neg.HaveNots {
+				fmt.Fprintf(os.Stderr, "need %s", item)
+			}
+		}()
+
+		// get negentropy message from argument or stdin pipe
+		var msg string
+		if isPiped() {
+			data, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return fmt.Errorf("failed to read from stdin: %w", err)
+			}
+			msg = string(data)
+		} else {
+			msg = c.Args().Get(1)
+		}
+
+		if msg == "" {
+			// initiate the process
+			out := neg.Start()
+			fmt.Println(out)
+		} else {
+			// process the message
+			out, err := neg.Reconcile(msg)
+			if err != nil {
+				return fmt.Errorf("negentropy failed: %s", err)
+			}
+			fmt.Println(out)
+		}
+
+		wg.Wait()
+		return nil
+	},
+}

eventstore/cmd/eventstore/query-or-save.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+// this is the default command when no subcommands are given, we will just try everything
+var queryOrSave = &cli.Command{
+	Hidden: true,
+	Name:   "query-or-save",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		line := getStdin()
+
+		ee := &nostr.EventEnvelope{}
+		re := &nostr.ReqEnvelope{}
+		e := &nostr.Event{}
+		f := &nostr.Filter{}
+		if json.Unmarshal([]byte(line), ee) == nil && ee.Event.ID != "" {
+			e = &ee.Event
+			return doSave(ctx, line, e)
+		}
+		if json.Unmarshal([]byte(line), e) == nil && e.ID != "" {
+			return doSave(ctx, line, e)
+		}
+		if json.Unmarshal([]byte(line), re) == nil && len(re.Filters) > 0 {
+			f = &re.Filters[0]
+			return doQuery(ctx, f)
+		}
+		if json.Unmarshal([]byte(line), f) == nil && len(f.String()) > 2 {
+			return doQuery(ctx, f)
+		}
+
+		return fmt.Errorf("couldn't parse input '%s'", line)
+	},
+}
+
+func doSave(ctx context.Context, line string, e *nostr.Event) error {
+	if err := db.SaveEvent(ctx, e); err != nil {
+		return fmt.Errorf("failed to save event '%s': %s", line, err)
+	}
+	fmt.Fprintf(os.Stderr, "saved %s", e.ID)
+	return nil
+}
+
+func doQuery(ctx context.Context, f *nostr.Filter) error {
+	ch, err := db.QueryEvents(ctx, *f)
+	if err != nil {
+		return fmt.Errorf("error querying: %w", err)
+	}
+
+	for evt := range ch {
+		fmt.Println(evt)
+	}
+	return nil
+}

eventstore/cmd/eventstore/query.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/urfave/cli/v3"
+)
+
+var query = &cli.Command{
+	Name:        "query",
+	ArgsUsage:   "[<filter-json>]",
+	Usage:       "queries an eventstore for events, takes a filter as argument",
+	Description: "applies the filter to the currently open eventstore, returning up to a million events.\n takes either a filter as an argument or reads a stream of filters from stdin.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			filter := nostr.Filter{}
+			if err := easyjson.Unmarshal([]byte(line), &filter); err != nil {
+				fmt.Fprintf(os.Stderr, "invalid filter '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			ch, err := db.QueryEvents(ctx, filter)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error querying: %s\n", err)
+				hasError = true
+				continue
+			}
+
+			for evt := range ch {
+				fmt.Println(evt)
+			}
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}

eventstore/cmd/eventstore/save.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v3"
+	"github.com/mailru/easyjson"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var save = &cli.Command{
+	Name:        "save",
+	ArgsUsage:   "[<event-json>]",
+	Usage:       "stores an event",
+	Description: "takes either an event as an argument or reads a stream of events from stdin and inserts those in the currently opened eventstore.\ndoesn't perform any kind of signature checking or replacement.",
+	Action: func(ctx context.Context, c *cli.Command) error {
+		hasError := false
+		for line := range getStdinLinesOrFirstArgument(c) {
+			var event nostr.Event
+			if err := easyjson.Unmarshal([]byte(line), &event); err != nil {
+				fmt.Fprintf(os.Stderr, "invalid event '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			if err := db.SaveEvent(ctx, &event); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to save event '%s': %s\n", line, err)
+				hasError = true
+				continue
+			}
+
+			fmt.Fprintf(os.Stderr, "saved %s\n", event.ID)
+		}
+
+		if hasError {
+			os.Exit(123)
+		}
+		return nil
+	},
+}