From 4f3addc4a44543a0daafacd9334c4e5947602d5a Mon Sep 17 00:00:00 2001 From: userAdityaa Date: Tue, 12 May 2026 01:22:46 +0545 Subject: [PATCH] feat(blossom): optional S3-compatible blob storage --- go.mod | 18 ++++++ go.sum | 36 ++++++++++++ zooid/api.go | 3 + zooid/blossom.go | 135 +++++++++++++++++++++++++++++++++++++------ zooid/config.go | 62 +++++++++++++++++++- zooid/config_test.go | 89 ++++++++++++++++++++++++++++ 6 files changed, 321 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 8352b60..f2dc4bb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,10 @@ require ( fiatjaf.com/nostr v0.0.0-20251104112613-38a6ca92b954 github.com/BurntSushi/toml v1.5.0 github.com/Masterminds/squirrel v1.5.4 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/fsnotify/fsnotify v1.9.0 github.com/gosimple/slug v1.15.0 github.com/livekit/protocol v1.43.5-0.20260114074149-a8bb8204ce69 @@ -22,6 +26,20 @@ require ( github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect diff --git a/go.sum b/go.sum index 4b6751e..68f3532 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,42 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/zooid/api.go b/zooid/api.go index e82ca45..9f2decc 100644 --- a/zooid/api.go +++ b/zooid/api.go @@ -358,6 +358,9 @@ func (api *APIHandler) validateConfig(config *Config) error { return fmt.Errorf("invalid info.pubkey: %w", err) } } + if err := validateBlossomFileStorage(config); err != nil { + return err + } return nil } diff --git a/zooid/blossom.go b/zooid/blossom.go index 12a659e..c36be57 100644 --- a/zooid/blossom.go +++ b/zooid/blossom.go @@ -3,12 +3,20 @@ package zooid import ( "bytes" "context" + "fmt" "io" + "log" "net/url" + "path/filepath" + "strings" "fiatjaf.com/nostr" "fiatjaf.com/nostr/eventstore" "fiatjaf.com/nostr/khatru/blossom" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gosimple/slug" "github.com/spf13/afero" ) @@ -18,10 +26,33 @@ type BlossomStore struct { Events eventstore.Store } +func newBlossomS3Client(ctx context.Context, cfg *Config) (*s3.Client, error) { + s := cfg.Blossom.S3 + secret := strings.TrimSpace(cfg.blossomS3SecretKey) + if secret == "" { + return nil, fmt.Errorf("s3 secret key is not configured") + } + + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(strings.TrimSpace(s.Region)), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(strings.TrimSpace(s.AccessKey), secret, "")), + ) + if err != nil { + return nil, err + } + + endpoint := strings.TrimRight(strings.TrimSpace(s.Endpoint), "/") + + return s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + } + o.UsePathStyle = s.UsePathStyle + }), nil +} + func (bl *BlossomStore) Enable(instance *Instance) { - dir := Env("MEDIA") + "/" + slug.Make(bl.Config.Schema) - fs := afero.NewOsFs() - fs.MkdirAll(dir, 0755) + slugName := slug.Make(bl.Config.Schema) backend := blossom.New(instance.Relay, "https://"+bl.Config.Host) backend.Store = blossom.EventStoreBlobIndexWrapper{ @@ -29,29 +60,95 @@ func (bl *BlossomStore) Enable(instance *Instance) { ServiceURL: "https://" + bl.Config.Host, } - backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { - file, err := fs.Create(dir + "/" + sha256) + fs := strings.ToLower(strings.TrimSpace(bl.Config.Blossom.FileStorage)) + if fs == "" { + fs = "local" + } + + switch fs { + case "local": + dir := filepath.Join(Env("MEDIA"), slugName) + osfs := afero.NewOsFs() + _ = osfs.MkdirAll(dir, 0755) + + backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { + file, err := osfs.Create(filepath.Join(dir, sha256)) + if err != nil { + return err + } + + if _, err := io.Copy(file, bytes.NewReader(body)); err != nil { + return err + } + + return nil + } + + backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { + file, err := osfs.Open(filepath.Join(dir, sha256)) + if err != nil { + return nil, nil, err + } + return file, nil, nil + } + + backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { + return osfs.Remove(filepath.Join(dir, sha256)) + } + + case "s3": + client, err := newBlossomS3Client(context.Background(), bl.Config) if err != nil { + log.Fatalf("blossom: s3 client: %v", err) + } + + bucket := strings.TrimSpace(bl.Config.Blossom.S3.Bucket) + makeKey := func(sha string) string { + prefix := strings.Trim(strings.TrimSpace(bl.Config.Blossom.S3.KeyPrefix), "/") + rel := slugName + "/" + sha + if prefix != "" { + return prefix + "/" + rel + } + return rel + } + + backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(makeKey(sha256)), + Body: bytes.NewReader(body), + }) return err } - if _, err := io.Copy(file, bytes.NewReader(body)); err != nil { + backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { + out, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(makeKey(sha256)), + }) + if err != nil { + return nil, nil, err + } + defer out.Body.Close() + + data, err := io.ReadAll(out.Body) + if err != nil { + return nil, nil, err + } + + return bytes.NewReader(data), nil, nil + } + + backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { + _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(makeKey(sha256)), + }) return err } - return nil - } - - backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { - file, err := fs.Open(dir + "/" + sha256) - if err != nil { - return nil, nil, err - } - return file, nil, nil - } - - backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { - return fs.Remove(dir + "/" + sha256) + default: + log.Fatalf("blossom: unknown file_storage %q (use local or s3)", bl.Config.Blossom.FileStorage) } backend.RejectUpload = func(ctx context.Context, auth *nostr.Event, size int, ext string) (bool, string, int) { diff --git a/zooid/config.go b/zooid/config.go index 3fa3ba9..3d7a545 100644 --- a/zooid/config.go +++ b/zooid/config.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "strings" ) type Role struct { @@ -45,7 +46,9 @@ type Config struct { } `toml:"management" json:"management"` Blossom struct { - Enabled bool `toml:"enabled" json:"enabled"` + Enabled bool `toml:"enabled" json:"enabled"` + FileStorage string `toml:"file_storage" json:"file_storage"` + S3 BlossomS3Settings `toml:"s3" json:"s3"` } `toml:"blossom" json:"blossom"` Livekit struct { @@ -57,8 +60,21 @@ type Config struct { Roles map[string]Role `toml:"roles" json:"roles"` // Private/parsed values - path string - secret nostr.SecretKey + path string + secret nostr.SecretKey + blossomS3SecretKey string +} + +// BlossomS3Settings configures S3-compatible object storage for Blossom blobs +// when [blossom] file_storage is "s3". +type BlossomS3Settings struct { + Endpoint string `toml:"endpoint" json:"endpoint"` + Region string `toml:"region" json:"region"` + Bucket string `toml:"bucket" json:"bucket"` + AccessKey string `toml:"access_key" json:"access_key"` + SecretKey string `toml:"secret_key" json:"secret_key"` + KeyPrefix string `toml:"key_prefix" json:"key_prefix"` + UsePathStyle bool `toml:"use_path_style" json:"use_path_style"` } func LoadConfig(filename string) (*Config, error) { @@ -97,12 +113,51 @@ func LoadConfigFromPath(path string) (*Config, error) { config.Secret = "" config.secret = secret + config.blossomS3SecretKey = strings.TrimSpace(config.Blossom.S3.SecretKey) + config.Blossom.S3.SecretKey = "" + + if err := validateBlossomFileStorage(&config); err != nil { + return nil, err + } + return &config, nil } +func validateBlossomFileStorage(c *Config) error { + if !c.Blossom.Enabled { + return nil + } + fs := strings.ToLower(strings.TrimSpace(c.Blossom.FileStorage)) + if fs == "" || fs == "local" { + return nil + } + if fs != "s3" { + return fmt.Errorf(`blossom.file_storage must be "local", "s3", or empty (defaults to local)`) + } + s := c.Blossom.S3 + if strings.TrimSpace(s.Bucket) == "" { + return fmt.Errorf("blossom.s3.bucket is required when blossom.file_storage is s3") + } + if strings.TrimSpace(s.Region) == "" { + return fmt.Errorf("blossom.s3.region is required when blossom.file_storage is s3") + } + if strings.TrimSpace(s.AccessKey) == "" { + return fmt.Errorf("blossom.s3.access_key is required when blossom.file_storage is s3") + } + secret := strings.TrimSpace(c.blossomS3SecretKey) + if secret == "" { + secret = strings.TrimSpace(c.Blossom.S3.SecretKey) + } + if secret == "" { + return fmt.Errorf("blossom.s3.secret_key is required when blossom.file_storage is s3") + } + return nil +} + func (config *Config) Save() error { // Restore the secret key to the public field for saving config.Secret = config.secret.Hex() + config.Blossom.S3.SecretKey = config.blossomS3SecretKey file, err := os.Create(config.path) if err != nil { @@ -117,6 +172,7 @@ func (config *Config) Save() error { // Clear the secret again config.Secret = "" + config.Blossom.S3.SecretKey = "" return nil } diff --git a/zooid/config_test.go b/zooid/config_test.go index 45a8dd2..4b93360 100644 --- a/zooid/config_test.go +++ b/zooid/config_test.go @@ -1,6 +1,8 @@ package zooid import ( + "os" + "path/filepath" "testing" "fiatjaf.com/nostr" @@ -154,3 +156,90 @@ func TestConfig_MemberRole(t *testing.T) { t.Error("Any pubkey should have member role permissions") } } + +func TestValidateBlossomFileStorage(t *testing.T) { + t.Run("blossom disabled skips validation", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = false + c.Blossom.FileStorage = "s3" + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + + t.Run("local storage needs no s3 fields", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.FileStorage = "local" + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + + t.Run("s3 requires bucket region keys and secret", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.FileStorage = "s3" + c.Blossom.S3.Region = "us-east-1" + if err := validateBlossomFileStorage(c); err == nil { + t.Fatal("expected error for missing bucket and credentials") + } + + c.Blossom.S3.Bucket = "b" + c.Blossom.S3.AccessKey = "k" + c.Blossom.S3.SecretKey = "s" + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil with all s3 fields set, got %v", err) + } + }) + + t.Run("invalid file_storage value", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.FileStorage = "nfs" + if err := validateBlossomFileStorage(c); err == nil { + t.Fatal("expected error for unknown file_storage") + } + }) +} + +func TestLoadConfigFromPath_BlossomS3SecretRedacted(t *testing.T) { + sk := nostr.Generate() + tmp := t.TempDir() + path := filepath.Join(tmp, "relay.toml") + tomlBody := `host = "r.example.com" +schema = "myrelay" +secret = "` + sk.Hex() + `" +inactive = false + +[info] +name = "n" +pubkey = "` + sk.Public().Hex() + `" + +[blossom] +enabled = true +file_storage = "s3" + +[blossom.s3] +region = "auto" +bucket = "test-bucket" +access_key = "AKIA" +secret_key = "topsecret" +endpoint = "http://127.0.0.1:9000" +use_path_style = true +` + if err := os.WriteFile(path, []byte(tomlBody), 0644); err != nil { + t.Fatal(err) + } + + cfg, err := LoadConfigFromPath(path) + if err != nil { + t.Fatalf("LoadConfigFromPath: %v", err) + } + if cfg.Blossom.S3.SecretKey != "" { + t.Error("expected blossom s3 secret_key cleared from loaded struct") + } + if cfg.blossomS3SecretKey != "topsecret" { + t.Errorf("expected private s3 secret retained, got %q", cfg.blossomS3SecretKey) + } +}