diff --git a/go.mod b/go.mod index 8352b60..f2dc4bb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,10 @@ require ( fiatjaf.com/nostr v0.0.0-20251104112613-38a6ca92b954 github.com/BurntSushi/toml v1.5.0 github.com/Masterminds/squirrel v1.5.4 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/fsnotify/fsnotify v1.9.0 github.com/gosimple/slug v1.15.0 github.com/livekit/protocol v1.43.5-0.20260114074149-a8bb8204ce69 @@ -22,6 +26,20 @@ require ( github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect diff --git a/go.sum b/go.sum index 4b6751e..68f3532 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,42 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/zooid/api.go b/zooid/api.go index e82ca45..7db35e8 100644 --- a/zooid/api.go +++ b/zooid/api.go @@ -358,6 +358,10 @@ func (api *APIHandler) validateConfig(config *Config) error { return fmt.Errorf("invalid info.pubkey: %w", err) } } + normalizeBlossomConfig(config) + if err := validateBlossomFileStorage(config); err != nil { + return err + } return nil } @@ -406,6 +410,7 @@ func (api *APIHandler) loadConfigFromPath(path string) (*Config, error) { if err != nil { return nil, err } + normalizeBlossomConfig(&config) return &config, nil } diff --git a/zooid/blossom.go b/zooid/blossom.go index b6f9bed..0ff6791 100644 --- a/zooid/blossom.go +++ b/zooid/blossom.go @@ -3,12 +3,19 @@ package zooid import ( "bytes" "context" + "fmt" "io" + "log" "net/url" + "path/filepath" "fiatjaf.com/nostr" "fiatjaf.com/nostr/eventstore" "fiatjaf.com/nostr/khatru/blossom" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gosimple/slug" "github.com/spf13/afero" ) @@ -18,19 +25,39 @@ type BlossomStore struct { Events eventstore.Store } -func (bl *BlossomStore) Enable(instance *Instance) { - dir := Env("MEDIA") + "/" + slug.Make(bl.Config.Schema) - fs := afero.NewOsFs() - fs.MkdirAll(dir, 0755) - backend := blossom.New(instance.Relay, "https://"+bl.Config.Host) +func loadAWSConfigForBlossomS3(ctx context.Context, s *BlossomS3Settings) (aws.Config, error) { + return awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(s.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.AccessKey, s.SecretKey, "")), + ) +} - backend.Store = blossom.EventStoreBlobIndexWrapper{ - Store: bl.Events, - ServiceURL: "https://" + bl.Config.Host, +func s3APIClientForBlossomSettings(awsCfg aws.Config, s *BlossomS3Settings) *s3.Client { + customEndpoint := s.Endpoint != "" + return s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if customEndpoint { + o.BaseEndpoint = aws.String(s.Endpoint) + // Custom endpoints (e.g. MinIO) expect path-style addressing. + o.UsePathStyle = true + } + }) +} + +func blossomS3ObjectKey(slugName, sha256, keyPrefix string) string { + rel := slugName + "/" + sha256 + if keyPrefix != "" { + return keyPrefix + "/" + rel } + return rel +} - backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { - file, err := fs.Create(dir + "/" + sha256) +func attachBlossomLocalBlobs(bs *blossom.BlossomServer, slugName string) { + dir := filepath.Join(Env("MEDIA"), slugName) + osfs := afero.NewOsFs() + _ = osfs.MkdirAll(dir, 0755) + + bs.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { + file, err := osfs.Create(filepath.Join(dir, sha256)) if err != nil { return err } @@ -42,16 +69,87 @@ func (bl *BlossomStore) Enable(instance *Instance) { return nil } - backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { - file, err := fs.Open(dir + "/" + sha256) + bs.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { + file, err := osfs.Open(filepath.Join(dir, sha256)) if err != nil { return nil, nil, err } return file, nil, nil } - backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { - return fs.Remove(dir + "/" + sha256) + bs.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { + return osfs.Remove(filepath.Join(dir, sha256)) + } +} + +func attachBlossomS3Blobs(bs *blossom.BlossomServer, cfg *Config, slugName string) error { + s := &cfg.Blossom.S3 + ctx := context.Background() + + awsCfg, err := loadAWSConfigForBlossomS3(ctx, s) + if err != nil { + return fmt.Errorf("aws config: %w", err) + } + + client := s3APIClientForBlossomSettings(awsCfg, s) + bucket := s.Bucket + + bs.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error { + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)), + Body: bytes.NewReader(body), + }) + return err + } + + bs.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) { + out, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)), + }) + if err != nil { + return nil, nil, err + } + defer out.Body.Close() + + data, err := io.ReadAll(out.Body) + if err != nil { + return nil, nil, err + } + + return bytes.NewReader(data), nil, nil + } + + bs.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error { + _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)), + }) + return err + } + + return nil +} + +func (bl *BlossomStore) Enable(instance *Instance) { + slugName := slug.Make(bl.Config.Schema) + backend := blossom.New(instance.Relay, "https://"+bl.Config.Host) + + backend.Store = blossom.EventStoreBlobIndexWrapper{ + Store: bl.Events, + ServiceURL: "https://" + bl.Config.Host, + } + + switch bl.Config.Blossom.Backend { + case "local": + attachBlossomLocalBlobs(backend, slugName) + case "s3": + if err := attachBlossomS3Blobs(backend, bl.Config, slugName); err != nil { + log.Fatalf("blossom: s3: %v", err) + } + default: + log.Fatalf("blossom: unknown backend %q (use local or s3)", bl.Config.Blossom.Backend) } backend.RejectUpload = func(ctx context.Context, auth *nostr.Event, size int, ext string) (bool, string, int) { diff --git a/zooid/config.go b/zooid/config.go index b6f2e13..b2d12ba 100644 --- a/zooid/config.go +++ b/zooid/config.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "strings" ) type Role struct { @@ -47,6 +48,8 @@ type Config struct { Blossom struct { Enabled bool `toml:"enabled" json:"enabled"` AuthenticatedRead bool `toml:"authenticated_read" json:"authenticated_read"` + Backend string `toml:"backend" json:"backend"` + S3 BlossomS3Settings `toml:"s3" json:"s3"` } `toml:"blossom" json:"blossom"` Livekit struct { @@ -62,6 +65,17 @@ type Config struct { secret nostr.SecretKey } +// BlossomS3Settings configures S3-compatible object storage for Blossom blobs +// when [blossom] backend is "s3". +type BlossomS3Settings struct { + Endpoint string `toml:"endpoint" json:"endpoint"` + Region string `toml:"region" json:"region"` + Bucket string `toml:"bucket" json:"bucket"` + AccessKey string `toml:"access_key" json:"access_key"` + SecretKey string `toml:"secret_key" json:"secret_key"` + KeyPrefix string `toml:"key_prefix" json:"key_prefix"` +} + func LoadConfig(filename string) (*Config, error) { path := filepath.Join(Env("CONFIG"), filename) @@ -74,6 +88,8 @@ func LoadConfigFromPath(path string) (*Config, error) { return nil, fmt.Errorf("Failed to parse config file %s: %w", path, err) } + normalizeBlossomConfig(&config) + if config.Host == "" { return nil, fmt.Errorf("host is required") } @@ -98,9 +114,56 @@ func LoadConfigFromPath(path string) (*Config, error) { config.Secret = "" config.secret = secret + if err := validateBlossomFileStorage(&config); err != nil { + return nil, err + } + return &config, nil } +func normalizeBlossomConfig(c *Config) { + s := &c.Blossom.S3 + s.Region = strings.TrimSpace(s.Region) + s.Bucket = strings.TrimSpace(s.Bucket) + s.AccessKey = strings.TrimSpace(s.AccessKey) + s.SecretKey = strings.TrimSpace(s.SecretKey) + s.Endpoint = strings.TrimRight(strings.TrimSpace(s.Endpoint), "/") + s.KeyPrefix = strings.Trim(strings.TrimSpace(s.KeyPrefix), "/") + + c.Blossom.Backend = strings.ToLower(strings.TrimSpace(c.Blossom.Backend)) + if c.Blossom.Backend == "" { + c.Blossom.Backend = "local" + } +} + +func validateBlossomFileStorage(c *Config) error { + if !c.Blossom.Enabled { + return nil + } + switch c.Blossom.Backend { + case "local": + return nil + case "s3": + // fall through + default: + return fmt.Errorf(`blossom.backend must be "local", "s3", or empty (defaults to local)`) + } + s := c.Blossom.S3 + if s.Bucket == "" { + return fmt.Errorf("blossom.s3.bucket is required when blossom.backend is s3") + } + if s.Region == "" { + return fmt.Errorf("blossom.s3.region is required when blossom.backend is s3") + } + if s.AccessKey == "" { + return fmt.Errorf("blossom.s3.access_key is required when blossom.backend is s3") + } + if s.SecretKey == "" { + return fmt.Errorf("blossom.s3.secret_key is required when blossom.backend is s3") + } + return nil +} + func (config *Config) Save() error { // Restore the secret key to the public field for saving config.Secret = config.secret.Hex() diff --git a/zooid/config_test.go b/zooid/config_test.go index 45a8dd2..8fa004a 100644 --- a/zooid/config_test.go +++ b/zooid/config_test.go @@ -1,6 +1,8 @@ package zooid import ( + "os" + "path/filepath" "testing" "fiatjaf.com/nostr" @@ -154,3 +156,94 @@ func TestConfig_MemberRole(t *testing.T) { t.Error("Any pubkey should have member role permissions") } } + +func TestValidateBlossomFileStorage(t *testing.T) { + t.Run("blossom disabled skips validation", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = false + c.Blossom.Backend = "s3" + normalizeBlossomConfig(c) + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + + t.Run("local storage needs no s3 fields", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.Backend = "local" + normalizeBlossomConfig(c) + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil, got %v", err) + } + }) + + t.Run("s3 requires bucket region keys and secret", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.Backend = "s3" + c.Blossom.S3.Region = "us-east-1" + normalizeBlossomConfig(c) + if err := validateBlossomFileStorage(c); err == nil { + t.Fatal("expected error for missing bucket and credentials") + } + + c.Blossom.S3.Bucket = "b" + c.Blossom.S3.AccessKey = "k" + c.Blossom.S3.SecretKey = "s" + normalizeBlossomConfig(c) + if err := validateBlossomFileStorage(c); err != nil { + t.Fatalf("expected nil with all s3 fields set, got %v", err) + } + }) + + t.Run("invalid backend value", func(t *testing.T) { + c := &Config{} + c.Blossom.Enabled = true + c.Blossom.Backend = "nfs" + normalizeBlossomConfig(c) + if err := validateBlossomFileStorage(c); err == nil { + t.Fatal("expected error for unknown backend") + } + }) +} + +func TestLoadConfigFromPath_BlossomS3(t *testing.T) { + sk := nostr.Generate() + tmp := t.TempDir() + path := filepath.Join(tmp, "relay.toml") + tomlBody := `host = "r.example.com" +schema = "myrelay" +secret = "` + sk.Hex() + `" +inactive = false + +[info] +name = "n" +pubkey = "` + sk.Public().Hex() + `" + +[blossom] +enabled = true +backend = "s3" + +[blossom.s3] +region = "auto" +bucket = "test-bucket" +access_key = "AKIA" +secret_key = "topsecret" +endpoint = "http://127.0.0.1:9000" +` + if err := os.WriteFile(path, []byte(tomlBody), 0644); err != nil { + t.Fatal(err) + } + + cfg, err := LoadConfigFromPath(path) + if err != nil { + t.Fatalf("LoadConfigFromPath: %v", err) + } + if cfg.Blossom.S3.SecretKey != "topsecret" { + t.Errorf("expected s3 secret_key retained in struct, got %q", cfg.Blossom.S3.SecretKey) + } + if cfg.Blossom.Backend != "s3" { + t.Errorf("backend: got %q", cfg.Blossom.Backend) + } +}