userAdityaa/zooid
1
0
Fork 0
forked from coracle/zooid
Code Pull Requests Activity

feat(blossom): optional S3-compatible blob storage

Browse Source
This commit is contained in:
Aditya Chaudhary
2026-05-12 01:22:46 +05:45
parent 9e56d47fc0
commit 732f9b6209
6 changed files with 327 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+18
View File
@@ -6,6 +6,10 @@ require (
fiatjaf.com/nostr v0.0.0-20251104112613-38a6ca92b954
github.com/BurntSushi/toml v1.5.0
github.com/Masterminds/squirrel v1.5.4
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/fsnotify/fsnotify v1.9.0
github.com/gosimple/slug v1.15.0
github.com/livekit/protocol v1.43.5-0.20260114074149-a8bb8204ce69
@@ -22,6 +26,20 @@ require (
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
github.com/aws/smithy-go v1.25.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bep/debounce v1.2.1 // indirect
go.sum
+36
View File
@@ -30,6 +30,42 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
zooid/api.go
+5
View File
@@ -358,6 +358,10 @@ func (api *APIHandler) validateConfig(config *Config) error {
return fmt.Errorf("invalid info.pubkey: %w", err)
}
}
normalizeBlossomConfig(config)
if err := validateBlossomFileStorage(config); err != nil {
return err
}
return nil
}
@@ -406,6 +410,7 @@ func (api *APIHandler) loadConfigFromPath(path string) (*Config, error) {
if err != nil {
return nil, err
}
normalizeBlossomConfig(&config)
return &config, nil
}
zooid/blossom.go
+112 -14
View File
@@ -3,12 +3,19 @@ package zooid
import (
"bytes"
"context"
"fmt"
"io"
"log"
"net/url"
"path/filepath"
"fiatjaf.com/nostr"
"fiatjaf.com/nostr/eventstore"
"fiatjaf.com/nostr/khatru/blossom"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gosimple/slug"
"github.com/spf13/afero"
)
@@ -18,19 +25,39 @@ type BlossomStore struct {
Events eventstore.Store
}
func (bl *BlossomStore) Enable(instance *Instance) {
dir := Env("MEDIA") + "/" + slug.Make(bl.Config.Schema)
fs := afero.NewOsFs()
fs.MkdirAll(dir, 0755)
backend := blossom.New(instance.Relay, "https://"+bl.Config.Host)
func loadAWSConfigForBlossomS3(ctx context.Context, s *BlossomS3Settings) (aws.Config, error) {
return awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(s.Region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.AccessKey, s.SecretKey, "")),
)
}
backend.Store = blossom.EventStoreBlobIndexWrapper{
Store: bl.Events,
ServiceURL: "https://" + bl.Config.Host,
func s3APIClientForBlossomSettings(awsCfg aws.Config, s *BlossomS3Settings) *s3.Client {
customEndpoint := s.Endpoint != ""
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if customEndpoint {
o.BaseEndpoint = aws.String(s.Endpoint)
// Custom endpoints (e.g. MinIO) expect path-style addressing.
o.UsePathStyle = true
}
})
}
func blossomS3ObjectKey(slugName, sha256, keyPrefix string) string {
rel := slugName + "/" + sha256
if keyPrefix != "" {
return keyPrefix + "/" + rel
}
return rel
}
backend.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error {
file, err := fs.Create(dir + "/" + sha256)
func attachBlossomLocalBlobs(bs *blossom.BlossomServer, slugName string) {
dir := filepath.Join(Env("MEDIA"), slugName)
osfs := afero.NewOsFs()
_ = osfs.MkdirAll(dir, 0755)
bs.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error {
file, err := osfs.Create(filepath.Join(dir, sha256))
if err != nil {
return err
}
@@ -42,16 +69,87 @@ func (bl *BlossomStore) Enable(instance *Instance) {
return nil
}
backend.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) {
file, err := fs.Open(dir + "/" + sha256)
bs.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) {
file, err := osfs.Open(filepath.Join(dir, sha256))
if err != nil {
return nil, nil, err
}
return file, nil, nil
}
backend.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error {
return fs.Remove(dir + "/" + sha256)
bs.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error {
return osfs.Remove(filepath.Join(dir, sha256))
}
}
func attachBlossomS3Blobs(bs *blossom.BlossomServer, cfg *Config, slugName string) error {
s := &cfg.Blossom.S3
ctx := context.Background()
awsCfg, err := loadAWSConfigForBlossomS3(ctx, s)
if err != nil {
return fmt.Errorf("aws config: %w", err)
}
client := s3APIClientForBlossomSettings(awsCfg, s)
bucket := s.Bucket
bs.StoreBlob = func(ctx context.Context, sha256 string, ext string, body []byte) error {
_, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)),
Body: bytes.NewReader(body),
})
return err
}
bs.LoadBlob = func(ctx context.Context, sha256 string, ext string) (io.ReadSeeker, *url.URL, error) {
out, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)),
})
if err != nil {
return nil, nil, err
}
defer out.Body.Close()
data, err := io.ReadAll(out.Body)
if err != nil {
return nil, nil, err
}
return bytes.NewReader(data), nil, nil
}
bs.DeleteBlob = func(ctx context.Context, sha256 string, ext string) error {
_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(blossomS3ObjectKey(slugName, sha256, s.KeyPrefix)),
})
return err
}
return nil
}
func (bl *BlossomStore) Enable(instance *Instance) {
slugName := slug.Make(bl.Config.Schema)
backend := blossom.New(instance.Relay, "https://"+bl.Config.Host)
backend.Store = blossom.EventStoreBlobIndexWrapper{
Store: bl.Events,
ServiceURL: "https://" + bl.Config.Host,
}
switch bl.Config.Blossom.Backend {
case "local":
attachBlossomLocalBlobs(backend, slugName)
case "s3":
if err := attachBlossomS3Blobs(backend, bl.Config, slugName); err != nil {
log.Fatalf("blossom: s3: %v", err)
}
default:
log.Fatalf("blossom: unknown backend %q (use local or s3)", bl.Config.Blossom.Backend)
}
backend.RejectUpload = func(ctx context.Context, auth *nostr.Event, size int, ext string) (bool, string, int) {
zooid/config.go
+63
View File
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"slices"
"strings"
)
type Role struct {
@@ -47,6 +48,8 @@ type Config struct {
Blossom struct {
Enabled bool `toml:"enabled" json:"enabled"`
AuthenticatedRead bool `toml:"authenticated_read" json:"authenticated_read"`
Backend string `toml:"backend" json:"backend"`
S3 BlossomS3Settings `toml:"s3" json:"s3"`
} `toml:"blossom" json:"blossom"`
Livekit struct {
@@ -62,6 +65,17 @@ type Config struct {
secret nostr.SecretKey
}
// BlossomS3Settings configures S3-compatible object storage for Blossom blobs
// when [blossom] backend is "s3".
type BlossomS3Settings struct {
Endpoint string `toml:"endpoint" json:"endpoint"`
Region string `toml:"region" json:"region"`
Bucket string `toml:"bucket" json:"bucket"`
AccessKey string `toml:"access_key" json:"access_key"`
SecretKey string `toml:"secret_key" json:"secret_key"`
KeyPrefix string `toml:"key_prefix" json:"key_prefix"`
}
func LoadConfig(filename string) (*Config, error) {
path := filepath.Join(Env("CONFIG"), filename)
@@ -74,6 +88,8 @@ func LoadConfigFromPath(path string) (*Config, error) {
return nil, fmt.Errorf("Failed to parse config file %s: %w", path, err)
}
normalizeBlossomConfig(&config)
if config.Host == "" {
return nil, fmt.Errorf("host is required")
}
@@ -98,9 +114,56 @@ func LoadConfigFromPath(path string) (*Config, error) {
config.Secret = ""
config.secret = secret
if err := validateBlossomFileStorage(&config); err != nil {
return nil, err
}
return &config, nil
}
func normalizeBlossomConfig(c *Config) {
s := &c.Blossom.S3
s.Region = strings.TrimSpace(s.Region)
s.Bucket = strings.TrimSpace(s.Bucket)
s.AccessKey = strings.TrimSpace(s.AccessKey)
s.SecretKey = strings.TrimSpace(s.SecretKey)
s.Endpoint = strings.TrimRight(strings.TrimSpace(s.Endpoint), "/")
s.KeyPrefix = strings.Trim(strings.TrimSpace(s.KeyPrefix), "/")
c.Blossom.Backend = strings.ToLower(strings.TrimSpace(c.Blossom.Backend))
if c.Blossom.Backend == "" {
c.Blossom.Backend = "local"
}
}
func validateBlossomFileStorage(c *Config) error {
if !c.Blossom.Enabled {
return nil
}
switch c.Blossom.Backend {
case "local":
return nil
case "s3":
// fall through
default:
return fmt.Errorf(`blossom.backend must be "local", "s3", or empty (defaults to local)`)
}
s := c.Blossom.S3
if s.Bucket == "" {
return fmt.Errorf("blossom.s3.bucket is required when blossom.backend is s3")
}
if s.Region == "" {
return fmt.Errorf("blossom.s3.region is required when blossom.backend is s3")
}
if s.AccessKey == "" {
return fmt.Errorf("blossom.s3.access_key is required when blossom.backend is s3")
}
if s.SecretKey == "" {
return fmt.Errorf("blossom.s3.secret_key is required when blossom.backend is s3")
}
return nil
}
func (config *Config) Save() error {
// Restore the secret key to the public field for saving
config.Secret = config.secret.Hex()
zooid/config_test.go
+93
View File
@@ -1,6 +1,8 @@
package zooid
import (
"os"
"path/filepath"
"testing"
"fiatjaf.com/nostr"
@@ -154,3 +156,94 @@ func TestConfig_MemberRole(t *testing.T) {
t.Error("Any pubkey should have member role permissions")
}
}
func TestValidateBlossomFileStorage(t *testing.T) {
t.Run("blossom disabled skips validation", func(t *testing.T) {
c := &Config{}
c.Blossom.Enabled = false
c.Blossom.Backend = "s3"
normalizeBlossomConfig(c)
if err := validateBlossomFileStorage(c); err != nil {
t.Fatalf("expected nil, got %v", err)
}
})
t.Run("local storage needs no s3 fields", func(t *testing.T) {
c := &Config{}
c.Blossom.Enabled = true
c.Blossom.Backend = "local"
normalizeBlossomConfig(c)
if err := validateBlossomFileStorage(c); err != nil {
t.Fatalf("expected nil, got %v", err)
}
})
t.Run("s3 requires bucket region keys and secret", func(t *testing.T) {
c := &Config{}
c.Blossom.Enabled = true
c.Blossom.Backend = "s3"
c.Blossom.S3.Region = "us-east-1"
normalizeBlossomConfig(c)
if err := validateBlossomFileStorage(c); err == nil {
t.Fatal("expected error for missing bucket and credentials")
}
c.Blossom.S3.Bucket = "b"
c.Blossom.S3.AccessKey = "k"
c.Blossom.S3.SecretKey = "s"
normalizeBlossomConfig(c)
if err := validateBlossomFileStorage(c); err != nil {
t.Fatalf("expected nil with all s3 fields set, got %v", err)
}
})
t.Run("invalid backend value", func(t *testing.T) {
c := &Config{}
c.Blossom.Enabled = true
c.Blossom.Backend = "nfs"
normalizeBlossomConfig(c)
if err := validateBlossomFileStorage(c); err == nil {
t.Fatal("expected error for unknown backend")
}
})
}
func TestLoadConfigFromPath_BlossomS3(t *testing.T) {
sk := nostr.Generate()
tmp := t.TempDir()
path := filepath.Join(tmp, "relay.toml")
tomlBody := `host = "r.example.com"
schema = "myrelay"
secret = "` + sk.Hex() + `"
inactive = false
[info]
name = "n"
pubkey = "` + sk.Public().Hex() + `"
[blossom]
enabled = true
backend = "s3"
[blossom.s3]
region = "auto"
bucket = "test-bucket"
access_key = "AKIA"
secret_key = "topsecret"
endpoint = "http://127.0.0.1:9000"
`
if err := os.WriteFile(path, []byte(tomlBody), 0644); err != nil {
t.Fatal(err)
}
cfg, err := LoadConfigFromPath(path)
if err != nil {
t.Fatalf("LoadConfigFromPath: %v", err)
}
if cfg.Blossom.S3.SecretKey != "topsecret" {
t.Errorf("expected s3 secret_key retained in struct, got %q", cfg.Blossom.S3.SecretKey)
}
if cfg.Blossom.Backend != "s3" {
t.Errorf("backend: got %q", cfg.Blossom.Backend)
}
}