4efc87559a
Backblaze B2's S3-compatible endpoint only implements POST Object on virtual-hosted-style URLs (https://<bucket>.s3.<region>.backblazeb2.com/). Path-style POST returns HTTP 501 Not Implemented. minio-go's BucketLookupAuto only flips to virtual-hosted for AWS, Google, and Aliyun endpoints — for B2 it falls through to path-style, which is why every PresignedPostPolicy() call has been handing the mobile clients a URL that B2 then refuses with 501. Force BucketLookupDNS only when the endpoint is backblazeb2.com so MinIO dev (no DNS for arbitrary buckets at minio:9000) keeps its path-style default. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
195 lines
6.1 KiB
Go
195 lines
6.1 KiB
Go
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// S3Backend stores files in S3-compatible storage (Backblaze B2, MinIO, AWS S3).
|
|
type S3Backend struct {
|
|
client *minio.Client
|
|
bucket string
|
|
}
|
|
|
|
// NewS3Backend creates an S3-compatible storage backend.
|
|
func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region string) (*S3Backend, error) {
|
|
if region == "" {
|
|
region = "us-east-1"
|
|
}
|
|
|
|
opts := &minio.Options{
|
|
Creds: credentials.NewStaticV4(keyID, appKey, ""),
|
|
Secure: useSSL,
|
|
Region: region,
|
|
}
|
|
// B2's S3-compatible endpoint only implements POST Object on
|
|
// virtual-hosted-style URLs (https://<bucket>.s3.<region>.backblazeb2.com/).
|
|
// Path-style POST returns HTTP 501. minio-go's auto-detection falls back
|
|
// to path-style for non-AWS endpoints, which breaks PresignedPostPolicy
|
|
// against B2. Force DNS lookup for B2 only — leave MinIO dev (no DNS for
|
|
// arbitrary buckets at minio:9000) on the path-style default.
|
|
if strings.Contains(endpoint, "backblazeb2.com") {
|
|
opts.BucketLookup = minio.BucketLookupDNS
|
|
}
|
|
|
|
client, err := minio.New(endpoint, opts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create S3 client: %w", err)
|
|
}
|
|
|
|
// Verify bucket exists
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
exists, err := client.BucketExists(ctx, bucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to check bucket %q: %w", bucket, err)
|
|
}
|
|
if !exists {
|
|
return nil, fmt.Errorf("bucket %q does not exist", bucket)
|
|
}
|
|
|
|
log.Info().
|
|
Str("endpoint", endpoint).
|
|
Str("bucket", bucket).
|
|
Bool("ssl", useSSL).
|
|
Msg("S3 storage backend initialized")
|
|
|
|
return &S3Backend{client: client, bucket: bucket}, nil
|
|
}
|
|
|
|
func (b *S3Backend) Write(key string, data []byte) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := b.client.PutObject(ctx, b.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to upload to S3: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *S3Backend) Read(key string) ([]byte, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get S3 object: %w", err)
|
|
}
|
|
defer obj.Close()
|
|
|
|
data, err := io.ReadAll(obj)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read S3 object: %w", err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
func (b *S3Backend) Delete(key string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
err := b.client.RemoveObject(ctx, b.bucket, key, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete S3 object: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
|
|
ctx := context.Background() // caller controls lifetime by closing the reader
|
|
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get S3 object stream: %w", err)
|
|
}
|
|
return obj, nil
|
|
}
|
|
|
|
// PresignedPostResult is the data a client needs to perform a direct multipart
|
|
// POST to S3-compatible storage. The caller assembles a multipart/form-data
|
|
// request with the fields below as form parts (in order) and the file last.
|
|
type PresignedPostResult struct {
|
|
URL string // e.g. https://s3.us-east-005.backblazeb2.com/honeyDueProd
|
|
Fields map[string]string // policy, x-amz-*, key, Content-Type, etc.
|
|
}
|
|
|
|
// PresignedPost generates a POST policy that constrains uploads at the protocol
|
|
// level: only the named key, only the named content-type, only sizes within
|
|
// the requested range. S3 (and B2's S3-compatible endpoint) reject anything
|
|
// that doesn't satisfy every condition before accepting the body.
|
|
//
|
|
// minBytes/maxBytes are inclusive. The returned URL + Fields can be sent
|
|
// straight to the client.
|
|
func (b *S3Backend) PresignedPost(ctx context.Context, key, contentType string, minBytes, maxBytes int64, ttl time.Duration) (*PresignedPostResult, error) {
|
|
policy := minio.NewPostPolicy()
|
|
if err := policy.SetBucket(b.bucket); err != nil {
|
|
return nil, fmt.Errorf("set bucket: %w", err)
|
|
}
|
|
if err := policy.SetKey(key); err != nil {
|
|
return nil, fmt.Errorf("set key: %w", err)
|
|
}
|
|
if err := policy.SetContentType(contentType); err != nil {
|
|
return nil, fmt.Errorf("set content-type: %w", err)
|
|
}
|
|
if err := policy.SetContentLengthRange(minBytes, maxBytes); err != nil {
|
|
return nil, fmt.Errorf("set content-length-range: %w", err)
|
|
}
|
|
if err := policy.SetExpires(time.Now().UTC().Add(ttl)); err != nil {
|
|
return nil, fmt.Errorf("set expires: %w", err)
|
|
}
|
|
|
|
u, fields, err := b.client.PresignedPostPolicy(ctx, policy)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("presign post policy: %w", err)
|
|
}
|
|
return &PresignedPostResult{
|
|
URL: stripQuery(u),
|
|
Fields: fields,
|
|
}, nil
|
|
}
|
|
|
|
// Stat returns object metadata without fetching the body. Used by the attach
|
|
// path to verify the uploaded object's size and content-type match what the
|
|
// client claimed when requesting the presign.
|
|
type ObjectInfo struct {
|
|
Size int64
|
|
ContentType string
|
|
ETag string
|
|
}
|
|
|
|
func (b *S3Backend) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
|
info, err := b.client.StatObject(ctx, b.bucket, key, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("stat S3 object: %w", err)
|
|
}
|
|
return &ObjectInfo{
|
|
Size: info.Size,
|
|
ContentType: info.ContentType,
|
|
ETag: info.ETag,
|
|
}, nil
|
|
}
|
|
|
|
// stripQuery returns the URL with its query string removed. minio-go encodes
|
|
// the policy/signature into both the form fields and the query; the form
|
|
// fields are the source of truth for POST policy uploads, and many clients
|
|
// (including Apple's NSURLSession) will reject the request if the same
|
|
// signature appears in both places.
|
|
func stripQuery(u *url.URL) string {
|
|
if u == nil {
|
|
return ""
|
|
}
|
|
clone := *u
|
|
clone.RawQuery = ""
|
|
return clone.String()
|
|
}
|