7cc5448a7c
Backblaze B2's S3-compatible endpoint does not implement the S3 POST Object operation. It returns HTTP 501 to every POST regardless of URL style — both path-style (https://s3.<region>.backblazeb2.com/<bucket>/) and virtual-hosted-style (https://<bucket>.s3.<region>.backblazeb2.com/). Yesterday's BucketLookupDNS fix produced virtual-hosted URLs, which is correct for AWS but doesn't help here — B2 rejects POST on either form. Verified with `curl -X POST https://...backblazeb2.com/honeyDueProd/` returning 501 directly, with no signature involved. Replace minio-go's PresignedPostPolicy with PresignHeader + http.MethodPut. The signed URL now points at a single PUT endpoint, with Content-Type and Content-Length signed via headers — B2/S3/MinIO all accept it. Drop the min/max content-length range (we sign exactly one length now); post-upload size verification still happens in VerifyAndClaim via HEAD. Response shape: - URL (was: signed POST endpoint) → now: signed PUT URL - Fields → renamed to Headers; client sends them as request headers, not multipart form parts - Method (new): always "PUT", emitted explicitly so clients don't have to hardcode Companion KMP/iOS commits switch the client paths from multipart POST to single PUT. Existing builds in the field will need to be rebuilt. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
183 lines
5.5 KiB
Go
183 lines
5.5 KiB
Go
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// S3Backend stores files in S3-compatible storage (Backblaze B2, MinIO, AWS S3).
|
|
type S3Backend struct {
|
|
client *minio.Client
|
|
bucket string
|
|
}
|
|
|
|
// NewS3Backend creates an S3-compatible storage backend.
|
|
func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region string) (*S3Backend, error) {
|
|
if region == "" {
|
|
region = "us-east-1"
|
|
}
|
|
|
|
opts := &minio.Options{
|
|
Creds: credentials.NewStaticV4(keyID, appKey, ""),
|
|
Secure: useSSL,
|
|
Region: region,
|
|
}
|
|
|
|
client, err := minio.New(endpoint, opts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create S3 client: %w", err)
|
|
}
|
|
|
|
// Verify bucket exists
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
exists, err := client.BucketExists(ctx, bucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to check bucket %q: %w", bucket, err)
|
|
}
|
|
if !exists {
|
|
return nil, fmt.Errorf("bucket %q does not exist", bucket)
|
|
}
|
|
|
|
log.Info().
|
|
Str("endpoint", endpoint).
|
|
Str("bucket", bucket).
|
|
Bool("ssl", useSSL).
|
|
Msg("S3 storage backend initialized")
|
|
|
|
return &S3Backend{client: client, bucket: bucket}, nil
|
|
}
|
|
|
|
func (b *S3Backend) Write(key string, data []byte) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := b.client.PutObject(ctx, b.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to upload to S3: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *S3Backend) Read(key string) ([]byte, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get S3 object: %w", err)
|
|
}
|
|
defer obj.Close()
|
|
|
|
data, err := io.ReadAll(obj)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read S3 object: %w", err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
func (b *S3Backend) Delete(key string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
err := b.client.RemoveObject(ctx, b.bucket, key, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete S3 object: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
|
|
ctx := context.Background() // caller controls lifetime by closing the reader
|
|
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get S3 object stream: %w", err)
|
|
}
|
|
return obj, nil
|
|
}
|
|
|
|
// PresignedPutResult is the data a client needs to upload a single object
|
|
// to S3-compatible storage via a single PUT request. The client sends the
|
|
// raw object bytes as the request body and includes Headers verbatim — the
|
|
// signature binds these headers, so any deviation invalidates it.
|
|
//
|
|
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
|
|
// not implement the S3 POST Object operation (returns HTTP 501 on both
|
|
// path-style and virtual-hosted-style URLs). PUT works against AWS S3, B2,
|
|
// and MinIO uniformly.
|
|
type PresignedPutResult struct {
|
|
URL string // signed PUT URL with query-string credentials
|
|
Headers map[string]string // headers the client must send on the PUT
|
|
}
|
|
|
|
// PresignedPut generates a single-use PUT URL that uploads exactly the
|
|
// claimed `contentLength` bytes with the claimed `contentType`. The size
|
|
// and content-type are signed via headers — B2/S3 reject anything that
|
|
// doesn't match.
|
|
//
|
|
// Post-upload verification is the caller's job (see UploadService.VerifyAndClaim
|
|
// which HEADs the object to confirm size + type before claiming).
|
|
func (b *S3Backend) PresignedPut(ctx context.Context, key, contentType string, contentLength int64, ttl time.Duration) (*PresignedPutResult, error) {
|
|
extra := http.Header{}
|
|
extra.Set("Content-Type", contentType)
|
|
extra.Set("Content-Length", strconv.FormatInt(contentLength, 10))
|
|
|
|
u, err := b.client.PresignHeader(ctx, http.MethodPut, b.bucket, key, ttl, nil, extra)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("presign put: %w", err)
|
|
}
|
|
return &PresignedPutResult{
|
|
URL: u.String(),
|
|
Headers: map[string]string{
|
|
"Content-Type": contentType,
|
|
"Content-Length": strconv.FormatInt(contentLength, 10),
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// Stat returns object metadata without fetching the body. Used by the attach
|
|
// path to verify the uploaded object's size and content-type match what the
|
|
// client claimed when requesting the presign.
|
|
type ObjectInfo struct {
|
|
Size int64
|
|
ContentType string
|
|
ETag string
|
|
}
|
|
|
|
func (b *S3Backend) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
|
info, err := b.client.StatObject(ctx, b.bucket, key, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("stat S3 object: %w", err)
|
|
}
|
|
return &ObjectInfo{
|
|
Size: info.Size,
|
|
ContentType: info.ContentType,
|
|
ETag: info.ETag,
|
|
}, nil
|
|
}
|
|
|
|
// stripQuery returns the URL with its query string removed. minio-go encodes
|
|
// the policy/signature into both the form fields and the query; the form
|
|
// fields are the source of truth for POST policy uploads, and many clients
|
|
// (including Apple's NSURLSession) will reject the request if the same
|
|
// signature appears in both places.
|
|
func stripQuery(u *url.URL) string {
|
|
if u == nil {
|
|
return ""
|
|
}
|
|
clone := *u
|
|
clone.RawQuery = ""
|
|
return clone.String()
|
|
}
|