Files
honeyDueAPI/internal/services/storage_backend_s3.go
T
Trey t 7cc5448a7c
Backend CI / Test (push) Has been cancelled
Backend CI / Contract Tests (push) Has been cancelled
Backend CI / Lint (push) Has been cancelled
Backend CI / Secret Scanning (push) Has been cancelled
Backend CI / Build (push) Has been cancelled
fix(uploads): switch from S3 POST policy to presigned PUT
Backblaze B2's S3-compatible endpoint does not implement the S3 POST
Object operation. It returns HTTP 501 to every POST regardless of URL
style — both path-style (https://s3.<region>.backblazeb2.com/<bucket>/)
and virtual-hosted-style (https://<bucket>.s3.<region>.backblazeb2.com/).

Yesterday's BucketLookupDNS fix produced virtual-hosted URLs, which is
correct for AWS but doesn't help here — B2 rejects POST on either form.
Verified with `curl -X POST https://...backblazeb2.com/honeyDueProd/`
returning 501 directly, with no signature involved.

Replace minio-go's PresignedPostPolicy with PresignHeader + http.MethodPut.
The signed URL now points at a single PUT endpoint, with Content-Type
and Content-Length signed via headers — B2/S3/MinIO all accept it. Drop
the min/max content-length range (we sign exactly one length now);
post-upload size verification still happens in VerifyAndClaim via HEAD.

Response shape:
  - URL  (was: signed POST endpoint)  → now: signed PUT URL
  - Fields → renamed to Headers; client sends them as request headers,
    not multipart form parts
  - Method (new): always "PUT", emitted explicitly so clients don't
    have to hardcode

Companion KMP/iOS commits switch the client paths from multipart POST
to single PUT. Existing builds in the field will need to be rebuilt.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-06 15:41:48 -05:00

183 lines
5.5 KiB
Go

package services
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/rs/zerolog/log"
)
// S3Backend stores files in S3-compatible storage (Backblaze B2, MinIO, AWS S3).
type S3Backend struct {
client *minio.Client
bucket string
}
// NewS3Backend creates an S3-compatible storage backend.
func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region string) (*S3Backend, error) {
if region == "" {
region = "us-east-1"
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(keyID, appKey, ""),
Secure: useSSL,
Region: region,
}
client, err := minio.New(endpoint, opts)
if err != nil {
return nil, fmt.Errorf("failed to create S3 client: %w", err)
}
// Verify bucket exists
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
exists, err := client.BucketExists(ctx, bucket)
if err != nil {
return nil, fmt.Errorf("failed to check bucket %q: %w", bucket, err)
}
if !exists {
return nil, fmt.Errorf("bucket %q does not exist", bucket)
}
log.Info().
Str("endpoint", endpoint).
Str("bucket", bucket).
Bool("ssl", useSSL).
Msg("S3 storage backend initialized")
return &S3Backend{client: client, bucket: bucket}, nil
}
func (b *S3Backend) Write(key string, data []byte) error {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
_, err := b.client.PutObject(ctx, b.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
if err != nil {
return fmt.Errorf("failed to upload to S3: %w", err)
}
return nil
}
func (b *S3Backend) Read(key string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get S3 object: %w", err)
}
defer obj.Close()
data, err := io.ReadAll(obj)
if err != nil {
return nil, fmt.Errorf("failed to read S3 object: %w", err)
}
return data, nil
}
func (b *S3Backend) Delete(key string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
err := b.client.RemoveObject(ctx, b.bucket, key, minio.RemoveObjectOptions{})
if err != nil {
return fmt.Errorf("failed to delete S3 object: %w", err)
}
return nil
}
func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
ctx := context.Background() // caller controls lifetime by closing the reader
obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get S3 object stream: %w", err)
}
return obj, nil
}
// PresignedPutResult is the data a client needs to upload a single object
// to S3-compatible storage via a single PUT request. The client sends the
// raw object bytes as the request body and includes Headers verbatim — the
// signature binds these headers, so any deviation invalidates it.
//
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
// not implement the S3 POST Object operation (returns HTTP 501 on both
// path-style and virtual-hosted-style URLs). PUT works against AWS S3, B2,
// and MinIO uniformly.
type PresignedPutResult struct {
URL string // signed PUT URL with query-string credentials
Headers map[string]string // headers the client must send on the PUT
}
// PresignedPut generates a single-use PUT URL that uploads exactly the
// claimed `contentLength` bytes with the claimed `contentType`. The size
// and content-type are signed via headers — B2/S3 reject anything that
// doesn't match.
//
// Post-upload verification is the caller's job (see UploadService.VerifyAndClaim
// which HEADs the object to confirm size + type before claiming).
func (b *S3Backend) PresignedPut(ctx context.Context, key, contentType string, contentLength int64, ttl time.Duration) (*PresignedPutResult, error) {
extra := http.Header{}
extra.Set("Content-Type", contentType)
extra.Set("Content-Length", strconv.FormatInt(contentLength, 10))
u, err := b.client.PresignHeader(ctx, http.MethodPut, b.bucket, key, ttl, nil, extra)
if err != nil {
return nil, fmt.Errorf("presign put: %w", err)
}
return &PresignedPutResult{
URL: u.String(),
Headers: map[string]string{
"Content-Type": contentType,
"Content-Length": strconv.FormatInt(contentLength, 10),
},
}, nil
}
// Stat returns object metadata without fetching the body. Used by the attach
// path to verify the uploaded object's size and content-type match what the
// client claimed when requesting the presign.
type ObjectInfo struct {
Size int64
ContentType string
ETag string
}
func (b *S3Backend) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
info, err := b.client.StatObject(ctx, b.bucket, key, minio.StatObjectOptions{})
if err != nil {
return nil, fmt.Errorf("stat S3 object: %w", err)
}
return &ObjectInfo{
Size: info.Size,
ContentType: info.ContentType,
ETag: info.ETag,
}, nil
}
// stripQuery returns the URL with its query string removed. minio-go encodes
// the policy/signature into both the form fields and the query; the form
// fields are the source of truth for POST policy uploads, and many clients
// (including Apple's NSURLSession) will reject the request if the same
// signature appears in both places.
func stripQuery(u *url.URL) string {
if u == nil {
return ""
}
clone := *u
clone.RawQuery = ""
return clone.String()
}