fix(uploads): switch from S3 POST policy to presigned PUT
Backend CI / Test (push) Has been cancelled
Backend CI / Contract Tests (push) Has been cancelled
Backend CI / Lint (push) Has been cancelled
Backend CI / Secret Scanning (push) Has been cancelled
Backend CI / Build (push) Has been cancelled

Backblaze B2's S3-compatible endpoint does not implement the S3 POST
Object operation. It returns HTTP 501 to every POST regardless of URL
style — both path-style (https://s3.<region>.backblazeb2.com/<bucket>/)
and virtual-hosted-style (https://<bucket>.s3.<region>.backblazeb2.com/).

Yesterday's BucketLookupDNS fix produced virtual-hosted URLs, which is
correct for AWS but doesn't help here — B2 rejects POST on either form.
Verified with `curl -X POST https://...backblazeb2.com/honeyDueProd/`
returning 501 directly, with no signature involved.

Replace minio-go's PresignedPostPolicy with PresignHeader + http.MethodPut.
The signed URL now points at a single PUT endpoint, with Content-Type
and Content-Length signed via headers — B2/S3/MinIO all accept it. Drop
the min/max content-length range (we sign exactly one length now);
post-upload size verification still happens in VerifyAndClaim via HEAD.

Response shape:
  - URL  (was: signed POST endpoint)  → now: signed PUT URL
  - Fields → renamed to Headers; client sends them as request headers,
    not multipart form parts
  - Method (new): always "PUT", emitted explicitly so clients don't
    have to hardcode

Companion KMP/iOS commits switch the client paths from multipart POST
to single PUT. Existing builds in the field will need to be rebuilt.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey t
2026-05-06 15:41:48 -05:00
parent 5d8559b495
commit 7cc5448a7c
3 changed files with 60 additions and 67 deletions
+19 -9
View File
@@ -2,21 +2,31 @@ package responses
// PresignUploadResponse is what /api/uploads/presign returns to the client.
//
// The client uses URL + Fields to build a multipart/form-data POST directly
// to S3-compatible storage (B2). Once the upload completes, the client calls
// the relevant entity-creation endpoint (POST /api/task-completions/, POST
// /api/documents/) with `upload_ids: [Id]` to claim and attach the object.
// Flow: the client makes one PUT request to URL with the raw object bytes
// as the body and Headers as the request headers (verbatim — the signature
// binds them). On success, the client passes ID back via upload_ids[] on
// POST /api/task-completions/ or POST /api/documents/ to claim and attach
// the object.
//
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
// not implement the S3 POST Object form upload — it returns HTTP 501 on
// every request style. PUT works against AWS S3, B2, and MinIO uniformly.
type PresignUploadResponse struct {
// ID is the pending_uploads.id the client passes back via upload_ids[].
ID uint `json:"id"`
// URL is the storage endpoint to POST to (no query string).
// URL is the signed PUT URL. Includes all auth as query parameters.
URL string `json:"upload_url"`
// Fields are the form fields (policy, signature, key, etc.) that must be
// submitted with the multipart form. The file part must be named "file"
// and come last per S3 POST policy rules.
Fields map[string]string `json:"fields"`
// Method is always "PUT" — emitted explicitly so clients don't have to
// hardcode it. Reserved for the rare case we ever offer alternative
// upload mechanisms.
Method string `json:"method"`
// Headers must be sent verbatim on the PUT request. Currently includes
// Content-Type and Content-Length; both are signed, and B2 will reject
// any PUT whose headers don't match.
Headers map[string]string `json:"headers"`
// Key is the object key chosen by the server. Echoed for client logging
// and debugging; the canonical reference is via ID.
+33 -45
View File
@@ -5,8 +5,9 @@ import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"strconv"
"time"
"github.com/minio/minio-go/v7"
@@ -31,15 +32,6 @@ func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region st
Secure: useSSL,
Region: region,
}
// B2's S3-compatible endpoint only implements POST Object on
// virtual-hosted-style URLs (https://<bucket>.s3.<region>.backblazeb2.com/).
// Path-style POST returns HTTP 501. minio-go's auto-detection falls back
// to path-style for non-AWS endpoints, which breaks PresignedPostPolicy
// against B2. Force DNS lookup for B2 only — leave MinIO dev (no DNS for
// arbitrary buckets at minio:9000) on the path-style default.
if strings.Contains(endpoint, "backblazeb2.com") {
opts.BucketLookup = minio.BucketLookupDNS
}
client, err := minio.New(endpoint, opts)
if err != nil {
@@ -115,46 +107,42 @@ func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
return obj, nil
}
// PresignedPostResult is the data a client needs to perform a direct multipart
// POST to S3-compatible storage. The caller assembles a multipart/form-data
// request with the fields below as form parts (in order) and the file last.
type PresignedPostResult struct {
URL string // e.g. https://s3.us-east-005.backblazeb2.com/honeyDueProd
Fields map[string]string // policy, x-amz-*, key, Content-Type, etc.
}
// PresignedPost generates a POST policy that constrains uploads at the protocol
// level: only the named key, only the named content-type, only sizes within
// the requested range. S3 (and B2's S3-compatible endpoint) reject anything
// that doesn't satisfy every condition before accepting the body.
// PresignedPutResult is the data a client needs to upload a single object
// to S3-compatible storage via a single PUT request. The client sends the
// raw object bytes as the request body and includes Headers verbatim — the
// signature binds these headers, so any deviation invalidates it.
//
// minBytes/maxBytes are inclusive. The returned URL + Fields can be sent
// straight to the client.
func (b *S3Backend) PresignedPost(ctx context.Context, key, contentType string, minBytes, maxBytes int64, ttl time.Duration) (*PresignedPostResult, error) {
policy := minio.NewPostPolicy()
if err := policy.SetBucket(b.bucket); err != nil {
return nil, fmt.Errorf("set bucket: %w", err)
}
if err := policy.SetKey(key); err != nil {
return nil, fmt.Errorf("set key: %w", err)
}
if err := policy.SetContentType(contentType); err != nil {
return nil, fmt.Errorf("set content-type: %w", err)
}
if err := policy.SetContentLengthRange(minBytes, maxBytes); err != nil {
return nil, fmt.Errorf("set content-length-range: %w", err)
}
if err := policy.SetExpires(time.Now().UTC().Add(ttl)); err != nil {
return nil, fmt.Errorf("set expires: %w", err)
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
// not implement the S3 POST Object operation (returns HTTP 501 on both
// path-style and virtual-hosted-style URLs). PUT works against AWS S3, B2,
// and MinIO uniformly.
type PresignedPutResult struct {
URL string // signed PUT URL with query-string credentials
Headers map[string]string // headers the client must send on the PUT
}
u, fields, err := b.client.PresignedPostPolicy(ctx, policy)
// PresignedPut generates a single-use PUT URL that uploads exactly the
// claimed `contentLength` bytes with the claimed `contentType`. The size
// and content-type are signed via headers — B2/S3 reject anything that
// doesn't match.
//
// Post-upload verification is the caller's job (see UploadService.VerifyAndClaim
// which HEADs the object to confirm size + type before claiming).
func (b *S3Backend) PresignedPut(ctx context.Context, key, contentType string, contentLength int64, ttl time.Duration) (*PresignedPutResult, error) {
extra := http.Header{}
extra.Set("Content-Type", contentType)
extra.Set("Content-Length", strconv.FormatInt(contentLength, 10))
u, err := b.client.PresignHeader(ctx, http.MethodPut, b.bucket, key, ttl, nil, extra)
if err != nil {
return nil, fmt.Errorf("presign post policy: %w", err)
return nil, fmt.Errorf("presign put: %w", err)
}
return &PresignedPostResult{
URL: stripQuery(u),
Fields: fields,
return &PresignedPutResult{
URL: u.String(),
Headers: map[string]string{
"Content-Type": contentType,
"Content-Length": strconv.FormatInt(contentLength, 10),
},
}, nil
}
+9 -14
View File
@@ -164,18 +164,12 @@ func (s *UploadService) Presign(
ext := extensionForContentType(contentType)
key := fmt.Sprintf("uploads/%s/%d/%s%s", subdir, userID, uuid.New().String(), ext)
// Sign the POST policy. The slack window lets the client encode once;
// the server still rejects anything materially different from claimed.
minB := contentLength - UploadPresignSlackBytes
if minB < 0 {
minB = 0
}
maxB := contentLength + UploadPresignSlackBytes
if maxB > UploadMaxBytes {
maxB = UploadMaxBytes
}
post, err := s.s3.PresignedPost(ctx, key, contentType, minB, maxB, UploadPresignTTL)
// Sign a single-use PUT URL that binds Content-Type and Content-Length.
// B2 does not implement S3's POST Object form upload (returns 501), so
// we use PUT — which works uniformly against AWS S3, B2, and MinIO.
// Size mismatch is caught later in VerifyAndClaim via HEAD; we don't
// need a min/max range here because the signature pins exactly one size.
put, err := s.s3.PresignedPut(ctx, key, contentType, contentLength, UploadPresignTTL)
if err != nil {
return nil, apperrors.Internal(fmt.Errorf("presign upload: %w", err))
}
@@ -195,8 +189,9 @@ func (s *UploadService) Presign(
return &responses.PresignUploadResponse{
ID: row.ID,
URL: post.URL,
Fields: post.Fields,
URL: put.URL,
Headers: put.Headers,
Method: "PUT",
Key: key,
ExpiresAt: expiresAt.Format(time.RFC3339),
}, nil