fix(uploads): switch from S3 POST policy to presigned PUT
Backend CI / Test (push) Has been cancelled
Backend CI / Contract Tests (push) Has been cancelled
Backend CI / Lint (push) Has been cancelled
Backend CI / Secret Scanning (push) Has been cancelled
Backend CI / Build (push) Has been cancelled

Backblaze B2's S3-compatible endpoint does not implement the S3 POST
Object operation. It returns HTTP 501 to every POST regardless of URL
style — both path-style (https://s3.<region>.backblazeb2.com/<bucket>/)
and virtual-hosted-style (https://<bucket>.s3.<region>.backblazeb2.com/).

Yesterday's BucketLookupDNS fix produced virtual-hosted URLs, which is
correct for AWS but doesn't help here — B2 rejects POST on either form.
Verified with `curl -X POST https://...backblazeb2.com/honeyDueProd/`
returning 501 directly, with no signature involved.

Replace minio-go's PresignedPostPolicy with PresignHeader + http.MethodPut.
The signed URL now points at a single PUT endpoint, with Content-Type
and Content-Length signed via headers — B2/S3/MinIO all accept it. Drop
the min/max content-length range (we sign exactly one length now);
post-upload size verification still happens in VerifyAndClaim via HEAD.

Response shape:
  - URL  (was: signed POST endpoint)  → now: signed PUT URL
  - Fields → renamed to Headers; client sends them as request headers,
    not multipart form parts
  - Method (new): always "PUT", emitted explicitly so clients don't
    have to hardcode

Companion KMP/iOS commits switch the client paths from multipart POST
to single PUT. Existing builds in the field will need to be rebuilt.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey t
2026-05-06 15:41:48 -05:00
parent 5d8559b495
commit 7cc5448a7c
3 changed files with 60 additions and 67 deletions
+19 -9
View File
@@ -2,21 +2,31 @@ package responses
// PresignUploadResponse is what /api/uploads/presign returns to the client. // PresignUploadResponse is what /api/uploads/presign returns to the client.
// //
// The client uses URL + Fields to build a multipart/form-data POST directly // Flow: the client makes one PUT request to URL with the raw object bytes
// to S3-compatible storage (B2). Once the upload completes, the client calls // as the body and Headers as the request headers (verbatim — the signature
// the relevant entity-creation endpoint (POST /api/task-completions/, POST // binds them). On success, the client passes ID back via upload_ids[] on
// /api/documents/) with `upload_ids: [Id]` to claim and attach the object. // POST /api/task-completions/ or POST /api/documents/ to claim and attach
// the object.
//
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
// not implement the S3 POST Object form upload — it returns HTTP 501 on
// every request style. PUT works against AWS S3, B2, and MinIO uniformly.
type PresignUploadResponse struct { type PresignUploadResponse struct {
// ID is the pending_uploads.id the client passes back via upload_ids[]. // ID is the pending_uploads.id the client passes back via upload_ids[].
ID uint `json:"id"` ID uint `json:"id"`
// URL is the storage endpoint to POST to (no query string). // URL is the signed PUT URL. Includes all auth as query parameters.
URL string `json:"upload_url"` URL string `json:"upload_url"`
// Fields are the form fields (policy, signature, key, etc.) that must be // Method is always "PUT" — emitted explicitly so clients don't have to
// submitted with the multipart form. The file part must be named "file" // hardcode it. Reserved for the rare case we ever offer alternative
// and come last per S3 POST policy rules. // upload mechanisms.
Fields map[string]string `json:"fields"` Method string `json:"method"`
// Headers must be sent verbatim on the PUT request. Currently includes
// Content-Type and Content-Length; both are signed, and B2 will reject
// any PUT whose headers don't match.
Headers map[string]string `json:"headers"`
// Key is the object key chosen by the server. Echoed for client logging // Key is the object key chosen by the server. Echoed for client logging
// and debugging; the canonical reference is via ID. // and debugging; the canonical reference is via ID.
+33 -45
View File
@@ -5,8 +5,9 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"net/http"
"net/url" "net/url"
"strings" "strconv"
"time" "time"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
@@ -31,15 +32,6 @@ func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region st
Secure: useSSL, Secure: useSSL,
Region: region, Region: region,
} }
// B2's S3-compatible endpoint only implements POST Object on
// virtual-hosted-style URLs (https://<bucket>.s3.<region>.backblazeb2.com/).
// Path-style POST returns HTTP 501. minio-go's auto-detection falls back
// to path-style for non-AWS endpoints, which breaks PresignedPostPolicy
// against B2. Force DNS lookup for B2 only — leave MinIO dev (no DNS for
// arbitrary buckets at minio:9000) on the path-style default.
if strings.Contains(endpoint, "backblazeb2.com") {
opts.BucketLookup = minio.BucketLookupDNS
}
client, err := minio.New(endpoint, opts) client, err := minio.New(endpoint, opts)
if err != nil { if err != nil {
@@ -115,46 +107,42 @@ func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
return obj, nil return obj, nil
} }
// PresignedPostResult is the data a client needs to perform a direct multipart // PresignedPutResult is the data a client needs to upload a single object
// POST to S3-compatible storage. The caller assembles a multipart/form-data // to S3-compatible storage via a single PUT request. The client sends the
// request with the fields below as form parts (in order) and the file last. // raw object bytes as the request body and includes Headers verbatim — the
type PresignedPostResult struct { // signature binds these headers, so any deviation invalidates it.
URL string // e.g. https://s3.us-east-005.backblazeb2.com/honeyDueProd
Fields map[string]string // policy, x-amz-*, key, Content-Type, etc.
}
// PresignedPost generates a POST policy that constrains uploads at the protocol
// level: only the named key, only the named content-type, only sizes within
// the requested range. S3 (and B2's S3-compatible endpoint) reject anything
// that doesn't satisfy every condition before accepting the body.
// //
// minBytes/maxBytes are inclusive. The returned URL + Fields can be sent // We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
// straight to the client. // not implement the S3 POST Object operation (returns HTTP 501 on both
func (b *S3Backend) PresignedPost(ctx context.Context, key, contentType string, minBytes, maxBytes int64, ttl time.Duration) (*PresignedPostResult, error) { // path-style and virtual-hosted-style URLs). PUT works against AWS S3, B2,
policy := minio.NewPostPolicy() // and MinIO uniformly.
if err := policy.SetBucket(b.bucket); err != nil { type PresignedPutResult struct {
return nil, fmt.Errorf("set bucket: %w", err) URL string // signed PUT URL with query-string credentials
} Headers map[string]string // headers the client must send on the PUT
if err := policy.SetKey(key); err != nil {
return nil, fmt.Errorf("set key: %w", err)
}
if err := policy.SetContentType(contentType); err != nil {
return nil, fmt.Errorf("set content-type: %w", err)
}
if err := policy.SetContentLengthRange(minBytes, maxBytes); err != nil {
return nil, fmt.Errorf("set content-length-range: %w", err)
}
if err := policy.SetExpires(time.Now().UTC().Add(ttl)); err != nil {
return nil, fmt.Errorf("set expires: %w", err)
} }
u, fields, err := b.client.PresignedPostPolicy(ctx, policy) // PresignedPut generates a single-use PUT URL that uploads exactly the
// claimed `contentLength` bytes with the claimed `contentType`. The size
// and content-type are signed via headers — B2/S3 reject anything that
// doesn't match.
//
// Post-upload verification is the caller's job (see UploadService.VerifyAndClaim
// which HEADs the object to confirm size + type before claiming).
func (b *S3Backend) PresignedPut(ctx context.Context, key, contentType string, contentLength int64, ttl time.Duration) (*PresignedPutResult, error) {
extra := http.Header{}
extra.Set("Content-Type", contentType)
extra.Set("Content-Length", strconv.FormatInt(contentLength, 10))
u, err := b.client.PresignHeader(ctx, http.MethodPut, b.bucket, key, ttl, nil, extra)
if err != nil { if err != nil {
return nil, fmt.Errorf("presign post policy: %w", err) return nil, fmt.Errorf("presign put: %w", err)
} }
return &PresignedPostResult{ return &PresignedPutResult{
URL: stripQuery(u), URL: u.String(),
Fields: fields, Headers: map[string]string{
"Content-Type": contentType,
"Content-Length": strconv.FormatInt(contentLength, 10),
},
}, nil }, nil
} }
+9 -14
View File
@@ -164,18 +164,12 @@ func (s *UploadService) Presign(
ext := extensionForContentType(contentType) ext := extensionForContentType(contentType)
key := fmt.Sprintf("uploads/%s/%d/%s%s", subdir, userID, uuid.New().String(), ext) key := fmt.Sprintf("uploads/%s/%d/%s%s", subdir, userID, uuid.New().String(), ext)
// Sign the POST policy. The slack window lets the client encode once; // Sign a single-use PUT URL that binds Content-Type and Content-Length.
// the server still rejects anything materially different from claimed. // B2 does not implement S3's POST Object form upload (returns 501), so
minB := contentLength - UploadPresignSlackBytes // we use PUT — which works uniformly against AWS S3, B2, and MinIO.
if minB < 0 { // Size mismatch is caught later in VerifyAndClaim via HEAD; we don't
minB = 0 // need a min/max range here because the signature pins exactly one size.
} put, err := s.s3.PresignedPut(ctx, key, contentType, contentLength, UploadPresignTTL)
maxB := contentLength + UploadPresignSlackBytes
if maxB > UploadMaxBytes {
maxB = UploadMaxBytes
}
post, err := s.s3.PresignedPost(ctx, key, contentType, minB, maxB, UploadPresignTTL)
if err != nil { if err != nil {
return nil, apperrors.Internal(fmt.Errorf("presign upload: %w", err)) return nil, apperrors.Internal(fmt.Errorf("presign upload: %w", err))
} }
@@ -195,8 +189,9 @@ func (s *UploadService) Presign(
return &responses.PresignUploadResponse{ return &responses.PresignUploadResponse{
ID: row.ID, ID: row.ID,
URL: post.URL, URL: put.URL,
Fields: post.Fields, Headers: put.Headers,
Method: "PUT",
Key: key, Key: key,
ExpiresAt: expiresAt.Format(time.RFC3339), ExpiresAt: expiresAt.Format(time.RFC3339),
}, nil }, nil