fix(uploads): switch from S3 POST policy to presigned PUT
Backblaze B2's S3-compatible endpoint does not implement the S3 POST Object operation. It returns HTTP 501 to every POST regardless of URL style — both path-style (https://s3.<region>.backblazeb2.com/<bucket>/) and virtual-hosted-style (https://<bucket>.s3.<region>.backblazeb2.com/). Yesterday's BucketLookupDNS fix produced virtual-hosted URLs, which is correct for AWS but doesn't help here — B2 rejects POST on either form. Verified with `curl -X POST https://...backblazeb2.com/honeyDueProd/` returning 501 directly, with no signature involved. Replace minio-go's PresignedPostPolicy with PresignHeader + http.MethodPut. The signed URL now points at a single PUT endpoint, with Content-Type and Content-Length signed via headers — B2/S3/MinIO all accept it. Drop the min/max content-length range (we sign exactly one length now); post-upload size verification still happens in VerifyAndClaim via HEAD. Response shape: - URL (was: signed POST endpoint) → now: signed PUT URL - Fields → renamed to Headers; client sends them as request headers, not multipart form parts - Method (new): always "PUT", emitted explicitly so clients don't have to hardcode Companion KMP/iOS commits switch the client paths from multipart POST to single PUT. Existing builds in the field will need to be rebuilt. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,21 +2,31 @@ package responses
|
|||||||
|
|
||||||
// PresignUploadResponse is what /api/uploads/presign returns to the client.
|
// PresignUploadResponse is what /api/uploads/presign returns to the client.
|
||||||
//
|
//
|
||||||
// The client uses URL + Fields to build a multipart/form-data POST directly
|
// Flow: the client makes one PUT request to URL with the raw object bytes
|
||||||
// to S3-compatible storage (B2). Once the upload completes, the client calls
|
// as the body and Headers as the request headers (verbatim — the signature
|
||||||
// the relevant entity-creation endpoint (POST /api/task-completions/, POST
|
// binds them). On success, the client passes ID back via upload_ids[] on
|
||||||
// /api/documents/) with `upload_ids: [Id]` to claim and attach the object.
|
// POST /api/task-completions/ or POST /api/documents/ to claim and attach
|
||||||
|
// the object.
|
||||||
|
//
|
||||||
|
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
|
||||||
|
// not implement the S3 POST Object form upload — it returns HTTP 501 on
|
||||||
|
// every request style. PUT works against AWS S3, B2, and MinIO uniformly.
|
||||||
type PresignUploadResponse struct {
|
type PresignUploadResponse struct {
|
||||||
// ID is the pending_uploads.id the client passes back via upload_ids[].
|
// ID is the pending_uploads.id the client passes back via upload_ids[].
|
||||||
ID uint `json:"id"`
|
ID uint `json:"id"`
|
||||||
|
|
||||||
// URL is the storage endpoint to POST to (no query string).
|
// URL is the signed PUT URL. Includes all auth as query parameters.
|
||||||
URL string `json:"upload_url"`
|
URL string `json:"upload_url"`
|
||||||
|
|
||||||
// Fields are the form fields (policy, signature, key, etc.) that must be
|
// Method is always "PUT" — emitted explicitly so clients don't have to
|
||||||
// submitted with the multipart form. The file part must be named "file"
|
// hardcode it. Reserved for the rare case we ever offer alternative
|
||||||
// and come last per S3 POST policy rules.
|
// upload mechanisms.
|
||||||
Fields map[string]string `json:"fields"`
|
Method string `json:"method"`
|
||||||
|
|
||||||
|
// Headers must be sent verbatim on the PUT request. Currently includes
|
||||||
|
// Content-Type and Content-Length; both are signed, and B2 will reject
|
||||||
|
// any PUT whose headers don't match.
|
||||||
|
Headers map[string]string `json:"headers"`
|
||||||
|
|
||||||
// Key is the object key chosen by the server. Echoed for client logging
|
// Key is the object key chosen by the server. Echoed for client logging
|
||||||
// and debugging; the canonical reference is via ID.
|
// and debugging; the canonical reference is via ID.
|
||||||
|
|||||||
@@ -5,8 +5,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
@@ -31,15 +32,6 @@ func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region st
|
|||||||
Secure: useSSL,
|
Secure: useSSL,
|
||||||
Region: region,
|
Region: region,
|
||||||
}
|
}
|
||||||
// B2's S3-compatible endpoint only implements POST Object on
|
|
||||||
// virtual-hosted-style URLs (https://<bucket>.s3.<region>.backblazeb2.com/).
|
|
||||||
// Path-style POST returns HTTP 501. minio-go's auto-detection falls back
|
|
||||||
// to path-style for non-AWS endpoints, which breaks PresignedPostPolicy
|
|
||||||
// against B2. Force DNS lookup for B2 only — leave MinIO dev (no DNS for
|
|
||||||
// arbitrary buckets at minio:9000) on the path-style default.
|
|
||||||
if strings.Contains(endpoint, "backblazeb2.com") {
|
|
||||||
opts.BucketLookup = minio.BucketLookupDNS
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := minio.New(endpoint, opts)
|
client, err := minio.New(endpoint, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,46 +107,42 @@ func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) {
|
|||||||
return obj, nil
|
return obj, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PresignedPostResult is the data a client needs to perform a direct multipart
|
// PresignedPutResult is the data a client needs to upload a single object
|
||||||
// POST to S3-compatible storage. The caller assembles a multipart/form-data
|
// to S3-compatible storage via a single PUT request. The client sends the
|
||||||
// request with the fields below as form parts (in order) and the file last.
|
// raw object bytes as the request body and includes Headers verbatim — the
|
||||||
type PresignedPostResult struct {
|
// signature binds these headers, so any deviation invalidates it.
|
||||||
URL string // e.g. https://s3.us-east-005.backblazeb2.com/honeyDueProd
|
//
|
||||||
Fields map[string]string // policy, x-amz-*, key, Content-Type, etc.
|
// We use PUT (not POST) because Backblaze B2's S3-compatible endpoint does
|
||||||
|
// not implement the S3 POST Object operation (returns HTTP 501 on both
|
||||||
|
// path-style and virtual-hosted-style URLs). PUT works against AWS S3, B2,
|
||||||
|
// and MinIO uniformly.
|
||||||
|
type PresignedPutResult struct {
|
||||||
|
URL string // signed PUT URL with query-string credentials
|
||||||
|
Headers map[string]string // headers the client must send on the PUT
|
||||||
}
|
}
|
||||||
|
|
||||||
// PresignedPost generates a POST policy that constrains uploads at the protocol
|
// PresignedPut generates a single-use PUT URL that uploads exactly the
|
||||||
// level: only the named key, only the named content-type, only sizes within
|
// claimed `contentLength` bytes with the claimed `contentType`. The size
|
||||||
// the requested range. S3 (and B2's S3-compatible endpoint) reject anything
|
// and content-type are signed via headers — B2/S3 reject anything that
|
||||||
// that doesn't satisfy every condition before accepting the body.
|
// doesn't match.
|
||||||
//
|
//
|
||||||
// minBytes/maxBytes are inclusive. The returned URL + Fields can be sent
|
// Post-upload verification is the caller's job (see UploadService.VerifyAndClaim
|
||||||
// straight to the client.
|
// which HEADs the object to confirm size + type before claiming).
|
||||||
func (b *S3Backend) PresignedPost(ctx context.Context, key, contentType string, minBytes, maxBytes int64, ttl time.Duration) (*PresignedPostResult, error) {
|
func (b *S3Backend) PresignedPut(ctx context.Context, key, contentType string, contentLength int64, ttl time.Duration) (*PresignedPutResult, error) {
|
||||||
policy := minio.NewPostPolicy()
|
extra := http.Header{}
|
||||||
if err := policy.SetBucket(b.bucket); err != nil {
|
extra.Set("Content-Type", contentType)
|
||||||
return nil, fmt.Errorf("set bucket: %w", err)
|
extra.Set("Content-Length", strconv.FormatInt(contentLength, 10))
|
||||||
}
|
|
||||||
if err := policy.SetKey(key); err != nil {
|
|
||||||
return nil, fmt.Errorf("set key: %w", err)
|
|
||||||
}
|
|
||||||
if err := policy.SetContentType(contentType); err != nil {
|
|
||||||
return nil, fmt.Errorf("set content-type: %w", err)
|
|
||||||
}
|
|
||||||
if err := policy.SetContentLengthRange(minBytes, maxBytes); err != nil {
|
|
||||||
return nil, fmt.Errorf("set content-length-range: %w", err)
|
|
||||||
}
|
|
||||||
if err := policy.SetExpires(time.Now().UTC().Add(ttl)); err != nil {
|
|
||||||
return nil, fmt.Errorf("set expires: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
u, fields, err := b.client.PresignedPostPolicy(ctx, policy)
|
u, err := b.client.PresignHeader(ctx, http.MethodPut, b.bucket, key, ttl, nil, extra)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("presign post policy: %w", err)
|
return nil, fmt.Errorf("presign put: %w", err)
|
||||||
}
|
}
|
||||||
return &PresignedPostResult{
|
return &PresignedPutResult{
|
||||||
URL: stripQuery(u),
|
URL: u.String(),
|
||||||
Fields: fields,
|
Headers: map[string]string{
|
||||||
|
"Content-Type": contentType,
|
||||||
|
"Content-Length": strconv.FormatInt(contentLength, 10),
|
||||||
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -164,18 +164,12 @@ func (s *UploadService) Presign(
|
|||||||
ext := extensionForContentType(contentType)
|
ext := extensionForContentType(contentType)
|
||||||
key := fmt.Sprintf("uploads/%s/%d/%s%s", subdir, userID, uuid.New().String(), ext)
|
key := fmt.Sprintf("uploads/%s/%d/%s%s", subdir, userID, uuid.New().String(), ext)
|
||||||
|
|
||||||
// Sign the POST policy. The slack window lets the client encode once;
|
// Sign a single-use PUT URL that binds Content-Type and Content-Length.
|
||||||
// the server still rejects anything materially different from claimed.
|
// B2 does not implement S3's POST Object form upload (returns 501), so
|
||||||
minB := contentLength - UploadPresignSlackBytes
|
// we use PUT — which works uniformly against AWS S3, B2, and MinIO.
|
||||||
if minB < 0 {
|
// Size mismatch is caught later in VerifyAndClaim via HEAD; we don't
|
||||||
minB = 0
|
// need a min/max range here because the signature pins exactly one size.
|
||||||
}
|
put, err := s.s3.PresignedPut(ctx, key, contentType, contentLength, UploadPresignTTL)
|
||||||
maxB := contentLength + UploadPresignSlackBytes
|
|
||||||
if maxB > UploadMaxBytes {
|
|
||||||
maxB = UploadMaxBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
post, err := s.s3.PresignedPost(ctx, key, contentType, minB, maxB, UploadPresignTTL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, apperrors.Internal(fmt.Errorf("presign upload: %w", err))
|
return nil, apperrors.Internal(fmt.Errorf("presign upload: %w", err))
|
||||||
}
|
}
|
||||||
@@ -195,8 +189,9 @@ func (s *UploadService) Presign(
|
|||||||
|
|
||||||
return &responses.PresignUploadResponse{
|
return &responses.PresignUploadResponse{
|
||||||
ID: row.ID,
|
ID: row.ID,
|
||||||
URL: post.URL,
|
URL: put.URL,
|
||||||
Fields: post.Fields,
|
Headers: put.Headers,
|
||||||
|
Method: "PUT",
|
||||||
Key: key,
|
Key: key,
|
||||||
ExpiresAt: expiresAt.Format(time.RFC3339),
|
ExpiresAt: expiresAt.Format(time.RFC3339),
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
Reference in New Issue
Block a user