feat(uploads): direct-to-B2 presigned uploads with content-length-range policy
Replaces the multipart-via-API path for image uploads with a three-step
direct-to-storage flow:
1. Client POSTs /api/uploads/presign with content_length + content_type;
server validates size (10 MB cap), mime allow-list per category, rate
limit (50/hour/user via Redis sliding window), and concurrent unclaimed
cap (10 in-flight per user). On success it persists a pending_uploads
row, signs an S3 POST policy with content-length-range bound to the
claimed length ±256 bytes, and returns the URL+fields.
2. Client POSTs the bytes directly to B2 using the signed policy. B2
enforces size, content-type, and key match before accepting.
3. Client passes upload_ids[] to /api/task-completions/ or /api/documents/.
Service HEADs each B2 object, verifies size matches expected_bytes
within slack, marks pending_uploads claimed_at, and creates the
associated TaskCompletionImage / DocumentImage rows.
Bytes never traverse our API server. The 1 MB Echo BodyLimit middleware
that was rejecting all task-completion image uploads becomes irrelevant
for this path. Existing multipart endpoints stay functional alongside,
soak-testing the new path before legacy removal.
Cleanup:
- cmd/worker registers a new hourly cron (TypeUploadCleanup, "30 * * * *")
that reaps pending_uploads where claimed_at IS NULL AND expires_at < NOW().
Reaps both the B2 object and the row.
- B2 bucket lifecycle rule on `uploads/` prefix (7 days hide → 1 day delete)
documented in deploy-k3s/manifests/b2-lifecycle.md as a backstop.
Schema:
- migrations/000002_pending_uploads.sql adds the table + partial index for
cleanup + nullable pending_upload_id FKs on task_taskcompletionimage and
task_documentimage.
Policy (single tier, no free/pro split):
- 10 MB cap per upload
- 50 presigns/hour/user
- 10 concurrent unclaimed uploads/user
- allow-list: jpeg/png/heic/heif/webp for image categories;
+ pdf for document_file
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -167,6 +167,20 @@ func main() {
|
||||
// Create job handler
|
||||
jobHandler := jobs.NewHandler(db, pushClient, emailService, notificationService, cfg)
|
||||
|
||||
// Wire upload service for the pending_uploads cleanup cron. Storage may
|
||||
// be local-disk (no S3 backend), in which case the upload service stays
|
||||
// nil and the cleanup handler no-ops. Cache is optional — the cleanup
|
||||
// path doesn't rate-limit and works fine with a nil cache.
|
||||
if storageService, sErr := services.NewStorageService(&cfg.Storage); sErr == nil {
|
||||
if s3 := storageService.S3Backend(); s3 != nil {
|
||||
pendingUploadRepo := repositories.NewPendingUploadRepository(db)
|
||||
uploadService := services.NewUploadService(pendingUploadRepo, s3, &cfg.Storage, nil)
|
||||
jobHandler.SetUploadService(uploadService)
|
||||
}
|
||||
} else {
|
||||
log.Warn().Err(sErr).Msg("Failed to initialize storage service for upload cleanup; cleanup cron will no-op")
|
||||
}
|
||||
|
||||
// Create Asynq mux and register handlers
|
||||
mux := asynq.NewServeMux()
|
||||
|
||||
@@ -180,6 +194,7 @@ func main() {
|
||||
mux.HandleFunc(jobs.TypeSendPush, jobHandler.HandleSendPush)
|
||||
mux.HandleFunc(jobs.TypeOnboardingEmails, jobHandler.HandleOnboardingEmails)
|
||||
mux.HandleFunc(jobs.TypeReminderLogCleanup, jobHandler.HandleReminderLogCleanup)
|
||||
mux.HandleFunc(jobs.TypeUploadCleanup, jobHandler.HandleUploadCleanup)
|
||||
|
||||
// Register email job handlers (welcome, verification, password reset, password changed)
|
||||
if emailService != nil {
|
||||
@@ -219,6 +234,15 @@ func main() {
|
||||
}
|
||||
log.Info().Str("cron", "0 3 * * *").Msg("Registered reminder log cleanup job (runs daily at 3:00 AM UTC)")
|
||||
|
||||
// Schedule pending_uploads cleanup (hourly at :30 to avoid colliding with
|
||||
// the top-of-hour reminder + digest crons). Reaps unclaimed expired
|
||||
// upload sessions; the B2 bucket lifecycle (7 days on uploads/ prefix)
|
||||
// is the backstop if this worker is offline for an extended period.
|
||||
if _, err := scheduler.Register("30 * * * *", asynq.NewTask(jobs.TypeUploadCleanup, nil)); err != nil {
|
||||
log.Fatal().Err(err).Msg("Failed to register upload cleanup job")
|
||||
}
|
||||
log.Info().Str("cron", "30 * * * *").Msg("Registered pending_uploads cleanup job (runs hourly)")
|
||||
|
||||
// Handle graceful shutdown
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
Reference in New Issue
Block a user