Harden prod deploy: versioned secrets, healthchecks, migration lock, dry-run

Swarm stack
- Resource limits on all services, stop_grace_period 60s on api/worker/admin
- Dozzle bound to manager loopback only (ssh -L required for access)
- Worker health server on :6060, admin /api/health endpoint
- Redis 200M LRU cap, B2/S3 env vars wired through to api service

Deploy script
- DRY_RUN=1 prints plan + exits
- Auto-rollback on failed healthcheck, docker logout at end
- Versioned-secret pruning keeps last SECRET_KEEP_VERSIONS (default 3)
- PUSH_LATEST_TAG default flipped to false
- B2 all-or-none validation before deploy

Code
- cmd/api takes pg_advisory_lock on a dedicated connection before
  AutoMigrate, serialising boot-time migrations across replicas
- cmd/worker exposes an HTTP /health endpoint with graceful shutdown

Docs
- deploy/DEPLOYING.md: step-by-step walkthrough for a real deploy
- deploy/shit_deploy_cant_do.md: manual prerequisites + recurring ops
- deploy/README.md updated with storage toggle, worker-replica caveat,
  multi-arch recipe, connection-pool tuning, renumbered sections

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

cmd/api/main.go

@@ -65,8 +65,10 @@ func main() {
 		log.Error().Err(dbErr).Msg("Failed to connect to database - API will start but database operations will fail")
 	} else {
 		defer database.Close()
-		// Run database migrations only if connected
-		if err := database.Migrate(); err != nil {
+		// Run database migrations only if connected.
+		// MigrateWithLock serialises parallel replica starts via a Postgres
+		// advisory lock so concurrent AutoMigrate calls don't race on DDL.
+		if err := database.MigrateWithLock(); err != nil {
 			log.Error().Err(err).Msg("Failed to run database migrations")
 		}
 	}

cmd/worker/main.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"context"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/hibiken/asynq"
 	"github.com/redis/go-redis/v9"
@@ -20,6 +22,8 @@ import (
 	"github.com/treytartt/honeydue-api/pkg/utils"
 )
 
+const workerHealthAddr = ":6060"
+
 func main() {
 	// Initialize logger
 	utils.InitLogger(true)
@@ -188,6 +192,25 @@ func main() {
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
 
+	// Health server (for container healthchecks; not externally published)
+	healthMux := http.NewServeMux()
+	healthMux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"status":"ok"}`))
+	})
+	healthSrv := &http.Server{
+		Addr:              workerHealthAddr,
+		Handler:           healthMux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+	go func() {
+		log.Info().Str("addr", workerHealthAddr).Msg("Health server listening")
+		if err := healthSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Warn().Err(err).Msg("Health server terminated")
+		}
+	}()
+
 	// Start scheduler in goroutine
 	go func() {
 		if err := scheduler.Run(); err != nil {
@@ -207,6 +230,9 @@ func main() {
 	log.Info().Msg("Shutting down worker...")
 
 	// Graceful shutdown
+	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer shutdownCancel()
+	_ = healthSrv.Shutdown(shutdownCtx)
 	srv.Shutdown()
 	scheduler.Shutdown()