diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f184f56 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,54 @@ +# Git +.git +.gitignore +.gitattributes +.github +.gitea + +# Deploy inputs (never bake into images) +deploy/*.env +deploy/secrets/*.txt +deploy/secrets/*.p8 +deploy/scripts/ + +# Local env files +.env +.env.* +!.env.example + +# Node (admin) +admin/node_modules +admin/.next +admin/out +admin/.turbo +admin/.vercel +admin/npm-debug.log* + +# Go build artifacts +bin/ +dist/ +tmp/ +*.test +*.out +coverage.out +coverage.html + +# Tooling / editor +.vscode +.idea +*.swp +*.swo +.DS_Store + +# Logs +*.log +logs/ + +# Tests / docs (not needed at runtime) +docs/ +*.md +!README.md + +# CI/compose locals (not needed for swarm image build) +docker-compose*.yml +Makefile diff --git a/cmd/api/main.go b/cmd/api/main.go index 8471725..cd7055b 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -65,8 +65,10 @@ func main() { log.Error().Err(dbErr).Msg("Failed to connect to database - API will start but database operations will fail") } else { defer database.Close() - // Run database migrations only if connected - if err := database.Migrate(); err != nil { + // Run database migrations only if connected. + // MigrateWithLock serialises parallel replica starts via a Postgres + // advisory lock so concurrent AutoMigrate calls don't race on DDL. + if err := database.MigrateWithLock(); err != nil { log.Error().Err(err).Msg("Failed to run database migrations") } } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 6cefc00..e20899e 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -2,9 +2,11 @@ package main import ( "context" + "net/http" "os" "os/signal" "syscall" + "time" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" @@ -20,6 +22,8 @@ import ( "github.com/treytartt/honeydue-api/pkg/utils" ) +const workerHealthAddr = ":6060" + func main() { // Initialize logger utils.InitLogger(true) @@ -188,6 +192,25 @@ func main() { quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + // Health server (for container healthchecks; not externally published) + healthMux := http.NewServeMux() + healthMux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status":"ok"}`)) + }) + healthSrv := &http.Server{ + Addr: workerHealthAddr, + Handler: healthMux, + ReadHeaderTimeout: 5 * time.Second, + } + go func() { + log.Info().Str("addr", workerHealthAddr).Msg("Health server listening") + if err := healthSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Warn().Err(err).Msg("Health server terminated") + } + }() + // Start scheduler in goroutine go func() { if err := scheduler.Run(); err != nil { @@ -207,6 +230,9 @@ func main() { log.Info().Msg("Shutting down worker...") // Graceful shutdown + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = healthSrv.Shutdown(shutdownCtx) srv.Shutdown() scheduler.Shutdown() diff --git a/deploy/DEPLOYING.md b/deploy/DEPLOYING.md new file mode 100644 index 0000000..61841e7 --- /dev/null +++ b/deploy/DEPLOYING.md @@ -0,0 +1,126 @@ +# Deploying Right Now + +Practical walkthrough for a prod deploy against the current Swarm stack. +Assumes infrastructure and cloud services already exist — if not, work +through [`shit_deploy_cant_do.md`](./shit_deploy_cant_do.md) first. + +See [`README.md`](./README.md) for the reference docs that back each step. + +--- + +## 0. Pre-flight — check local state + +```bash +cd honeyDueAPI-go + +git status # clean working tree? +git log -1 --oneline # deploying this SHA + +ls deploy/cluster.env deploy/registry.env deploy/prod.env +ls deploy/secrets/*.txt deploy/secrets/*.p8 +``` + +## 1. Reconcile your envs with current defaults + +These two values **must** be right — the script does not enforce them: + +```bash +# deploy/cluster.env +WORKER_REPLICAS=1 # >1 → duplicate cron jobs (Asynq scheduler is a singleton) +PUSH_LATEST_TAG=false # keeps prod images SHA-pinned +SECRET_KEEP_VERSIONS=3 # optional; 3 is the default +``` + +Decide storage backend in `deploy/prod.env`: + +- **Multi-replica safe (recommended):** set all four of `B2_ENDPOINT`, + `B2_KEY_ID`, `B2_APP_KEY`, `B2_BUCKET_NAME`. Uploads go to B2. +- **Single-node ok:** leave all four empty. Script will warn. In this + mode you must also set `API_REPLICAS=1` — otherwise uploads are + invisible from 2/3 of requests. + +## 2. Dry run + +```bash +DRY_RUN=1 ./.deploy_prod +``` + +Confirm in the output: +- `Storage backend: S3 (...)` OR the `LOCAL VOLUME` warning matches intent +- `Replicas: api=3, worker=1, admin=1` (or `api=1` if local storage) +- Image SHA matches `git rev-parse --short HEAD` +- `Manager:` host is correct +- `Secret retention: 3 versions` + +Fix envs and re-run until the plan looks right. Nothing touches the cluster yet. + +## 3. Real deploy + +```bash +./.deploy_prod +``` + +Do **not** pass `SKIP_BUILD=1` after code changes — the worker's health +server and `MigrateWithLock` both require a fresh build. + +End-to-end: ~3–8 minutes. The script prints each phase. + +## 4. Post-deploy verification + +```bash +# Stack health (replicas X/X = desired) +ssh docker stack services honeydue + +# API smoke +curl -fsS https://api./api/health/ && echo OK + +# Logs via Dozzle (loopback-bound, needs SSH tunnel) +ssh -p -L 9999:127.0.0.1:9999 @ +# Then browse http://localhost:9999 +``` + +What the logs should show on a healthy boot: +- `api`: exactly one replica logs `Migration advisory lock acquired`, + the others log `Migration advisory lock acquired` after waiting, then + `released`. +- `worker`: `Health server listening addr=:6060`, `Starting worker server...`, + four `Registered ... job` lines. +- No `Failed to connect to Redis` / `Failed to connect to database`. + +## 5. If it goes wrong + +Auto-rollback triggers when `DEPLOY_HEALTHCHECK_URL` fails — every service +is rolled back to its previous spec, script exits non-zero. + +Triage: + +```bash +ssh docker service logs --tail 200 honeydue_api +ssh docker service ps honeydue_api --no-trunc +``` + +Manual rollback (if auto didn't catch it): + +```bash +ssh bash -c ' + for svc in $(docker stack services honeydue --format "{{.Name}}"); do + docker service rollback "$svc" + done' +``` + +Redeploy a known-good SHA: + +```bash +DEPLOY_TAG= SKIP_BUILD=1 ./.deploy_prod +# Only valid if that image was previously pushed to the registry. +``` + +## 6. Pre-deploy honesty check + +Before pulling the trigger: + +- [ ] Tested Neon PITR restore (not just "backups exist")? +- [ ] `WORKER_REPLICAS=1` — otherwise duplicate push notifications next cron tick +- [ ] Cloudflare-only firewall rule on 80/443 — otherwise origin IP is on the public internet +- [ ] If storage is LOCAL, `API_REPLICAS=1` too +- [ ] Last deploy's secrets still valid (rotation hasn't expired any creds) diff --git a/deploy/README.md b/deploy/README.md index 366ea43..7110975 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -2,13 +2,18 @@ This folder is the full production deploy toolkit for `honeyDueAPI-go`. -Run deploy with: +**Recommended flow — always dry-run first:** ```bash -./.deploy_prod +DRY_RUN=1 ./.deploy_prod # validates everything, prints the plan, no changes +./.deploy_prod # then the real deploy ``` -The script will refuse to run until all required values are set. +The script refuses to run until all required values are set. + +- Step-by-step walkthrough for a real deploy: [`DEPLOYING.md`](./DEPLOYING.md) +- Manual prerequisites the script cannot automate (Swarm init, firewall, + Cloudflare, Neon, APNS, etc.): [`shit_deploy_cant_do.md`](./shit_deploy_cant_do.md) ## First-Time Prerequisite: Create The Swarm Cluster @@ -84,16 +89,159 @@ AllowUsers deploy ### 6) Dozzle Hardening -- Keep Dozzle private (no public DNS/ingress). +Dozzle exposes the full Docker log stream with no built-in auth — logs contain +secrets, tokens, and user data. The stack binds Dozzle to `127.0.0.1` on the +manager node only (`mode: host`, `host_ip: 127.0.0.1`), so it is **not +reachable from the public internet or from other Swarm nodes**. + +To view logs, open an SSH tunnel from your workstation: + +```bash +ssh -p "${DEPLOY_MANAGER_SSH_PORT}" \ + -L "${DOZZLE_PORT}:127.0.0.1:${DOZZLE_PORT}" \ + "${DEPLOY_MANAGER_USER}@${DEPLOY_MANAGER_HOST}" +# Then browse http://localhost:${DOZZLE_PORT} +``` + +Additional hardening if you ever need to expose Dozzle over a network: + - Put auth/SSO in front (Cloudflare Access or equivalent). -- Prefer a Docker socket proxy with restricted read-only scope. +- Replace the raw `/var/run/docker.sock` mount with a Docker socket proxy + limited to read-only log endpoints. +- Prefer a persistent log aggregator (Loki, Datadog, CloudWatch) for prod — + Dozzle is ephemeral and not a substitute for audit trails. ### 7) Backup + Restore Readiness -- Postgres PITR path tested in staging. -- Redis persistence enabled and restore path tested. -- Written runbook for restore and secret rotation. -- Named owner for incident response. +Treat this as a pre-launch checklist. Nothing below is automated by +`./.deploy_prod`. + +- [ ] Postgres PITR path tested in staging (restore a real dump, validate app boots). +- [x] Redis AOF persistence enabled (`appendonly yes --appendfsync everysec` in stack). +- [ ] Redis restore path tested (verify AOF replays on a fresh node). +- [ ] Written runbook for restore + secret rotation (see §4 and `shit_deploy_cant_do.md`). +- [ ] Named owner for incident response. +- [ ] Uploads bucket (Backblaze B2) lifecycle / versioning reviewed — deletes are + handled by the app, not by retention rules. + +### 8) Storage Backend (Uploads) + +The stack supports two storage backends. The choice is **runtime-only** — the +same image runs in both modes, selected by env vars in `prod.env`: + +| Mode | When to use | Config | +|---|---|---| +| **Local volume** | Dev / single-node prod | Leave all `B2_*` empty. Files land on `/app/uploads` via the named volume. | +| **S3-compatible** (B2, MinIO) | Multi-replica prod | Set all four of `B2_ENDPOINT`, `B2_KEY_ID`, `B2_APP_KEY`, `B2_BUCKET_NAME`. | + +The deploy script enforces **all-or-none** for the B2 vars — a partial config +fails fast rather than silently falling back to the local volume. + +**Why this matters:** Docker Swarm named volumes are **per-node**. With 3 API +replicas spread across nodes, an upload written on node A is invisible to +replicas on nodes B and C (the client sees a random 404 two-thirds of the +time). In multi-replica prod you **must** use S3-compatible storage. + +The `uploads:` volume is still declared as a harmless fallback: when B2 is +configured, nothing writes to it. `./.deploy_prod` prints the selected +backend at the start of each run. + +### 9) Worker Replicas & Scheduler + +Keep `WORKER_REPLICAS=1` in `cluster.env` until Asynq `PeriodicTaskManager` +is wired up. The current `asynq.Scheduler` in `cmd/worker/main.go` has no +Redis-based leader election, so each replica independently enqueues the +same cron task — users see duplicate daily digests / onboarding emails. + +Asynq workers (task consumers) are already safe to scale horizontally; it's +only the scheduler singleton that is constrained. Future work: migrate to +`asynq.NewPeriodicTaskManager(...)` with `PeriodicTaskConfigProvider` so +multiple scheduler replicas coordinate via Redis. + +### 10) Database Migrations + +`cmd/api/main.go` runs `database.MigrateWithLock()` on startup, which takes a +Postgres session-level `pg_advisory_lock` on a dedicated connection before +calling `AutoMigrate`. This serialises boot-time migrations across all API +replicas — the first replica migrates, the rest wait, then each sees an +already-current schema and `AutoMigrate` is a no-op. + +The lock is released on connection close, so a crashed replica can't leave +a stale lock behind. + +For very large schema changes, run migrations as a separate pre-deploy +step (there is no dedicated `cmd/migrate` binary today — this is a future +improvement). + +### 11) Redis Redundancy + +Redis runs as a **single replica** with an AOF-persisted named volume. If +the node running Redis dies, Swarm reschedules the container but the named +volume is per-node — the new Redis boots **empty**. + +Impact: +- **Cache** (ETag lookups, static data): regenerates on first request. +- **Asynq queue**: in-flight jobs at the moment of the crash are lost; Asynq + retry semantics cover most re-enqueues. Scheduled-but-not-yet-fired cron + events are re-triggered on the next cron tick. +- **Sessions / auth tokens**: not stored in Redis, so unaffected. + +This is an accepted limitation today. Options to harden later: Redis +Sentinel, a managed Redis (Upstash, Dragonfly Cloud), or restoring from the +AOF on a pinned node. + +### 12) Multi-Arch Builds + +`./.deploy_prod` builds images for the **host** architecture of the machine +running the script. If your Swarm nodes are a different arch (e.g. ARM64 +Ampere VMs), use `docker buildx` explicitly: + +```bash +docker buildx create --use +docker buildx build --platform linux/arm64 --target api -t --push . +# repeat for worker, admin +SKIP_BUILD=1 ./.deploy_prod # then deploy the already-pushed images +``` + +The Go stages cross-compile cleanly (`TARGETARCH` is already honoured). +The Node/admin stages require QEMU emulation (`docker run --privileged --rm +tonistiigi/binfmt --install all` on the build host) since native deps may +need to be rebuilt for the target arch. + +### 13) Connection Pool & TLS Tuning + +Because Postgres is external (Neon/RDS), each replica opens its own pool. +Sizing matters: total open connections across the cluster must stay under +the database's configured limit. Defaults in `prod.env.example`: + +| Setting | Default | Notes | +|---|---|---| +| `DB_SSLMODE` | `require` | Never set to `disable` in prod. For Neon use `require`. | +| `DB_MAX_OPEN_CONNS` | `25` | Per-replica cap. Worst case: 25 × (API+worker replicas). | +| `DB_MAX_IDLE_CONNS` | `10` | Keep warm connections ready without exhausting the pool. | +| `DB_MAX_LIFETIME` | `600s` | Recycle before Neon's idle disconnect (typically 5 min). | + +Worked example with default replicas (3 API + 1 worker — see §9 for why +worker is pinned to 1): + +``` +3 × 25 + 1 × 25 = 100 peak open connections +``` + +That lands exactly on Neon's free-tier ceiling (100 concurrent connections), +which is risky with even one transient spike. For Neon free tier drop +`DB_MAX_OPEN_CONNS=15` (→ 60 peak). Paid tiers (Neon Scale, 1000+ +connections) can keep the default or raise it. + +Operational checklist: + +- Confirm Neon IP allowlist includes every Swarm node IP. +- After changing pool sizes, redeploy and watch `pg_stat_activity` / + Neon metrics for saturation. +- Keep `DB_MAX_LIFETIME` ≤ Neon idle timeout to avoid "terminating + connection due to administrator command" errors in the API logs. +- For read-heavy workloads, consider a Neon read replica and split + query traffic at the application layer. ## Files You Fill In @@ -113,20 +261,51 @@ If one is missing, the deploy script auto-copies it from its `.example` template ## What `./.deploy_prod` Does 1. Validates all required config files and credentials. -2. Builds and pushes `api`, `worker`, and `admin` images. -3. Uploads deploy bundle to your Swarm manager over SSH. -4. Creates versioned Docker secrets on the manager. -5. Deploys the stack with `docker stack deploy --with-registry-auth`. -6. Waits until service replicas converge. -7. Runs an HTTP health check (if `DEPLOY_HEALTHCHECK_URL` is set). +2. Validates the storage-backend toggle (all-or-none for `B2_*`). Prints + the selected backend (S3 or local volume) before continuing. +3. Builds and pushes `api`, `worker`, and `admin` images (skip with + `SKIP_BUILD=1`). +4. Uploads deploy bundle to your Swarm manager over SSH. +5. Creates versioned Docker secrets on the manager. +6. Deploys the stack with `docker stack deploy --with-registry-auth`. +7. Waits until service replicas converge. +8. Prunes old secret versions, keeping the last `SECRET_KEEP_VERSIONS` + (default 3). +9. Runs an HTTP health check (if `DEPLOY_HEALTHCHECK_URL` is set). **On + failure, automatically runs `docker service rollback` for every service + in the stack and exits non-zero.** +10. Logs out of the registry on both the dev host and the manager so the + token doesn't linger in `~/.docker/config.json`. ## Useful Flags Environment flags: -- `SKIP_BUILD=1 ./.deploy_prod` to deploy already-pushed images. -- `SKIP_HEALTHCHECK=1 ./.deploy_prod` to skip final URL check. -- `DEPLOY_TAG= ./.deploy_prod` to deploy a specific image tag. +- `DRY_RUN=1 ./.deploy_prod` — validate config and print the deploy plan + without building, pushing, or touching the cluster. Use this before every + production deploy to review images, replicas, and secret names. +- `SKIP_BUILD=1 ./.deploy_prod` — deploy already-pushed images. +- `SKIP_HEALTHCHECK=1 ./.deploy_prod` — skip final URL check. +- `DEPLOY_TAG= ./.deploy_prod` — deploy a specific image tag. +- `PUSH_LATEST_TAG=true ./.deploy_prod` — also push `:latest` to the registry + (default is `false` so prod pins to the SHA tag and stays reproducible). +- `SECRET_KEEP_VERSIONS= ./.deploy_prod` — how many versions of each + Swarm secret to retain after deploy (default: 3). Older unused versions + are pruned automatically once the stack converges. + +## Secret Versioning & Pruning + +Each deploy creates a fresh set of Swarm secrets named +`__` (for example +`honeydue_secret_key_abc1234_20260413120000`). The stack file references the +current names via `${POSTGRES_PASSWORD_SECRET}` etc., so rolling updates never +reuse a secret that a running task still holds open. + +After the new stack converges, `./.deploy_prod` SSHes to the manager and +prunes old versions per base name, keeping the most recent +`SECRET_KEEP_VERSIONS` (default 3). Anything still referenced by a running +task is left alone (Docker refuses to delete in-use secrets) and will be +pruned on the next deploy. ## Important diff --git a/deploy/cluster.env.example b/deploy/cluster.env.example index 1caeb79..b63ef24 100644 --- a/deploy/cluster.env.example +++ b/deploy/cluster.env.example @@ -12,11 +12,21 @@ DEPLOY_HEALTHCHECK_URL=https://api.honeyDue.treytartt.com/api/health/ # Replicas and published ports API_REPLICAS=3 -WORKER_REPLICAS=2 +# IMPORTANT: keep WORKER_REPLICAS=1 until Asynq PeriodicTaskManager is wired. +# The current asynq.Scheduler in cmd/worker/main.go has no Redis-based +# leader election, so running >1 replica fires every cron task once per +# replica → duplicate daily digests / onboarding emails / etc. +WORKER_REPLICAS=1 ADMIN_REPLICAS=1 API_PORT=8000 ADMIN_PORT=3000 DOZZLE_PORT=9999 # Build behavior -PUSH_LATEST_TAG=true +# PUSH_LATEST_TAG=true also tags and pushes :latest on the registry. +# Leave false in production to keep image tags immutable (SHA-pinned only). +PUSH_LATEST_TAG=false + +# Secret retention: number of versioned Swarm secrets to keep per name after each deploy. +# Older unused versions are pruned post-convergence. Default: 3. +SECRET_KEEP_VERSIONS=3 diff --git a/deploy/prod.env.example b/deploy/prod.env.example index cb8a310..937aedb 100644 --- a/deploy/prod.env.example +++ b/deploy/prod.env.example @@ -50,6 +50,27 @@ STORAGE_BASE_URL=/uploads STORAGE_MAX_FILE_SIZE=10485760 STORAGE_ALLOWED_TYPES=image/jpeg,image/png,image/gif,image/webp,application/pdf +# Storage backend (S3-compatible: Backblaze B2 or MinIO) +# +# Leave all B2_* vars empty to use the local filesystem at STORAGE_UPLOAD_DIR. +# - Safe for single-node setups (dev / single-VPS prod). +# - NOT SAFE for multi-replica prod: named volumes are per-node in Swarm, +# so uploads written on one node are invisible to the other replicas. +# +# Set ALL FOUR of B2_ENDPOINT, B2_KEY_ID, B2_APP_KEY, B2_BUCKET_NAME to +# switch to S3-compatible storage. The deploy script enforces all-or-none. +# +# Example for Backblaze B2 (us-west-004): +# B2_ENDPOINT=s3.us-west-004.backblazeb2.com +# B2_USE_SSL=true +# B2_REGION=us-west-004 +B2_ENDPOINT= +B2_KEY_ID= +B2_APP_KEY= +B2_BUCKET_NAME= +B2_USE_SSL=true +B2_REGION=us-east-1 + # Feature flags FEATURE_PUSH_ENABLED=true FEATURE_EMAIL_ENABLED=true diff --git a/deploy/scripts/deploy_prod.sh b/deploy/scripts/deploy_prod.sh index f2cf698..52504a1 100755 --- a/deploy/scripts/deploy_prod.sh +++ b/deploy/scripts/deploy_prod.sh @@ -18,6 +18,8 @@ SECRET_APNS_KEY="${DEPLOY_DIR}/secrets/apns_auth_key.p8" SKIP_BUILD="${SKIP_BUILD:-0}" SKIP_HEALTHCHECK="${SKIP_HEALTHCHECK:-0}" +DRY_RUN="${DRY_RUN:-0}" +SECRET_KEEP_VERSIONS="${SECRET_KEEP_VERSIONS:-3}" log() { printf '[deploy] %s\n' "$*" @@ -91,9 +93,13 @@ Usage: ./.deploy_prod Optional environment flags: - SKIP_BUILD=1 Deploy existing image tags without rebuilding/pushing. - SKIP_HEALTHCHECK=1 Skip final HTTP health check. - DEPLOY_TAG= Override image tag (default: git short sha). + DRY_RUN=1 Print the deployment plan and exit without changes. + SKIP_BUILD=1 Deploy existing image tags without rebuilding/pushing. + SKIP_HEALTHCHECK=1 Skip final HTTP health check. + DEPLOY_TAG= Override image tag (default: git short sha). + PUSH_LATEST_TAG=true|false Also tag/push :latest (default: false — SHA only). + SECRET_KEEP_VERSIONS= How many versions of each Swarm secret to retain + (default: 3). Older unused versions are pruned. EOF } @@ -144,7 +150,7 @@ DEPLOY_STACK_NAME="${DEPLOY_STACK_NAME:-honeydue}" DEPLOY_REMOTE_DIR="${DEPLOY_REMOTE_DIR:-/opt/honeydue/deploy}" DEPLOY_WAIT_SECONDS="${DEPLOY_WAIT_SECONDS:-420}" DEPLOY_TAG="${DEPLOY_TAG:-$(git -C "${REPO_DIR}" rev-parse --short HEAD)}" -PUSH_LATEST_TAG="${PUSH_LATEST_TAG:-true}" +PUSH_LATEST_TAG="${PUSH_LATEST_TAG:-false}" require_var DEPLOY_MANAGER_HOST require_var DEPLOY_MANAGER_USER @@ -173,6 +179,27 @@ require_var APNS_AUTH_KEY_ID require_var APNS_TEAM_ID require_var APNS_TOPIC +# Storage backend validation: B2 is all-or-none. If any var is filled with +# a real value, require all four core vars. Empty means "use local volume". +b2_any_set=0 +b2_all_set=1 +for b2_var in B2_ENDPOINT B2_KEY_ID B2_APP_KEY B2_BUCKET_NAME; do + val="${!b2_var:-}" + if [[ -n "${val}" ]] && ! contains_placeholder "${val}"; then + b2_any_set=1 + else + b2_all_set=0 + fi +done +if (( b2_any_set == 1 && b2_all_set == 0 )); then + die "Partial B2 configuration detected. Set all four of B2_ENDPOINT, B2_KEY_ID, B2_APP_KEY, B2_BUCKET_NAME, or leave all four empty to use the local volume." +fi +if (( b2_all_set == 1 )); then + log "Storage backend: S3 (${B2_ENDPOINT} / bucket=${B2_BUCKET_NAME})" +else + warn "Storage backend: LOCAL VOLUME. This is not safe for multi-replica prod — uploads will only exist on one node. Set B2_* in prod.env to use object storage." +fi + if [[ ! "$(tr -d '\r\n' < "${SECRET_APNS_KEY}")" =~ BEGIN[[:space:]]+PRIVATE[[:space:]]+KEY ]]; then die "APNS key file does not look like a private key: ${SECRET_APNS_KEY}" fi @@ -200,6 +227,50 @@ if [[ -n "${SSH_KEY_PATH}" ]]; then SCP_OPTS+=(-i "${SSH_KEY_PATH}") fi +if [[ "${DRY_RUN}" == "1" ]]; then + cat < + ${DEPLOY_STACK_NAME}_secret_key_ + ${DEPLOY_STACK_NAME}_email_host_password_ + ${DEPLOY_STACK_NAME}_fcm_server_key_ + ${DEPLOY_STACK_NAME}_apns_auth_key_ + +No changes made. Re-run without DRY_RUN=1 to deploy. +================================================= + +EOF + exit 0 +fi + log "Validating SSH access to ${SSH_TARGET}" if ! ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "echo ok" >/dev/null 2>&1; then die "SSH connection failed to ${SSH_TARGET}" @@ -384,11 +455,77 @@ while true; do sleep 10 done +log "Pruning old secret versions (keeping last ${SECRET_KEEP_VERSIONS})" +ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}' '${SECRET_KEEP_VERSIONS}'" <<'EOF' || warn "Secret pruning reported errors (non-fatal)" +set -euo pipefail + +STACK_NAME="$1" +KEEP="$2" + +prune_prefix() { + local prefix="$1" + # List matching secrets with creation time, sorted newest-first. + local all + all="$(docker secret ls --format '{{.CreatedAt}}|{{.Name}}' 2>/dev/null \ + | grep "|${prefix}_" \ + | sort -r \ + || true)" + if [[ -z "${all}" ]]; then + return 0 + fi + + local total + total="$(printf '%s\n' "${all}" | wc -l | tr -d ' ')" + if (( total <= KEEP )); then + echo "[cleanup] ${prefix}: ${total} version(s) — nothing to prune" + return 0 + fi + + local to_remove + to_remove="$(printf '%s\n' "${all}" | tail -n +$((KEEP + 1)) | awk -F'|' '{print $2}')" + + while IFS= read -r name; do + [[ -z "${name}" ]] && continue + if docker secret rm "${name}" >/dev/null 2>&1; then + echo "[cleanup] removed: ${name}" + else + echo "[cleanup] in-use (kept): ${name}" + fi + done <<< "${to_remove}" +} + +for base in postgres_password secret_key email_host_password fcm_server_key apns_auth_key; do + prune_prefix "${STACK_NAME}_${base}" +done +EOF + +rollback_stack() { + warn "Rolling back stack ${DEPLOY_STACK_NAME} on ${SSH_TARGET}" + ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}'" <<'EOF' || true +set +e +STACK="$1" +for svc in $(docker stack services "${STACK}" --format '{{.Name}}'); do + echo "[rollback] ${svc}" + docker service rollback "${svc}" || echo "[rollback] ${svc}: nothing to roll back" +done +EOF +} + if [[ "${SKIP_HEALTHCHECK}" != "1" && -n "${DEPLOY_HEALTHCHECK_URL:-}" ]]; then log "Running health check: ${DEPLOY_HEALTHCHECK_URL}" - curl -fsS --max-time 20 "${DEPLOY_HEALTHCHECK_URL}" >/dev/null + if ! curl -fsS --max-time 20 "${DEPLOY_HEALTHCHECK_URL}" >/dev/null; then + warn "Health check FAILED for ${DEPLOY_HEALTHCHECK_URL}" + rollback_stack + die "Deploy rolled back due to failed health check." + fi fi +# Best-effort registry logout — the token should not linger in +# ~/.docker/config.json after deploy completes. Failures are non-fatal. +log "Logging out of registry (local + remote)" +docker logout "${REGISTRY}" >/dev/null 2>&1 || true +ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "docker logout '${REGISTRY}' >/dev/null 2>&1 || true" + log "Deploy completed successfully." log "Stack: ${DEPLOY_STACK_NAME}" log "Images:" diff --git a/deploy/shit_deploy_cant_do.md b/deploy/shit_deploy_cant_do.md new file mode 100644 index 0000000..cef7652 --- /dev/null +++ b/deploy/shit_deploy_cant_do.md @@ -0,0 +1,208 @@ +# Shit `./.deploy_prod` Can't Do + +Everything listed here is **manual**. The deploy script orchestrates builds, +secrets, and the stack — it does not provision infrastructure, touch DNS, +configure Cloudflare, or rotate external credentials. Work through this list +once before your first prod deploy, then revisit after every cloud-side +change. + +See [`README.md`](./README.md) for the security checklist that complements +this file. + +--- + +## One-Time: Infrastructure + +### Swarm Cluster + +- [ ] Provision manager + worker VMs (Hetzner, DO, etc.). +- [ ] `docker swarm init --advertise-addr ` on manager #1. +- [ ] `docker swarm join-token {manager,worker}` → join additional nodes. +- [ ] `docker node ls` to verify — all nodes `Ready` and `Active`. +- [ ] Label nodes if you want placement constraints beyond the defaults. + +### Node Hardening (every node) + +- [ ] SSH: non-default port, key-only auth, no root login — see README §2. +- [ ] Firewall: allow 22 (or 2222), 80, 443 from CF IPs only; 2377/tcp, + 7946/tcp+udp, 4789/udp Swarm-nodes only; block the rest — see README §1. +- [ ] Install unattended-upgrades (or equivalent) for security patches. +- [ ] Disable password auth in `/etc/ssh/sshd_config`. +- [ ] Create the `deploy` user (`AllowUsers deploy` in sshd_config). + +### DNS + Cloudflare + +- [ ] Add A records for `api.`, `admin.` pointing to the LB + or manager IPs. Keep them **proxied** (orange cloud). +- [ ] Create a Cloudflare tunnel or enable "Authenticated Origin Pulls" if + you want to lock the origin to CF only. +- [ ] Firewall rule on the nodes: only accept 80/443 from Cloudflare IP ranges + (). +- [ ] Configure CF Access (or equivalent SSO) in front of admin panel if + exposing it publicly. + +--- + +## One-Time: External Services + +### Postgres (Neon) + +- [ ] Create project + database (`honeydue`). +- [ ] Create a dedicated DB user with least privilege — not the project owner. +- [ ] Enable IP allowlist, add every Swarm node's egress IP. +- [ ] Verify `DB_SSLMODE=require` works end-to-end. +- [ ] Turn on PITR (paid tier) or schedule automated `pg_dump` backups. +- [ ] Do one restore drill — boot a staging stack from a real backup. If you + haven't done this, you do not have backups. + +### Redis + +- Redis runs **inside** the stack on a named volume. No external setup + needed today. See README §11 — this is an accepted SPOF. +- [ ] If you move Redis external (Upstash, Dragonfly Cloud): update + `REDIS_URL` in `prod.env`, remove the `redis` service + volume from + the stack. + +### Backblaze B2 (or MinIO) + +Skip this section if you're running a single-node prod and are OK with +uploads on a local volume. Required for multi-replica prod — see README §8. + +- [ ] Create B2 account + bucket (private). +- [ ] Create a **scoped** application key bound to that single bucket — + not the master key. +- [ ] Set lifecycle rules: keep only the current version of each file, + or whatever matches your policy. +- [ ] Populate `B2_ENDPOINT`, `B2_KEY_ID`, `B2_APP_KEY`, `B2_BUCKET_NAME` + in `deploy/prod.env`. Optionally set `B2_USE_SSL` and `B2_REGION`. +- [ ] Verify uploads round-trip across replicas after the first deploy + (upload a file via client A → fetch via client B in a different session). + +### APNS (Apple Push) + +- [ ] Create an APNS auth key (`.p8`) in the Apple Developer portal. +- [ ] Save to `deploy/secrets/apns_auth_key.p8` — the script enforces it + contains a real `-----BEGIN PRIVATE KEY-----` block. +- [ ] Fill `APNS_AUTH_KEY_ID`, `APNS_TEAM_ID`, `APNS_TOPIC` (bundle ID) in + `deploy/prod.env`. +- [ ] Decide `APNS_USE_SANDBOX` / `APNS_PRODUCTION` based on build target. + +### FCM (Android Push) + +- [ ] Create Firebase project + legacy server key (or migrate to HTTP v1 — + the code currently uses the legacy server key). +- [ ] Save to `deploy/secrets/fcm_server_key.txt`. + +### SMTP (Email) + +- [ ] Provision SMTP credentials (Gmail app password, SES, Postmark, etc.). +- [ ] Fill `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER`, + `DEFAULT_FROM_EMAIL`, `EMAIL_USE_TLS` in `deploy/prod.env`. +- [ ] Save the password to `deploy/secrets/email_host_password.txt`. +- [ ] Verify SPF, DKIM, DMARC on the sending domain if you care about + deliverability. + +### Registry (GHCR / other) + +- [ ] Create a personal access token with `write:packages` + `read:packages`. +- [ ] Fill `REGISTRY`, `REGISTRY_NAMESPACE`, `REGISTRY_USERNAME`, + `REGISTRY_TOKEN` in `deploy/registry.env`. +- [ ] Rotate the token on a schedule (quarterly at minimum). + +### Apple / Google IAP (optional) + +- [ ] Apple: create App Store Connect API key, fill the `APPLE_IAP_*` vars. +- [ ] Google: create a service account with Play Developer API access, + store JSON at a path referenced by `GOOGLE_IAP_SERVICE_ACCOUNT_PATH`. + +--- + +## Recurring Operations + +### Secret Rotation + +After any compromise, annually at minimum, and when a team member leaves: + +1. Generate the new value (e.g. `openssl rand -base64 32 > deploy/secrets/secret_key.txt`). +2. `./.deploy_prod` — creates a new versioned Swarm secret and redeploys + services to pick it up. +3. The old secret lingers until `SECRET_KEEP_VERSIONS` bumps it out (see + README "Secret Versioning & Pruning"). +4. For external creds (Neon, B2, APNS, etc.) rotate at the provider first, + update the local secret file, then redeploy. + +### Backup Drills + +- [ ] Quarterly: pull a Neon backup, restore to a scratch project, boot a + staging stack against it, verify login + basic reads. +- [ ] Monthly: spot-check that B2 objects are actually present and the + app key still works. +- [ ] After any schema change: confirm PITR coverage includes the new + columns before relying on it. + +### Certificate Management + +- TLS is terminated by Cloudflare today, so there are no origin certs to + renew. If you ever move TLS on-origin (Traefik, Caddy), automate renewal + — don't add it to this list and expect it to happen. + +### Multi-Arch Builds + +`./.deploy_prod` builds for the host arch. If target ≠ host: + +- [ ] Enable buildx: `docker buildx create --use`. +- [ ] Install QEMU: `docker run --privileged --rm tonistiigi/binfmt --install all`. +- [ ] Build + push images manually per target platform. +- [ ] Run `SKIP_BUILD=1 ./.deploy_prod` so the script just deploys. + +### Node Maintenance / Rolling Upgrades + +- [ ] `docker node update --availability drain ` before OS upgrades. +- [ ] Reboot, verify, then `docker node update --availability active `. +- [ ] Re-converge with `docker stack deploy -c swarm-stack.prod.yml honeydue`. + +--- + +## Incident Response + +### Redis Node Dies + +Named volume is per-node and doesn't follow. Accept the loss: + +1. Let Swarm reschedule Redis on a new node. +2. In-flight Asynq jobs are lost; retry semantics cover most of them. +3. Scheduled cron events fire again on the next tick (hourly for smart + reminders and daily digest; daily for onboarding + cleanup). +4. Cache repopulates on first request. + +### Deploy Rolled Back Automatically + +`./.deploy_prod` triggers `docker service rollback` on every service if +`DEPLOY_HEALTHCHECK_URL` fails. Diagnose with: + +```bash +ssh docker stack services honeydue +ssh docker service logs --tail 200 honeydue_api +# Or open an SSH tunnel to Dozzle: ssh -L 9999:127.0.0.1:9999 +``` + +### Lost Ability to Deploy + +- Registry token revoked → regenerate, update `deploy/registry.env`, re-run. +- Manager host key changed → verify legitimacy, update `~/.ssh/known_hosts`. +- All secrets accidentally pruned → restore the `deploy/secrets/*` files + locally and redeploy; new Swarm secret versions will be created. + +--- + +## Known Gaps (Future Work) + +- No dedicated `cmd/migrate` binary — migrations run at API boot (see + README §10). Large schema changes still need manual coordination. +- `asynq.Scheduler` has no leader election; `WORKER_REPLICAS` must stay 1 + until we migrate to `asynq.PeriodicTaskManager` (README §9). +- No Prometheus / Grafana / alerting in the stack. `/metrics` is exposed + on the API but nothing scrapes it. +- No automated TLS renewal on-origin — add if you ever move off Cloudflare. +- No staging environment wired to the deploy script — `DEPLOY_TAG=` + is the closest thing. A proper staging flow is future work. diff --git a/deploy/swarm-stack.prod.yml b/deploy/swarm-stack.prod.yml index bdb6c32..7f41a12 100644 --- a/deploy/swarm-stack.prod.yml +++ b/deploy/swarm-stack.prod.yml @@ -3,7 +3,7 @@ version: "3.8" services: redis: image: redis:7-alpine - command: redis-server --appendonly yes --appendfsync everysec + command: redis-server --appendonly yes --appendfsync everysec --maxmemory 200mb --maxmemory-policy allkeys-lru volumes: - redis_data:/data healthcheck: @@ -18,6 +18,13 @@ services: delay: 5s placement: max_replicas_per_node: 1 + resources: + limits: + cpus: "0.50" + memory: 256M + reservations: + cpus: "0.10" + memory: 64M networks: - honeydue-network @@ -67,6 +74,17 @@ services: STORAGE_MAX_FILE_SIZE: "${STORAGE_MAX_FILE_SIZE}" STORAGE_ALLOWED_TYPES: "${STORAGE_ALLOWED_TYPES}" + # S3-compatible object storage (Backblaze B2, MinIO). When all B2_* vars + # are set, uploads/media are stored in the bucket and the local volume + # mount becomes a no-op fallback. Required for multi-replica prod — + # without it uploads only exist on one node. + B2_ENDPOINT: "${B2_ENDPOINT}" + B2_KEY_ID: "${B2_KEY_ID}" + B2_APP_KEY: "${B2_APP_KEY}" + B2_BUCKET_NAME: "${B2_BUCKET_NAME}" + B2_USE_SSL: "${B2_USE_SSL}" + B2_REGION: "${B2_REGION}" + FEATURE_PUSH_ENABLED: "${FEATURE_PUSH_ENABLED}" FEATURE_EMAIL_ENABLED: "${FEATURE_EMAIL_ENABLED}" FEATURE_WEBHOOKS_ENABLED: "${FEATURE_WEBHOOKS_ENABLED}" @@ -86,6 +104,7 @@ services: APPLE_IAP_SANDBOX: "${APPLE_IAP_SANDBOX}" GOOGLE_IAP_SERVICE_ACCOUNT_PATH: "${GOOGLE_IAP_SERVICE_ACCOUNT_PATH}" GOOGLE_IAP_PACKAGE_NAME: "${GOOGLE_IAP_PACKAGE_NAME}" + stop_grace_period: 60s command: - /bin/sh - -lc @@ -128,6 +147,13 @@ services: parallelism: 1 delay: 5s order: stop-first + resources: + limits: + cpus: "1.00" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M networks: - honeydue-network @@ -142,10 +168,12 @@ services: PORT: "3000" HOSTNAME: "0.0.0.0" NEXT_PUBLIC_API_URL: "${NEXT_PUBLIC_API_URL}" + stop_grace_period: 60s healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/admin/"] + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/api/health"] interval: 30s timeout: 10s + start_period: 20s retries: 3 deploy: replicas: ${ADMIN_REPLICAS} @@ -160,6 +188,13 @@ services: parallelism: 1 delay: 5s order: stop-first + resources: + limits: + cpus: "0.50" + memory: 384M + reservations: + cpus: "0.10" + memory: 128M networks: - honeydue-network @@ -201,6 +236,7 @@ services: FEATURE_ONBOARDING_EMAILS_ENABLED: "${FEATURE_ONBOARDING_EMAILS_ENABLED}" FEATURE_PDF_REPORTS_ENABLED: "${FEATURE_PDF_REPORTS_ENABLED}" FEATURE_WORKER_ENABLED: "${FEATURE_WORKER_ENABLED}" + stop_grace_period: 60s command: - /bin/sh - -lc @@ -222,6 +258,12 @@ services: target: fcm_server_key - source: ${APNS_AUTH_KEY_SECRET} target: apns_auth_key + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:6060/health"] + interval: 30s + timeout: 10s + start_period: 15s + retries: 3 deploy: replicas: ${WORKER_REPLICAS} restart_policy: @@ -235,16 +277,28 @@ services: parallelism: 1 delay: 5s order: stop-first + resources: + limits: + cpus: "1.00" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M networks: - honeydue-network dozzle: + # NOTE: Dozzle exposes the full Docker log stream with no built-in auth. + # Bound to manager loopback only — access via SSH tunnel: + # ssh -L ${DOZZLE_PORT}:127.0.0.1:${DOZZLE_PORT} + # Then browse http://localhost:${DOZZLE_PORT} image: amir20/dozzle:latest ports: - target: 8080 published: ${DOZZLE_PORT} protocol: tcp - mode: ingress + mode: host + host_ip: 127.0.0.1 environment: DOZZLE_NO_ANALYTICS: "true" volumes: @@ -257,6 +311,13 @@ services: placement: constraints: - node.role == manager + resources: + limits: + cpus: "0.25" + memory: 128M + reservations: + cpus: "0.05" + memory: 32M networks: - honeydue-network diff --git a/internal/database/database.go b/internal/database/database.go index 1e7a0cd..52fb7af 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,6 +1,7 @@ package database import ( + "context" "fmt" "time" @@ -15,6 +16,11 @@ import ( "github.com/treytartt/honeydue-api/internal/models" ) +// migrationAdvisoryLockKey is the pg_advisory_lock key that serializes +// Migrate() across API replicas booting in parallel. Value is arbitrary but +// stable ("hdmg" as bytes = honeydue migration). +const migrationAdvisoryLockKey int64 = 0x68646d67 + // zerologGormWriter adapts zerolog for GORM's logger interface type zerologGormWriter struct{} @@ -121,6 +127,54 @@ func Paginate(page, pageSize int) func(db *gorm.DB) *gorm.DB { } } +// MigrateWithLock runs Migrate() under a Postgres session-level advisory lock +// so that multiple API replicas booting in parallel don't race on AutoMigrate. +// On non-Postgres dialects (sqlite in tests) it falls through to Migrate(). +func MigrateWithLock() error { + if db == nil { + return fmt.Errorf("database not initialised") + } + if db.Dialector.Name() != "postgres" { + return Migrate() + } + + sqlDB, err := db.DB() + if err != nil { + return fmt.Errorf("get underlying sql.DB: %w", err) + } + + // Give ourselves up to 5 min to acquire the lock — long enough for a + // slow migration on a peer replica, short enough to fail fast if Postgres + // is hung. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + conn, err := sqlDB.Conn(ctx) + if err != nil { + return fmt.Errorf("acquire dedicated migration connection: %w", err) + } + defer conn.Close() + + log.Info().Int64("lock_key", migrationAdvisoryLockKey).Msg("Acquiring migration advisory lock...") + if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", migrationAdvisoryLockKey); err != nil { + return fmt.Errorf("pg_advisory_lock: %w", err) + } + log.Info().Msg("Migration advisory lock acquired") + + defer func() { + // Unlock with a fresh context — the outer ctx may have expired. + unlockCtx, unlockCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer unlockCancel() + if _, err := conn.ExecContext(unlockCtx, "SELECT pg_advisory_unlock($1)", migrationAdvisoryLockKey); err != nil { + log.Warn().Err(err).Msg("Failed to release migration advisory lock (session close will also release)") + } else { + log.Info().Msg("Migration advisory lock released") + } + }() + + return Migrate() +} + // Migrate runs database migrations for all models func Migrate() error { log.Info().Msg("Running database migrations...")