Trey t
6f303dbbaa
Infrastructure:
- Stack now runs on K3s v1.34.6 HA (3 Hetzner CX33 nodes as managers)
- Traefik DaemonSet + hostNetwork replaces Caddy + ingress mesh
- All manifests in deploy-k3s/manifests/; Swarm config (deploy/) kept
temporarily for reference
Bug fixes surfaced during migration:
- Dockerfile: golang:1.24-alpine -> 1.25-alpine (go.mod requires 1.25)
- cache_service.go: remove sync.Once reassignment from inside Do()
callback (was causing 'unlock of unlocked mutex' fatal after
Redis Ping failure)
- router.go: relax CSP from 'default-src none' to 'default-src self'
+ allowlist fonts.googleapis.com so the marketing landing page CSS
actually loads in browsers
- deploy/scripts/deploy_prod.sh: use docker buildx with
--platform linux/amd64 so arm64 (Apple Silicon) dev machines produce
images runnable on x86_64 Hetzner nodes; fix array expansion under
set -u
- deploy/swarm-stack.prod.yml: fix secret source references to use
top-level aliases (the '\${X_SECRET}' form never actually resolved);
dozzle ports: long-form host_ip is rejected by Swarm, switched to
short-form (bound to 0.0.0.0 with UFW-based loopback restriction);
worker replicas 2 -> 1 (Asynq scheduler singleton)
- deploy-k3s/manifests/admin/deployment.yaml: probe path '/admin/' -> '/'
(Next.js serves at root; /admin/ returned 404 and killed pods);
startupProbe failureThreshold 12 -> 24
- deploy-k3s/manifests/pod-disruption-budgets.yaml: worker minAvailable
1 -> 0 (singleton)
- deploy-k3s/manifests/api/deployment.yaml: startupProbe failureThreshold
12 -> 48 (MigrateWithLock serializes across 3 replicas on first-boot;
real startup takes up to 240s)
- .gitignore: tighten 'api' -> '/api' (was matching deploy-k3s/manifests/api/
and admin/src/app/api/*, hiding legitimate files)
New files:
- deploy-k3s/manifests/traefik-helmchartconfig.yaml: DaemonSet +
hostNetwork override for k3s-bundled Traefik
- deploy-k3s/manifests/ingress/ingress-simple.yaml: plain Ingress
without TLS (CF Flexible SSL) and without middleware
- deploy-k3s/MIGRATION_NOTES.md: operator-facing migration log
Documentation:
- docs/deployment/ — full deployment book, 26 files, ~42k words:
- Part I Overview, infrastructure, orchestrator choice (Ch 0-2)
- Part II Networking, firewall, Cloudflare (Ch 3-4, 13)
- Part III Security, Traefik ingress (Ch 5-6)
- Part IV Services, DB, storage, secrets, registry (Ch 7-11)
- Part V Data flow, deploy process, observability, failures, runbook
(Ch 12, 14-17)
- Part VI Cost, Swarm postmortem, roadmap (Ch 18-20)
- Appendices: glossary, kubectl cheat sheet, file locations,
consolidated citations
- README.md: Production Deployment section replaced with pointer to
the book; Go version bumped to 1.25
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
415 lines
12 KiB
YAML
415 lines
12 KiB
YAML
version: "3.8"
|
|
|
|
services:
|
|
# Edge reverse proxy — the only service publishing :80/:443 publicly.
|
|
# Routes by Host header to internal `api` and `admin` services over the
|
|
# overlay network. Runs one replica per node via ingress mesh, so any node
|
|
# can terminate incoming traffic.
|
|
caddy:
|
|
image: caddy:2-alpine
|
|
ports:
|
|
- target: 80
|
|
published: 80
|
|
protocol: tcp
|
|
mode: ingress
|
|
- target: 443
|
|
published: 443
|
|
protocol: tcp
|
|
mode: ingress
|
|
configs:
|
|
- source: caddyfile
|
|
target: /etc/caddy/Caddyfile
|
|
mode: 0444
|
|
volumes:
|
|
- caddy_data:/data
|
|
- caddy_config:/config
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1/"]
|
|
interval: 30s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
deploy:
|
|
replicas: 3
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
update_config:
|
|
parallelism: 1
|
|
delay: 10s
|
|
order: start-first
|
|
rollback_config:
|
|
parallelism: 1
|
|
delay: 5s
|
|
order: stop-first
|
|
placement:
|
|
max_replicas_per_node: 1
|
|
resources:
|
|
limits:
|
|
cpus: "0.25"
|
|
memory: 128M
|
|
reservations:
|
|
cpus: "0.05"
|
|
memory: 32M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
redis:
|
|
image: redis:7-alpine
|
|
command: redis-server --appendonly yes --appendfsync everysec --maxmemory 200mb --maxmemory-policy allkeys-lru
|
|
volumes:
|
|
- redis_data:/data
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
deploy:
|
|
replicas: 1
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
placement:
|
|
max_replicas_per_node: 1
|
|
resources:
|
|
limits:
|
|
cpus: "0.50"
|
|
memory: 256M
|
|
reservations:
|
|
cpus: "0.10"
|
|
memory: 64M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
api:
|
|
image: ${API_IMAGE}
|
|
# No `ports:` block — Caddy edge service proxies to api:8000 over the
|
|
# overlay network. Port 8000 is never publicly exposed.
|
|
environment:
|
|
PORT: "8000"
|
|
DEBUG: "${DEBUG}"
|
|
ALLOWED_HOSTS: "${ALLOWED_HOSTS}"
|
|
CORS_ALLOWED_ORIGINS: "${CORS_ALLOWED_ORIGINS}"
|
|
TIMEZONE: "${TIMEZONE}"
|
|
BASE_URL: "${BASE_URL}"
|
|
ADMIN_PANEL_URL: "${ADMIN_PANEL_URL}"
|
|
|
|
DB_HOST: "${DB_HOST}"
|
|
DB_PORT: "${DB_PORT}"
|
|
POSTGRES_USER: "${POSTGRES_USER}"
|
|
POSTGRES_DB: "${POSTGRES_DB}"
|
|
DB_SSLMODE: "${DB_SSLMODE}"
|
|
DB_MAX_OPEN_CONNS: "${DB_MAX_OPEN_CONNS}"
|
|
DB_MAX_IDLE_CONNS: "${DB_MAX_IDLE_CONNS}"
|
|
DB_MAX_LIFETIME: "${DB_MAX_LIFETIME}"
|
|
|
|
REDIS_URL: "${REDIS_URL}"
|
|
REDIS_DB: "${REDIS_DB}"
|
|
|
|
EMAIL_HOST: "${EMAIL_HOST}"
|
|
EMAIL_PORT: "${EMAIL_PORT}"
|
|
EMAIL_HOST_USER: "${EMAIL_HOST_USER}"
|
|
DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL}"
|
|
EMAIL_USE_TLS: "${EMAIL_USE_TLS}"
|
|
|
|
APNS_AUTH_KEY_PATH: "/run/secrets/apns_auth_key"
|
|
APNS_AUTH_KEY_ID: "${APNS_AUTH_KEY_ID}"
|
|
APNS_TEAM_ID: "${APNS_TEAM_ID}"
|
|
APNS_TOPIC: "${APNS_TOPIC}"
|
|
APNS_USE_SANDBOX: "${APNS_USE_SANDBOX}"
|
|
APNS_PRODUCTION: "${APNS_PRODUCTION}"
|
|
|
|
STORAGE_UPLOAD_DIR: "${STORAGE_UPLOAD_DIR}"
|
|
STORAGE_BASE_URL: "${STORAGE_BASE_URL}"
|
|
STORAGE_MAX_FILE_SIZE: "${STORAGE_MAX_FILE_SIZE}"
|
|
STORAGE_ALLOWED_TYPES: "${STORAGE_ALLOWED_TYPES}"
|
|
|
|
# S3-compatible object storage (Backblaze B2, MinIO). When all B2_* vars
|
|
# are set, uploads/media are stored in the bucket and the local volume
|
|
# mount becomes a no-op fallback. Required for multi-replica prod —
|
|
# without it uploads only exist on one node.
|
|
B2_ENDPOINT: "${B2_ENDPOINT}"
|
|
B2_KEY_ID: "${B2_KEY_ID}"
|
|
B2_APP_KEY: "${B2_APP_KEY}"
|
|
B2_BUCKET_NAME: "${B2_BUCKET_NAME}"
|
|
B2_USE_SSL: "${B2_USE_SSL}"
|
|
B2_REGION: "${B2_REGION}"
|
|
|
|
FEATURE_PUSH_ENABLED: "${FEATURE_PUSH_ENABLED}"
|
|
FEATURE_EMAIL_ENABLED: "${FEATURE_EMAIL_ENABLED}"
|
|
FEATURE_WEBHOOKS_ENABLED: "${FEATURE_WEBHOOKS_ENABLED}"
|
|
FEATURE_ONBOARDING_EMAILS_ENABLED: "${FEATURE_ONBOARDING_EMAILS_ENABLED}"
|
|
FEATURE_PDF_REPORTS_ENABLED: "${FEATURE_PDF_REPORTS_ENABLED}"
|
|
FEATURE_WORKER_ENABLED: "${FEATURE_WORKER_ENABLED}"
|
|
|
|
APPLE_CLIENT_ID: "${APPLE_CLIENT_ID}"
|
|
APPLE_TEAM_ID: "${APPLE_TEAM_ID}"
|
|
GOOGLE_CLIENT_ID: "${GOOGLE_CLIENT_ID}"
|
|
GOOGLE_ANDROID_CLIENT_ID: "${GOOGLE_ANDROID_CLIENT_ID}"
|
|
GOOGLE_IOS_CLIENT_ID: "${GOOGLE_IOS_CLIENT_ID}"
|
|
APPLE_IAP_KEY_PATH: "${APPLE_IAP_KEY_PATH}"
|
|
APPLE_IAP_KEY_ID: "${APPLE_IAP_KEY_ID}"
|
|
APPLE_IAP_ISSUER_ID: "${APPLE_IAP_ISSUER_ID}"
|
|
APPLE_IAP_BUNDLE_ID: "${APPLE_IAP_BUNDLE_ID}"
|
|
APPLE_IAP_SANDBOX: "${APPLE_IAP_SANDBOX}"
|
|
GOOGLE_IAP_SERVICE_ACCOUNT_PATH: "${GOOGLE_IAP_SERVICE_ACCOUNT_PATH}"
|
|
GOOGLE_IAP_PACKAGE_NAME: "${GOOGLE_IAP_PACKAGE_NAME}"
|
|
|
|
# Seeded on first migration (idempotent — skipped if admin_users row exists)
|
|
ADMIN_EMAIL: "${ADMIN_EMAIL}"
|
|
ADMIN_PASSWORD: "${ADMIN_PASSWORD}"
|
|
stop_grace_period: 60s
|
|
command:
|
|
- /bin/sh
|
|
- -lc
|
|
- |
|
|
set -eu
|
|
export POSTGRES_PASSWORD="$$(cat /run/secrets/postgres_password)"
|
|
export SECRET_KEY="$$(cat /run/secrets/secret_key)"
|
|
export EMAIL_HOST_PASSWORD="$$(cat /run/secrets/email_host_password)"
|
|
export FCM_SERVER_KEY="$$(cat /run/secrets/fcm_server_key)"
|
|
exec /app/api
|
|
secrets:
|
|
- source: postgres_password
|
|
target: postgres_password
|
|
- source: secret_key
|
|
target: secret_key
|
|
- source: email_host_password
|
|
target: email_host_password
|
|
- source: fcm_server_key
|
|
target: fcm_server_key
|
|
- source: apns_auth_key
|
|
target: apns_auth_key
|
|
volumes:
|
|
- uploads:/app/uploads
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://127.0.0.1:8000/api/health/"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
# Single-replica AutoMigrate on a fresh DB takes ~90s; subsequent
|
|
# replicas are ~2s (idempotent). 180s gives honest headroom for the
|
|
# first replica to finish, without masking cascade failures.
|
|
start_period: 180s
|
|
retries: 3
|
|
deploy:
|
|
replicas: ${API_REPLICAS}
|
|
# DNS round-robin instead of VIP. VIP's kernel IPVS state can go stale
|
|
# during replica churn (rolling updates, task restarts), causing
|
|
# intermittent i/o timeouts from clients on the overlay network (Caddy).
|
|
# dnsrr resolves to live task IPs directly and bypasses IPVS.
|
|
endpoint_mode: dnsrr
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
update_config:
|
|
parallelism: 1
|
|
delay: 10s
|
|
order: start-first
|
|
rollback_config:
|
|
parallelism: 1
|
|
delay: 5s
|
|
order: stop-first
|
|
resources:
|
|
limits:
|
|
cpus: "1.00"
|
|
memory: 512M
|
|
reservations:
|
|
cpus: "0.25"
|
|
memory: 128M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
admin:
|
|
image: ${ADMIN_IMAGE}
|
|
# No `ports:` block — reached via Caddy on admin.myhoneydue.com using
|
|
# Swarm's embedded DNS and default VIP endpoint_mode.
|
|
environment:
|
|
PORT: "3000"
|
|
HOSTNAME: "0.0.0.0"
|
|
NEXT_PUBLIC_API_URL: "${NEXT_PUBLIC_API_URL}"
|
|
stop_grace_period: 60s
|
|
healthcheck:
|
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000/api/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 20s
|
|
retries: 3
|
|
deploy:
|
|
replicas: ${ADMIN_REPLICAS}
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
update_config:
|
|
parallelism: 1
|
|
delay: 10s
|
|
order: start-first
|
|
rollback_config:
|
|
parallelism: 1
|
|
delay: 5s
|
|
order: stop-first
|
|
resources:
|
|
limits:
|
|
cpus: "0.50"
|
|
memory: 384M
|
|
reservations:
|
|
cpus: "0.10"
|
|
memory: 128M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
worker:
|
|
image: ${WORKER_IMAGE}
|
|
environment:
|
|
DB_HOST: "${DB_HOST}"
|
|
DB_PORT: "${DB_PORT}"
|
|
POSTGRES_USER: "${POSTGRES_USER}"
|
|
POSTGRES_DB: "${POSTGRES_DB}"
|
|
DB_SSLMODE: "${DB_SSLMODE}"
|
|
DB_MAX_OPEN_CONNS: "${DB_MAX_OPEN_CONNS}"
|
|
DB_MAX_IDLE_CONNS: "${DB_MAX_IDLE_CONNS}"
|
|
DB_MAX_LIFETIME: "${DB_MAX_LIFETIME}"
|
|
|
|
REDIS_URL: "${REDIS_URL}"
|
|
REDIS_DB: "${REDIS_DB}"
|
|
|
|
EMAIL_HOST: "${EMAIL_HOST}"
|
|
EMAIL_PORT: "${EMAIL_PORT}"
|
|
EMAIL_HOST_USER: "${EMAIL_HOST_USER}"
|
|
DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL}"
|
|
EMAIL_USE_TLS: "${EMAIL_USE_TLS}"
|
|
|
|
APNS_AUTH_KEY_PATH: "/run/secrets/apns_auth_key"
|
|
APNS_AUTH_KEY_ID: "${APNS_AUTH_KEY_ID}"
|
|
APNS_TEAM_ID: "${APNS_TEAM_ID}"
|
|
APNS_TOPIC: "${APNS_TOPIC}"
|
|
APNS_USE_SANDBOX: "${APNS_USE_SANDBOX}"
|
|
APNS_PRODUCTION: "${APNS_PRODUCTION}"
|
|
|
|
TASK_REMINDER_HOUR: "${TASK_REMINDER_HOUR}"
|
|
OVERDUE_REMINDER_HOUR: "${OVERDUE_REMINDER_HOUR}"
|
|
DAILY_DIGEST_HOUR: "${DAILY_DIGEST_HOUR}"
|
|
|
|
FEATURE_PUSH_ENABLED: "${FEATURE_PUSH_ENABLED}"
|
|
FEATURE_EMAIL_ENABLED: "${FEATURE_EMAIL_ENABLED}"
|
|
FEATURE_WEBHOOKS_ENABLED: "${FEATURE_WEBHOOKS_ENABLED}"
|
|
FEATURE_ONBOARDING_EMAILS_ENABLED: "${FEATURE_ONBOARDING_EMAILS_ENABLED}"
|
|
FEATURE_PDF_REPORTS_ENABLED: "${FEATURE_PDF_REPORTS_ENABLED}"
|
|
FEATURE_WORKER_ENABLED: "${FEATURE_WORKER_ENABLED}"
|
|
stop_grace_period: 60s
|
|
command:
|
|
- /bin/sh
|
|
- -lc
|
|
- |
|
|
set -eu
|
|
export POSTGRES_PASSWORD="$$(cat /run/secrets/postgres_password)"
|
|
export SECRET_KEY="$$(cat /run/secrets/secret_key)"
|
|
export EMAIL_HOST_PASSWORD="$$(cat /run/secrets/email_host_password)"
|
|
export FCM_SERVER_KEY="$$(cat /run/secrets/fcm_server_key)"
|
|
exec /app/worker
|
|
secrets:
|
|
- source: postgres_password
|
|
target: postgres_password
|
|
- source: secret_key
|
|
target: secret_key
|
|
- source: email_host_password
|
|
target: email_host_password
|
|
- source: fcm_server_key
|
|
target: fcm_server_key
|
|
- source: apns_auth_key
|
|
target: apns_auth_key
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://127.0.0.1:6060/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
start_period: 15s
|
|
retries: 3
|
|
deploy:
|
|
replicas: ${WORKER_REPLICAS}
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
update_config:
|
|
parallelism: 1
|
|
delay: 10s
|
|
order: start-first
|
|
rollback_config:
|
|
parallelism: 1
|
|
delay: 5s
|
|
order: stop-first
|
|
resources:
|
|
limits:
|
|
cpus: "1.00"
|
|
memory: 512M
|
|
reservations:
|
|
cpus: "0.25"
|
|
memory: 128M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
dozzle:
|
|
# NOTE: Dozzle exposes the full Docker log stream with no built-in auth.
|
|
# Bound to manager loopback only — access via SSH tunnel:
|
|
# ssh -L ${DOZZLE_PORT}:127.0.0.1:${DOZZLE_PORT} <manager>
|
|
# Then browse http://localhost:${DOZZLE_PORT}
|
|
image: amir20/dozzle:latest
|
|
# Bind to loopback only on the manager. Swarm's long-form port spec
|
|
# rejects `host_ip`, so we use the short form — 127.0.0.1:<port>:8080.
|
|
# Access via SSH tunnel: ssh -L ${DOZZLE_PORT}:127.0.0.1:${DOZZLE_PORT} <manager>
|
|
ports:
|
|
- "127.0.0.1:${DOZZLE_PORT}:8080"
|
|
environment:
|
|
DOZZLE_NO_ANALYTICS: "true"
|
|
volumes:
|
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
deploy:
|
|
replicas: 1
|
|
restart_policy:
|
|
condition: any
|
|
delay: 5s
|
|
placement:
|
|
constraints:
|
|
- node.role == manager
|
|
resources:
|
|
limits:
|
|
cpus: "0.25"
|
|
memory: 128M
|
|
reservations:
|
|
cpus: "0.05"
|
|
memory: 32M
|
|
networks:
|
|
- honeydue-network
|
|
|
|
volumes:
|
|
redis_data:
|
|
uploads:
|
|
caddy_data:
|
|
caddy_config:
|
|
|
|
networks:
|
|
honeydue-network:
|
|
driver: overlay
|
|
driver_opts:
|
|
encrypted: "true"
|
|
|
|
configs:
|
|
caddyfile:
|
|
external: true
|
|
name: ${CADDYFILE_CONFIG}
|
|
|
|
secrets:
|
|
postgres_password:
|
|
external: true
|
|
name: ${POSTGRES_PASSWORD_SECRET}
|
|
secret_key:
|
|
external: true
|
|
name: ${SECRET_KEY_SECRET}
|
|
email_host_password:
|
|
external: true
|
|
name: ${EMAIL_HOST_PASSWORD_SECRET}
|
|
fcm_server_key:
|
|
external: true
|
|
name: ${FCM_SERVER_KEY_SECRET}
|
|
apns_auth_key:
|
|
external: true
|
|
name: ${APNS_AUTH_KEY_SECRET}
|