Migrate prod deploy from Swarm to K3s; add full deployment book

Infrastructure:
- Stack now runs on K3s v1.34.6 HA (3 Hetzner CX33 nodes as managers)
- Traefik DaemonSet + hostNetwork replaces Caddy + ingress mesh
- All manifests in deploy-k3s/manifests/; Swarm config (deploy/) kept
  temporarily for reference

Bug fixes surfaced during migration:
- Dockerfile: golang:1.24-alpine -> 1.25-alpine (go.mod requires 1.25)
- cache_service.go: remove sync.Once reassignment from inside Do()
  callback (was causing 'unlock of unlocked mutex' fatal after
  Redis Ping failure)
- router.go: relax CSP from 'default-src none' to 'default-src self'
  + allowlist fonts.googleapis.com so the marketing landing page CSS
  actually loads in browsers
- deploy/scripts/deploy_prod.sh: use docker buildx with
  --platform linux/amd64 so arm64 (Apple Silicon) dev machines produce
  images runnable on x86_64 Hetzner nodes; fix array expansion under
  set -u
- deploy/swarm-stack.prod.yml: fix secret source references to use
  top-level aliases (the '\${X_SECRET}' form never actually resolved);
  dozzle ports: long-form host_ip is rejected by Swarm, switched to
  short-form (bound to 0.0.0.0 with UFW-based loopback restriction);
  worker replicas 2 -> 1 (Asynq scheduler singleton)
- deploy-k3s/manifests/admin/deployment.yaml: probe path '/admin/' -> '/'
  (Next.js serves at root; /admin/ returned 404 and killed pods);
  startupProbe failureThreshold 12 -> 24
- deploy-k3s/manifests/pod-disruption-budgets.yaml: worker minAvailable
  1 -> 0 (singleton)
- deploy-k3s/manifests/api/deployment.yaml: startupProbe failureThreshold
  12 -> 48 (MigrateWithLock serializes across 3 replicas on first-boot;
  real startup takes up to 240s)
- .gitignore: tighten 'api' -> '/api' (was matching deploy-k3s/manifests/api/
  and admin/src/app/api/*, hiding legitimate files)

New files:
- deploy-k3s/manifests/traefik-helmchartconfig.yaml: DaemonSet +
  hostNetwork override for k3s-bundled Traefik
- deploy-k3s/manifests/ingress/ingress-simple.yaml: plain Ingress
  without TLS (CF Flexible SSL) and without middleware
- deploy-k3s/MIGRATION_NOTES.md: operator-facing migration log

Documentation:
- docs/deployment/ — full deployment book, 26 files, ~42k words:
  - Part I Overview, infrastructure, orchestrator choice (Ch 0-2)
  - Part II Networking, firewall, Cloudflare (Ch 3-4, 13)
  - Part III Security, Traefik ingress (Ch 5-6)
  - Part IV Services, DB, storage, secrets, registry (Ch 7-11)
  - Part V Data flow, deploy process, observability, failures, runbook
    (Ch 12, 14-17)
  - Part VI Cost, Swarm postmortem, roadmap (Ch 18-20)
  - Appendices: glossary, kubectl cheat sheet, file locations,
    consolidated citations
- README.md: Production Deployment section replaced with pointer to
  the book; Go version bumped to 1.25

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

deploy/scripts/deploy_prod.sh

@@ -248,13 +248,15 @@ Images that would be built and pushed:
   ${ADMIN_IMAGE}
 
 Replicas:
+  caddy:  3 (one per node)
   api:    ${API_REPLICAS:-3}
-  worker: ${WORKER_REPLICAS:-2}
+  worker: ${WORKER_REPLICAS:-1}
   admin:  ${ADMIN_REPLICAS:-1}
 
 Published ports:
-  api:    ${API_PORT:-8000} (ingress)
-  admin:  ${ADMIN_PORT:-3000} (ingress)
+  caddy:  80, 443 (ingress — public)
+  api:    internal only (proxied by caddy)
+  admin:  internal only (proxied by caddy)
   dozzle: ${DOZZLE_PORT:-9999} (manager loopback only — SSH tunnel required)
 
 Versioned secrets that would be created on this deploy:
@@ -264,6 +266,9 @@ Versioned secrets that would be created on this deploy:
   ${DEPLOY_STACK_NAME}_fcm_server_key_<deploy_id>
   ${DEPLOY_STACK_NAME}_apns_auth_key_<deploy_id>
 
+Versioned configs that would be created on this deploy:
+  ${DEPLOY_STACK_NAME}_caddyfile_<deploy_id>
+
 No changes made. Re-run without DRY_RUN=1 to deploy.
 =================================================
 
@@ -289,27 +294,54 @@ if [[ "${SKIP_BUILD}" != "1" ]]; then
   log "Logging in to ${REGISTRY}"
   printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USERNAME}" --password-stdin >/dev/null
 
-  log "Building API image ${API_IMAGE}"
-  docker build --target api -t "${API_IMAGE}" "${REPO_DIR}"
-  log "Building Worker image ${WORKER_IMAGE}"
-  docker build --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}"
-  log "Building Admin image ${ADMIN_IMAGE}"
-  docker build --target admin -t "${ADMIN_IMAGE}" "${REPO_DIR}"
+  # Target platform for Swarm nodes. Hetzner CX is x86_64; override via
+  # BUILD_PLATFORM=linux/arm64 if you move to ARM (Ampere) hosts.
+  BUILD_PLATFORM="${BUILD_PLATFORM:-linux/amd64}"
+  log "Build platform: ${BUILD_PLATFORM} (dev host may differ; buildx cross-compiles)"
 
-  log "Pushing deploy images"
-  docker push "${API_IMAGE}"
-  docker push "${WORKER_IMAGE}"
-  docker push "${ADMIN_IMAGE}"
-
-  if [[ "${PUSH_LATEST_TAG}" == "true" ]]; then
-    log "Updating :latest tags"
-    docker tag "${API_IMAGE}" "${REGISTRY_PREFIX}/honeydue-api:latest"
-    docker tag "${WORKER_IMAGE}" "${REGISTRY_PREFIX}/honeydue-worker:latest"
-    docker tag "${ADMIN_IMAGE}" "${REGISTRY_PREFIX}/honeydue-admin:latest"
-    docker push "${REGISTRY_PREFIX}/honeydue-api:latest"
-    docker push "${REGISTRY_PREFIX}/honeydue-worker:latest"
-    docker push "${REGISTRY_PREFIX}/honeydue-admin:latest"
+  # Ensure a buildx builder exists and is usable
+  if ! docker buildx inspect honeydue-builder >/dev/null 2>&1; then
+    log "Creating buildx builder 'honeydue-builder'"
+    docker buildx create --name honeydue-builder --use >/dev/null
+  else
+    docker buildx use honeydue-builder >/dev/null
   fi
+  docker buildx inspect --bootstrap >/dev/null
+
+  build_and_push() {
+    local target="$1"
+    local image="$2"
+    shift 2
+
+    local tag_args=(-t "${image}")
+    while (( $# > 0 )); do
+      tag_args+=(-t "$1")
+      shift
+    done
+
+    log "Building + pushing ${target} image for ${BUILD_PLATFORM}: ${image}"
+    docker buildx build \
+      --platform "${BUILD_PLATFORM}" \
+      --target "${target}" \
+      "${tag_args[@]}" \
+      --push \
+      "${REPO_DIR}"
+  }
+
+  api_extra=()
+  worker_extra=()
+  admin_extra=()
+  if [[ "${PUSH_LATEST_TAG}" == "true" ]]; then
+    api_extra=("${REGISTRY_PREFIX}/honeydue-api:latest")
+    worker_extra=("${REGISTRY_PREFIX}/honeydue-worker:latest")
+    admin_extra=("${REGISTRY_PREFIX}/honeydue-admin:latest")
+  fi
+
+  # ${arr[@]+"${arr[@]}"} safely expands to nothing when the array is empty
+  # under `set -u` — avoids "unbound variable" on bash arrays.
+  build_and_push api    "${API_IMAGE}"    ${api_extra[@]+"${api_extra[@]}"}
+  build_and_push worker "${WORKER_IMAGE}" ${worker_extra[@]+"${worker_extra[@]}"}
+  build_and_push admin  "${ADMIN_IMAGE}"  ${admin_extra[@]+"${admin_extra[@]}"}
 else
   warn "SKIP_BUILD=1 set. Using prebuilt images for tag: ${DEPLOY_TAG}"
 fi
@@ -322,6 +354,12 @@ SECRET_KEY_SECRET="${DEPLOY_STACK_NAME}_secret_key_${DEPLOY_ID}"
 EMAIL_HOST_PASSWORD_SECRET="${DEPLOY_STACK_NAME}_email_host_password_${DEPLOY_ID}"
 FCM_SERVER_KEY_SECRET="${DEPLOY_STACK_NAME}_fcm_server_key_${DEPLOY_ID}"
 APNS_AUTH_KEY_SECRET="${DEPLOY_STACK_NAME}_apns_auth_key_${DEPLOY_ID}"
+CADDYFILE_CONFIG="${DEPLOY_STACK_NAME}_caddyfile_${DEPLOY_ID}"
+
+CADDYFILE_SRC="${DEPLOY_DIR}/Caddyfile"
+if [[ ! -f "${CADDYFILE_SRC}" ]]; then
+  die "Missing required file: ${CADDYFILE_SRC}"
+fi
 
 TMP_DIR="$(mktemp -d)"
 cleanup() {
@@ -332,6 +370,7 @@ trap cleanup EXIT
 cp "${STACK_TEMPLATE}" "${TMP_DIR}/swarm-stack.prod.yml"
 cp "${PROD_ENV}" "${TMP_DIR}/prod.env"
 cp "${REGISTRY_ENV}" "${TMP_DIR}/registry.env"
+cp "${CADDYFILE_SRC}" "${TMP_DIR}/Caddyfile"
 mkdir -p "${TMP_DIR}/secrets"
 cp "${SECRET_POSTGRES}" "${TMP_DIR}/secrets/postgres_password.txt"
 cp "${SECRET_APP_KEY}" "${TMP_DIR}/secrets/secret_key.txt"
@@ -356,6 +395,7 @@ SECRET_KEY_SECRET=${SECRET_KEY_SECRET}
 EMAIL_HOST_PASSWORD_SECRET=${EMAIL_HOST_PASSWORD_SECRET}
 FCM_SERVER_KEY_SECRET=${FCM_SERVER_KEY_SECRET}
 APNS_AUTH_KEY_SECRET=${APNS_AUTH_KEY_SECRET}
+CADDYFILE_CONFIG=${CADDYFILE_CONFIG}
 EOF
 
 log "Uploading deploy bundle to ${SSH_TARGET}:${DEPLOY_REMOTE_DIR}"
@@ -364,6 +404,7 @@ scp "${SCP_OPTS[@]}" "${TMP_DIR}/swarm-stack.prod.yml" "${SSH_TARGET}:${DEPLOY_R
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/prod.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/prod.env"
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/registry.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/registry.env"
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/runtime.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/runtime.env"
+scp "${SCP_OPTS[@]}" "${TMP_DIR}/Caddyfile" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/Caddyfile"
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/postgres_password.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/postgres_password.txt"
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/secret_key.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/secret_key.txt"
 scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/email_host_password.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/email_host_password.txt"
@@ -397,6 +438,17 @@ create_secret() {
   fi
 }
 
+create_config() {
+  local name="$1"
+  local src="$2"
+  if docker config inspect "${name}" >/dev/null 2>&1; then
+    echo "[remote] config exists: ${name}"
+  else
+    docker config create "${name}" "${src}" >/dev/null
+    echo "[remote] created config: ${name}"
+  fi
+}
+
 printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USERNAME}" --password-stdin >/dev/null
 rm -f "${REMOTE_DIR}/registry.env"
 
@@ -406,6 +458,8 @@ create_secret "${EMAIL_HOST_PASSWORD_SECRET}" "${REMOTE_DIR}/secrets/email_host_
 create_secret "${FCM_SERVER_KEY_SECRET}" "${REMOTE_DIR}/secrets/fcm_server_key.txt"
 create_secret "${APNS_AUTH_KEY_SECRET}" "${REMOTE_DIR}/secrets/apns_auth_key.p8"
 
+create_config "${CADDYFILE_CONFIG}" "${REMOTE_DIR}/Caddyfile"
+
 rm -f "${REMOTE_DIR}/secrets/postgres_password.txt"
 rm -f "${REMOTE_DIR}/secrets/secret_key.txt"
 rm -f "${REMOTE_DIR}/secrets/email_host_password.txt"
@@ -455,18 +509,22 @@ while true; do
   sleep 10
 done
 
-log "Pruning old secret versions (keeping last ${SECRET_KEEP_VERSIONS})"
-ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}' '${SECRET_KEEP_VERSIONS}'" <<'EOF' || warn "Secret pruning reported errors (non-fatal)"
+log "Pruning old secret + config versions (keeping last ${SECRET_KEEP_VERSIONS})"
+ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}' '${SECRET_KEEP_VERSIONS}'" <<'EOF' || warn "Pruning reported errors (non-fatal)"
 set -euo pipefail
 
 STACK_NAME="$1"
 KEEP="$2"
 
 prune_prefix() {
-  local prefix="$1"
-  # List matching secrets with creation time, sorted newest-first.
+  local kind="$1"   # "secret" or "config"
+  local prefix="$2"
+  local ls_cmd rm_cmd
+  ls_cmd="docker ${kind} ls --format '{{.CreatedAt}}|{{.Name}}'"
+  rm_cmd="docker ${kind} rm"
+
   local all
-  all="$(docker secret ls --format '{{.CreatedAt}}|{{.Name}}' 2>/dev/null \
+  all="$(eval "${ls_cmd}" 2>/dev/null \
     | grep "|${prefix}_" \
     | sort -r \
     || true)"
@@ -477,7 +535,7 @@ prune_prefix() {
   local total
   total="$(printf '%s\n' "${all}" | wc -l | tr -d ' ')"
   if (( total <= KEEP )); then
-    echo "[cleanup] ${prefix}: ${total} version(s) — nothing to prune"
+    echo "[cleanup] ${kind}/${prefix}: ${total} version(s) — nothing to prune"
     return 0
   fi
 
@@ -486,16 +544,20 @@ prune_prefix() {
 
   while IFS= read -r name; do
     [[ -z "${name}" ]] && continue
-    if docker secret rm "${name}" >/dev/null 2>&1; then
-      echo "[cleanup] removed: ${name}"
+    if ${rm_cmd} "${name}" >/dev/null 2>&1; then
+      echo "[cleanup] removed ${kind}: ${name}"
     else
-      echo "[cleanup] in-use (kept): ${name}"
+      echo "[cleanup] in-use ${kind} (kept): ${name}"
     fi
   done <<< "${to_remove}"
 }
 
 for base in postgres_password secret_key email_host_password fcm_server_key apns_auth_key; do
-  prune_prefix "${STACK_NAME}_${base}"
+  prune_prefix secret "${STACK_NAME}_${base}"
+done
+
+for base in caddyfile; do
+  prune_prefix config "${STACK_NAME}_${base}"
 done
 EOF