honeyDueAPI/deploy/scripts/deploy_prod.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)"

STACK_TEMPLATE="${DEPLOY_DIR}/swarm-stack.prod.yml"
CLUSTER_ENV="${DEPLOY_DIR}/cluster.env"
REGISTRY_ENV="${DEPLOY_DIR}/registry.env"
PROD_ENV="${DEPLOY_DIR}/prod.env"

SECRET_POSTGRES="${DEPLOY_DIR}/secrets/postgres_password.txt"
SECRET_APP_KEY="${DEPLOY_DIR}/secrets/secret_key.txt"
SECRET_EMAIL_PASS="${DEPLOY_DIR}/secrets/email_host_password.txt"
SECRET_FCM_KEY="${DEPLOY_DIR}/secrets/fcm_server_key.txt"
SECRET_APNS_KEY="${DEPLOY_DIR}/secrets/apns_auth_key.p8"

SKIP_BUILD="${SKIP_BUILD:-0}"
SKIP_HEALTHCHECK="${SKIP_HEALTHCHECK:-0}"
DRY_RUN="${DRY_RUN:-0}"
SECRET_KEEP_VERSIONS="${SECRET_KEEP_VERSIONS:-3}"

log() {
  printf '[deploy] %s\n' "$*"
}

warn() {
  printf '[deploy][warn] %s\n' "$*" >&2
}

die() {
  printf '[deploy][error] %s\n' "$*" >&2
  exit 1
}

require_cmd() {
  command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
}

contains_placeholder() {
  local value="$1"
  [[ -z "${value}" ]] && return 0
  local lowered
  lowered="$(printf '%s' "${value}" | tr '[:upper:]' '[:lower:]')"
  case "${lowered}" in
    *changeme*|*replace_me*|*example.com*|*your-*|*todo*|*fill_me*|*paste_here*)
      return 0
      ;;
    *)
      return 1
      ;;
  esac
}

ensure_file_from_example() {
  local path="$1"
  local example="${path}.example"
  if [[ -f "${path}" ]]; then
    return
  fi
  if [[ -f "${example}" ]]; then
    cp "${example}" "${path}"
    die "Created ${path} from template. Fill it in and rerun."
  fi
  die "Missing required file: ${path}"
}

require_var() {
  local name="$1"
  local value="${!name:-}"
  [[ -n "${value}" ]] || die "Missing required value: ${name}"
  if contains_placeholder "${value}"; then
    die "Value still uses placeholder text: ${name}=${value}"
  fi
}

require_secret_file() {
  local path="$1"
  local label="$2"
  ensure_file_from_example "${path}"
  local contents
  contents="$(tr -d '\r' < "${path}" | sed 's/[[:space:]]*$//')"
  [[ -n "${contents}" ]] || die "Secret file is empty: ${path}"
  if contains_placeholder "${contents}"; then
    die "Secret file still has placeholder text (${label}): ${path}"
  fi
}

print_usage() {
  cat <<'EOF'
Usage:
  ./.deploy_prod

Optional environment flags:
  DRY_RUN=1                  Print the deployment plan and exit without changes.
  SKIP_BUILD=1               Deploy existing image tags without rebuilding/pushing.
  SKIP_HEALTHCHECK=1         Skip final HTTP health check.
  DEPLOY_TAG=<tag>           Override image tag (default: git short sha).
  PUSH_LATEST_TAG=true|false Also tag/push :latest (default: false — SHA only).
  SECRET_KEEP_VERSIONS=<n>   How many versions of each Swarm secret to retain
                             (default: 3). Older unused versions are pruned.
EOF
}

while (($# > 0)); do
  case "$1" in
    -h|--help)
      print_usage
      exit 0
      ;;
    *)
      die "Unknown argument: $1"
      ;;
  esac
done

require_cmd docker
require_cmd ssh
require_cmd scp
require_cmd git
require_cmd awk
require_cmd sed
require_cmd grep
require_cmd mktemp
require_cmd date
require_cmd curl

ensure_file_from_example "${CLUSTER_ENV}"
ensure_file_from_example "${REGISTRY_ENV}"
ensure_file_from_example "${PROD_ENV}"

require_secret_file "${SECRET_POSTGRES}" "Postgres password"
require_secret_file "${SECRET_APP_KEY}" "SECRET_KEY"
require_secret_file "${SECRET_EMAIL_PASS}" "SMTP password"
require_secret_file "${SECRET_FCM_KEY}" "FCM server key"
require_secret_file "${SECRET_APNS_KEY}" "APNS private key"

set -a
# shellcheck disable=SC1090
source "${CLUSTER_ENV}"
# shellcheck disable=SC1090
source "${REGISTRY_ENV}"
# shellcheck disable=SC1090
source "${PROD_ENV}"
set +a

DEPLOY_MANAGER_SSH_PORT="${DEPLOY_MANAGER_SSH_PORT:-22}"
DEPLOY_STACK_NAME="${DEPLOY_STACK_NAME:-honeydue}"
DEPLOY_REMOTE_DIR="${DEPLOY_REMOTE_DIR:-/opt/honeydue/deploy}"
DEPLOY_WAIT_SECONDS="${DEPLOY_WAIT_SECONDS:-420}"
DEPLOY_TAG="${DEPLOY_TAG:-$(git -C "${REPO_DIR}" rev-parse --short HEAD)}"
PUSH_LATEST_TAG="${PUSH_LATEST_TAG:-false}"

require_var DEPLOY_MANAGER_HOST
require_var DEPLOY_MANAGER_USER
require_var DEPLOY_STACK_NAME
require_var DEPLOY_REMOTE_DIR
require_var REGISTRY
require_var REGISTRY_NAMESPACE
require_var REGISTRY_USERNAME
require_var REGISTRY_TOKEN

require_var ALLOWED_HOSTS
require_var CORS_ALLOWED_ORIGINS
require_var BASE_URL
require_var NEXT_PUBLIC_API_URL
require_var DB_HOST
require_var DB_PORT
require_var POSTGRES_USER
require_var POSTGRES_DB
require_var DB_SSLMODE
require_var REDIS_URL
require_var EMAIL_HOST
require_var EMAIL_PORT
require_var EMAIL_HOST_USER
require_var DEFAULT_FROM_EMAIL
require_var APNS_AUTH_KEY_ID
require_var APNS_TEAM_ID
require_var APNS_TOPIC

# Storage backend validation: B2 is all-or-none. If any var is filled with
# a real value, require all four core vars. Empty means "use local volume".
b2_any_set=0
b2_all_set=1
for b2_var in B2_ENDPOINT B2_KEY_ID B2_APP_KEY B2_BUCKET_NAME; do
  val="${!b2_var:-}"
  if [[ -n "${val}" ]] && ! contains_placeholder "${val}"; then
    b2_any_set=1
  else
    b2_all_set=0
  fi
done
if (( b2_any_set == 1 && b2_all_set == 0 )); then
  die "Partial B2 configuration detected. Set all four of B2_ENDPOINT, B2_KEY_ID, B2_APP_KEY, B2_BUCKET_NAME, or leave all four empty to use the local volume."
fi
if (( b2_all_set == 1 )); then
  log "Storage backend: S3 (${B2_ENDPOINT} / bucket=${B2_BUCKET_NAME})"
else
  warn "Storage backend: LOCAL VOLUME. This is not safe for multi-replica prod — uploads will only exist on one node. Set B2_* in prod.env to use object storage."
fi

if [[ ! "$(tr -d '\r\n' < "${SECRET_APNS_KEY}")" =~ BEGIN[[:space:]]+PRIVATE[[:space:]]+KEY ]]; then
  die "APNS key file does not look like a private key: ${SECRET_APNS_KEY}"
fi

app_secret_len="$(tr -d '\r\n' < "${SECRET_APP_KEY}" | wc -c | tr -d ' ')"
if (( app_secret_len < 32 )); then
  die "deploy/secrets/secret_key.txt must be at least 32 characters."
fi

REGISTRY_PREFIX="${REGISTRY%/}/${REGISTRY_NAMESPACE#/}"
API_IMAGE="${REGISTRY_PREFIX}/honeydue-api:${DEPLOY_TAG}"
WORKER_IMAGE="${REGISTRY_PREFIX}/honeydue-worker:${DEPLOY_TAG}"
ADMIN_IMAGE="${REGISTRY_PREFIX}/honeydue-admin:${DEPLOY_TAG}"

SSH_KEY_PATH="${DEPLOY_SSH_KEY_PATH:-}"
if [[ -n "${SSH_KEY_PATH}" ]]; then
  SSH_KEY_PATH="${SSH_KEY_PATH/#\~/${HOME}}"
fi

SSH_TARGET="${DEPLOY_MANAGER_USER}@${DEPLOY_MANAGER_HOST}"
SSH_OPTS=(-p "${DEPLOY_MANAGER_SSH_PORT}")
SCP_OPTS=(-P "${DEPLOY_MANAGER_SSH_PORT}")
if [[ -n "${SSH_KEY_PATH}" ]]; then
  SSH_OPTS+=(-i "${SSH_KEY_PATH}")
  SCP_OPTS+=(-i "${SSH_KEY_PATH}")
fi

if [[ "${DRY_RUN}" == "1" ]]; then
  cat <<EOF

==================== DRY RUN ====================
Validation passed. Would deploy:

  Stack name:       ${DEPLOY_STACK_NAME}
  Manager:          ${SSH_TARGET}:${DEPLOY_MANAGER_SSH_PORT}
  Remote dir:       ${DEPLOY_REMOTE_DIR}
  Deploy tag:       ${DEPLOY_TAG}
  Push :latest:     ${PUSH_LATEST_TAG}
  Skip build:       ${SKIP_BUILD}
  Skip healthcheck: ${SKIP_HEALTHCHECK}
  Secret retention: ${SECRET_KEEP_VERSIONS} versions per name

Images that would be built and pushed:
  ${API_IMAGE}
  ${WORKER_IMAGE}
  ${ADMIN_IMAGE}

Replicas:
  caddy:  3 (one per node)
  api:    ${API_REPLICAS:-3}
  worker: ${WORKER_REPLICAS:-1}
  admin:  ${ADMIN_REPLICAS:-1}

Published ports:
  caddy:  80, 443 (ingress — public)
  api:    internal only (proxied by caddy)
  admin:  internal only (proxied by caddy)
  dozzle: ${DOZZLE_PORT:-9999} (manager loopback only — SSH tunnel required)

Versioned secrets that would be created on this deploy:
  ${DEPLOY_STACK_NAME}_postgres_password_<deploy_id>
  ${DEPLOY_STACK_NAME}_secret_key_<deploy_id>
  ${DEPLOY_STACK_NAME}_email_host_password_<deploy_id>
  ${DEPLOY_STACK_NAME}_fcm_server_key_<deploy_id>
  ${DEPLOY_STACK_NAME}_apns_auth_key_<deploy_id>

Versioned configs that would be created on this deploy:
  ${DEPLOY_STACK_NAME}_caddyfile_<deploy_id>

No changes made. Re-run without DRY_RUN=1 to deploy.
=================================================

EOF
  exit 0
fi

log "Validating SSH access to ${SSH_TARGET}"
if ! ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "echo ok" >/dev/null 2>&1; then
  die "SSH connection failed to ${SSH_TARGET}"
fi

remote_swarm_state="$(ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "docker info --format '{{.Swarm.LocalNodeState}} {{.Swarm.ControlAvailable}}'" 2>/dev/null || true)"
if [[ -z "${remote_swarm_state}" ]]; then
  die "Could not read Docker Swarm state on manager. Is Docker installed/running?"
fi

if [[ "${remote_swarm_state}" != "active true" ]]; then
  die "Remote node must be a Swarm manager. Got: ${remote_swarm_state}"
fi

if [[ "${SKIP_BUILD}" != "1" ]]; then
  log "Logging in to ${REGISTRY}"
  printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USERNAME}" --password-stdin >/dev/null

  # Target platform for Swarm nodes. Hetzner CX is x86_64; override via
  # BUILD_PLATFORM=linux/arm64 if you move to ARM (Ampere) hosts.
  BUILD_PLATFORM="${BUILD_PLATFORM:-linux/amd64}"
  log "Build platform: ${BUILD_PLATFORM} (dev host may differ; buildx cross-compiles)"

  # Ensure a buildx builder exists and is usable
  if ! docker buildx inspect honeydue-builder >/dev/null 2>&1; then
    log "Creating buildx builder 'honeydue-builder'"
    docker buildx create --name honeydue-builder --use >/dev/null
  else
    docker buildx use honeydue-builder >/dev/null
  fi
  docker buildx inspect --bootstrap >/dev/null

  build_and_push() {
    local target="$1"
    local image="$2"
    shift 2

    local tag_args=(-t "${image}")
    while (( $# > 0 )); do
      tag_args+=(-t "$1")
      shift
    done

    log "Building + pushing ${target} image for ${BUILD_PLATFORM}: ${image}"
    docker buildx build \
      --platform "${BUILD_PLATFORM}" \
      --target "${target}" \
      "${tag_args[@]}" \
      --push \
      "${REPO_DIR}"
  }

  api_extra=()
  worker_extra=()
  admin_extra=()
  if [[ "${PUSH_LATEST_TAG}" == "true" ]]; then
    api_extra=("${REGISTRY_PREFIX}/honeydue-api:latest")
    worker_extra=("${REGISTRY_PREFIX}/honeydue-worker:latest")
    admin_extra=("${REGISTRY_PREFIX}/honeydue-admin:latest")
  fi

  # ${arr[@]+"${arr[@]}"} safely expands to nothing when the array is empty
  # under `set -u` — avoids "unbound variable" on bash arrays.
  build_and_push api    "${API_IMAGE}"    ${api_extra[@]+"${api_extra[@]}"}
  build_and_push worker "${WORKER_IMAGE}" ${worker_extra[@]+"${worker_extra[@]}"}
  build_and_push admin  "${ADMIN_IMAGE}"  ${admin_extra[@]+"${admin_extra[@]}"}
else
  warn "SKIP_BUILD=1 set. Using prebuilt images for tag: ${DEPLOY_TAG}"
fi

DEPLOY_ID_RAW="${DEPLOY_TAG}-$(date +%Y%m%d%H%M%S)"
DEPLOY_ID="$(printf '%s' "${DEPLOY_ID_RAW}" | tr -c 'a-zA-Z0-9_-' '_')"

POSTGRES_PASSWORD_SECRET="${DEPLOY_STACK_NAME}_postgres_password_${DEPLOY_ID}"
SECRET_KEY_SECRET="${DEPLOY_STACK_NAME}_secret_key_${DEPLOY_ID}"
EMAIL_HOST_PASSWORD_SECRET="${DEPLOY_STACK_NAME}_email_host_password_${DEPLOY_ID}"
FCM_SERVER_KEY_SECRET="${DEPLOY_STACK_NAME}_fcm_server_key_${DEPLOY_ID}"
APNS_AUTH_KEY_SECRET="${DEPLOY_STACK_NAME}_apns_auth_key_${DEPLOY_ID}"
CADDYFILE_CONFIG="${DEPLOY_STACK_NAME}_caddyfile_${DEPLOY_ID}"

CADDYFILE_SRC="${DEPLOY_DIR}/Caddyfile"
if [[ ! -f "${CADDYFILE_SRC}" ]]; then
  die "Missing required file: ${CADDYFILE_SRC}"
fi

TMP_DIR="$(mktemp -d)"
cleanup() {
  rm -rf "${TMP_DIR}"
}
trap cleanup EXIT

cp "${STACK_TEMPLATE}" "${TMP_DIR}/swarm-stack.prod.yml"
cp "${PROD_ENV}" "${TMP_DIR}/prod.env"
cp "${REGISTRY_ENV}" "${TMP_DIR}/registry.env"
cp "${CADDYFILE_SRC}" "${TMP_DIR}/Caddyfile"
mkdir -p "${TMP_DIR}/secrets"
cp "${SECRET_POSTGRES}" "${TMP_DIR}/secrets/postgres_password.txt"
cp "${SECRET_APP_KEY}" "${TMP_DIR}/secrets/secret_key.txt"
cp "${SECRET_EMAIL_PASS}" "${TMP_DIR}/secrets/email_host_password.txt"
cp "${SECRET_FCM_KEY}" "${TMP_DIR}/secrets/fcm_server_key.txt"
cp "${SECRET_APNS_KEY}" "${TMP_DIR}/secrets/apns_auth_key.p8"

cat > "${TMP_DIR}/runtime.env" <<EOF
API_IMAGE=${API_IMAGE}
WORKER_IMAGE=${WORKER_IMAGE}
ADMIN_IMAGE=${ADMIN_IMAGE}

API_REPLICAS=${API_REPLICAS:-3}
WORKER_REPLICAS=${WORKER_REPLICAS:-2}
ADMIN_REPLICAS=${ADMIN_REPLICAS:-1}
API_PORT=${API_PORT:-8000}
ADMIN_PORT=${ADMIN_PORT:-3000}
DOZZLE_PORT=${DOZZLE_PORT:-9999}

POSTGRES_PASSWORD_SECRET=${POSTGRES_PASSWORD_SECRET}
SECRET_KEY_SECRET=${SECRET_KEY_SECRET}
EMAIL_HOST_PASSWORD_SECRET=${EMAIL_HOST_PASSWORD_SECRET}
FCM_SERVER_KEY_SECRET=${FCM_SERVER_KEY_SECRET}
APNS_AUTH_KEY_SECRET=${APNS_AUTH_KEY_SECRET}
CADDYFILE_CONFIG=${CADDYFILE_CONFIG}
EOF

log "Uploading deploy bundle to ${SSH_TARGET}:${DEPLOY_REMOTE_DIR}"
ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "mkdir -p '${DEPLOY_REMOTE_DIR}/secrets'"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/swarm-stack.prod.yml" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/swarm-stack.prod.yml"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/prod.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/prod.env"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/registry.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/registry.env"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/runtime.env" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/runtime.env"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/Caddyfile" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/Caddyfile"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/postgres_password.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/postgres_password.txt"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/secret_key.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/secret_key.txt"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/email_host_password.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/email_host_password.txt"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/fcm_server_key.txt" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/fcm_server_key.txt"
scp "${SCP_OPTS[@]}" "${TMP_DIR}/secrets/apns_auth_key.p8" "${SSH_TARGET}:${DEPLOY_REMOTE_DIR}/secrets/apns_auth_key.p8"

log "Creating Docker secrets and deploying stack on manager"
ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_REMOTE_DIR}' '${DEPLOY_STACK_NAME}'" <<'EOF'
set -euo pipefail

REMOTE_DIR="$1"
STACK_NAME="$2"

set -a
# shellcheck disable=SC1090
source "${REMOTE_DIR}/registry.env"
# shellcheck disable=SC1090
source "${REMOTE_DIR}/prod.env"
# shellcheck disable=SC1090
source "${REMOTE_DIR}/runtime.env"
set +a

create_secret() {
  local name="$1"
  local src="$2"
  if docker secret inspect "${name}" >/dev/null 2>&1; then
    echo "[remote] secret exists: ${name}"
  else
    docker secret create "${name}" "${src}" >/dev/null
    echo "[remote] created secret: ${name}"
  fi
}

create_config() {
  local name="$1"
  local src="$2"
  if docker config inspect "${name}" >/dev/null 2>&1; then
    echo "[remote] config exists: ${name}"
  else
    docker config create "${name}" "${src}" >/dev/null
    echo "[remote] created config: ${name}"
  fi
}

printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USERNAME}" --password-stdin >/dev/null
rm -f "${REMOTE_DIR}/registry.env"

create_secret "${POSTGRES_PASSWORD_SECRET}" "${REMOTE_DIR}/secrets/postgres_password.txt"
create_secret "${SECRET_KEY_SECRET}" "${REMOTE_DIR}/secrets/secret_key.txt"
create_secret "${EMAIL_HOST_PASSWORD_SECRET}" "${REMOTE_DIR}/secrets/email_host_password.txt"
create_secret "${FCM_SERVER_KEY_SECRET}" "${REMOTE_DIR}/secrets/fcm_server_key.txt"
create_secret "${APNS_AUTH_KEY_SECRET}" "${REMOTE_DIR}/secrets/apns_auth_key.p8"

create_config "${CADDYFILE_CONFIG}" "${REMOTE_DIR}/Caddyfile"

rm -f "${REMOTE_DIR}/secrets/postgres_password.txt"
rm -f "${REMOTE_DIR}/secrets/secret_key.txt"
rm -f "${REMOTE_DIR}/secrets/email_host_password.txt"
rm -f "${REMOTE_DIR}/secrets/fcm_server_key.txt"
rm -f "${REMOTE_DIR}/secrets/apns_auth_key.p8"

set -a
# shellcheck disable=SC1090
source "${REMOTE_DIR}/prod.env"
# shellcheck disable=SC1090
source "${REMOTE_DIR}/runtime.env"
set +a

docker stack deploy --with-registry-auth -c "${REMOTE_DIR}/swarm-stack.prod.yml" "${STACK_NAME}"
EOF

log "Waiting for stack convergence (${DEPLOY_WAIT_SECONDS}s max)"
start_epoch="$(date +%s)"
while true; do
  services="$(ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "docker stack services '${DEPLOY_STACK_NAME}' --format '{{.Name}} {{.Replicas}}'" 2>/dev/null || true)"

  if [[ -n "${services}" ]]; then
    all_ready=1
    while IFS=' ' read -r svc replicas; do
      [[ -z "${svc}" ]] && continue
      current="${replicas%%/*}"
      desired="${replicas##*/}"
      if [[ "${desired}" == "0" ]]; then
        continue
      fi
      if [[ "${current}" != "${desired}" ]]; then
        all_ready=0
      fi
    done <<< "${services}"

    if [[ "${all_ready}" -eq 1 ]]; then
      break
    fi
  fi

  now_epoch="$(date +%s)"
  elapsed=$((now_epoch - start_epoch))
  if (( elapsed >= DEPLOY_WAIT_SECONDS )); then
    die "Timed out waiting for stack to converge. Check: ssh ${SSH_TARGET} docker stack services ${DEPLOY_STACK_NAME}"
  fi

  sleep 10
done

log "Pruning old secret + config versions (keeping last ${SECRET_KEEP_VERSIONS})"
ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}' '${SECRET_KEEP_VERSIONS}'" <<'EOF' || warn "Pruning reported errors (non-fatal)"
set -euo pipefail

STACK_NAME="$1"
KEEP="$2"

prune_prefix() {
  local kind="$1"   # "secret" or "config"
  local prefix="$2"
  local ls_cmd rm_cmd
  ls_cmd="docker ${kind} ls --format '{{.CreatedAt}}|{{.Name}}'"
  rm_cmd="docker ${kind} rm"

  local all
  all="$(eval "${ls_cmd}" 2>/dev/null \
    | grep "|${prefix}_" \
    | sort -r \
    || true)"
  if [[ -z "${all}" ]]; then
    return 0
  fi

  local total
  total="$(printf '%s\n' "${all}" | wc -l | tr -d ' ')"
  if (( total <= KEEP )); then
    echo "[cleanup] ${kind}/${prefix}: ${total} version(s) — nothing to prune"
    return 0
  fi

  local to_remove
  to_remove="$(printf '%s\n' "${all}" | tail -n +$((KEEP + 1)) | awk -F'|' '{print $2}')"

  while IFS= read -r name; do
    [[ -z "${name}" ]] && continue
    if ${rm_cmd} "${name}" >/dev/null 2>&1; then
      echo "[cleanup] removed ${kind}: ${name}"
    else
      echo "[cleanup] in-use ${kind} (kept): ${name}"
    fi
  done <<< "${to_remove}"
}

for base in postgres_password secret_key email_host_password fcm_server_key apns_auth_key; do
  prune_prefix secret "${STACK_NAME}_${base}"
done

for base in caddyfile; do
  prune_prefix config "${STACK_NAME}_${base}"
done
EOF

rollback_stack() {
  warn "Rolling back stack ${DEPLOY_STACK_NAME} on ${SSH_TARGET}"
  ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "bash -s -- '${DEPLOY_STACK_NAME}'" <<'EOF' || true
set +e
STACK="$1"
for svc in $(docker stack services "${STACK}" --format '{{.Name}}'); do
  echo "[rollback] ${svc}"
  docker service rollback "${svc}" || echo "[rollback] ${svc}: nothing to roll back"
done
EOF
}

if [[ "${SKIP_HEALTHCHECK}" != "1" && -n "${DEPLOY_HEALTHCHECK_URL:-}" ]]; then
  log "Running health check: ${DEPLOY_HEALTHCHECK_URL}"
  if ! curl -fsS --max-time 20 "${DEPLOY_HEALTHCHECK_URL}" >/dev/null; then
    warn "Health check FAILED for ${DEPLOY_HEALTHCHECK_URL}"
    rollback_stack
    die "Deploy rolled back due to failed health check."
  fi
fi

# Best-effort registry logout — the token should not linger in
# ~/.docker/config.json after deploy completes. Failures are non-fatal.
log "Logging out of registry (local + remote)"
docker logout "${REGISTRY}" >/dev/null 2>&1 || true
ssh "${SSH_OPTS[@]}" "${SSH_TARGET}" "docker logout '${REGISTRY}' >/dev/null 2>&1 || true"

log "Deploy completed successfully."
log "Stack: ${DEPLOY_STACK_NAME}"
log "Images:"
log "  ${API_IMAGE}"
log "  ${WORKER_IMAGE}"
log "  ${ADMIN_IMAGE}"