Add K3s dev deployment setup for single-node VPS

Mirrors the prod deploy-k3s/ setup but runs all services in-cluster on a single node: PostgreSQL (replaces Neon), MinIO S3-compatible storage (replaces B2), Redis, API, worker, and admin. Includes fully automated setup scripts (00-init through 04-verify), server hardening (SSH, fail2ban, ufw), Let's Encrypt TLS via Traefik, network policies, RBAC, and security contexts matching prod. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 21:30:39 -05:00
parent 00fd674b56
commit 34553f3bec
52 changed files with 5319 additions and 0 deletions
--- a/deploy-k3s-dev/scripts/00-init.sh
+++ b/deploy-k3s-dev/scripts/00-init.sh
@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+SECRETS_DIR="${DEPLOY_DIR}/secrets"
+CONFIG_FILE="${DEPLOY_DIR}/config.yaml"
+
+log() { printf '[init] %s\n' "$*"; }
+warn() { printf '[init][warn] %s\n' "$*" >&2; }
+die() { printf '[init][error] %s\n' "$*" >&2; exit 1; }
+
+# --- Prerequisites ---
+
+command -v openssl >/dev/null 2>&1 || die "Missing: openssl"
+command -v python3 >/dev/null 2>&1 || die "Missing: python3"
+
+echo ""
+echo "============================================"
+echo "  honeyDue Dev Server — Initial Setup"
+echo "============================================"
+echo ""
+echo "This script will:"
+echo "  1. Generate any missing random secrets"
+echo "  2. Ask for anything not already filled in"
+echo "  3. Create config.yaml with everything filled in"
+echo ""
+
+mkdir -p "${SECRETS_DIR}"
+
+# --- Generate random secrets (skip if already exist) ---
+
+generate_if_missing() {
+  local file="$1" label="$2" cmd="$3"
+  if [[ -f "${file}" && -s "${file}" ]]; then
+    log "  ${label} — already exists, keeping"
+  else
+    eval "${cmd}" > "${file}"
+    log "  ${label} — generated"
+  fi
+}
+
+log "Checking secrets..."
+generate_if_missing "${SECRETS_DIR}/secret_key.txt" "secrets/secret_key.txt" "openssl rand -base64 48"
+generate_if_missing "${SECRETS_DIR}/postgres_password.txt" "secrets/postgres_password.txt" "openssl rand -base64 24"
+generate_if_missing "${SECRETS_DIR}/minio_root_password.txt" "secrets/minio_root_password.txt" "openssl rand -base64 24"
+generate_if_missing "${SECRETS_DIR}/email_host_password.txt" "secrets/email_host_password.txt" "echo PLACEHOLDER"
+log "  secrets/fcm_server_key.txt — skipped (Android not ready)"
+generate_if_missing "${SECRETS_DIR}/apns_auth_key.p8" "secrets/apns_auth_key.p8" "echo ''"
+
+REDIS_PW="$(openssl rand -base64 24)"
+log "  Redis password — generated"
+
+# --- Collect only what's missing ---
+
+ask() {
+  local var_name="$1" prompt="$2" default="${3:-}"
+  local val
+  if [[ -n "${default}" ]]; then
+    read -rp "${prompt} [${default}]: " val
+    val="${val:-${default}}"
+  else
+    read -rp "${prompt}: " val
+  fi
+  eval "${var_name}='${val}'"
+}
+
+echo ""
+echo "--- Server ---"
+ask SERVER_HOST "Server IP or SSH alias" "honeyDueDevUpdate"
+[[ -n "${SERVER_HOST}" ]] || die "Server host is required"
+ask SERVER_USER "SSH user" "root"
+ask SSH_KEY "SSH key path" "~/.ssh/id_ed25519"
+
+echo ""
+echo "--- Container Registry (GHCR) ---"
+ask GHCR_USER "GitHub username" "treytartt"
+[[ -n "${GHCR_USER}" ]] || die "GitHub username is required"
+ask GHCR_TOKEN "GitHub PAT (read:packages, write:packages)" "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"
+[[ -n "${GHCR_TOKEN}" ]] || die "GitHub PAT is required"
+
+echo ""
+echo "--- TLS ---"
+ask LE_EMAIL "Let's Encrypt email" "treytartt@fastmail.com"
+
+echo ""
+echo "--- Admin Panel ---"
+ask ADMIN_USER "Admin basic auth username" "admin"
+ADMIN_PW="$(openssl rand -base64 16)"
+
+# --- Known values from existing Dokku setup ---
+
+EMAIL_USER="treytartt@fastmail.com"
+APNS_KEY_ID="9R5Q7ZX874"
+APNS_TEAM_ID="V3PF3M6B6U"
+
+log ""
+log "Pre-filled from existing dev server:"
+log "  Email user: ${EMAIL_USER}"
+log "  APNS Key ID: ${APNS_KEY_ID}"
+log "  APNS Team ID: ${APNS_TEAM_ID}"
+
+# --- Generate config.yaml ---
+
+log "Generating config.yaml..."
+
+cat > "${CONFIG_FILE}" <<YAML
+# config.yaml — auto-generated by 00-init.sh
+# This file is gitignored — never commit it with real values.
+
+# --- Server ---
+server:
+  host: "${SERVER_HOST}"
+  user: "${SERVER_USER}"
+  ssh_key: "${SSH_KEY}"
+
+# --- Domains ---
+domains:
+  api: devapi.myhoneydue.com
+  admin: devadmin.myhoneydue.com
+  base: dev.myhoneydue.com
+
+# --- Container Registry (GHCR) ---
+registry:
+  server: ghcr.io
+  namespace: "${GHCR_USER}"
+  username: "${GHCR_USER}"
+  token: "${GHCR_TOKEN}"
+
+# --- Database (in-cluster PostgreSQL) ---
+database:
+  name: honeydue_dev
+  user: honeydue
+  max_open_conns: 10
+  max_idle_conns: 5
+  max_lifetime: "600s"
+
+# --- Email (Fastmail) ---
+email:
+  host: smtp.fastmail.com
+  port: 587
+  user: "${EMAIL_USER}"
+  from: "honeyDue DEV <${EMAIL_USER}>"
+  use_tls: true
+
+# --- Push Notifications ---
+push:
+  apns_key_id: "${APNS_KEY_ID}"
+  apns_team_id: "${APNS_TEAM_ID}"
+  apns_topic: com.tt.honeyDue
+  apns_production: false
+  apns_use_sandbox: true
+
+# --- Object Storage (in-cluster MinIO) ---
+storage:
+  minio_root_user: honeydue
+  bucket: honeydue-dev
+  max_file_size: 10485760
+  allowed_types: "image/jpeg,image/png,image/gif,image/webp,application/pdf"
+
+# --- Worker Schedules (UTC hours) ---
+worker:
+  task_reminder_hour: 14
+  overdue_reminder_hour: 15
+  daily_digest_hour: 3
+
+# --- Feature Flags ---
+features:
+  push_enabled: true
+  email_enabled: false
+  webhooks_enabled: false
+  onboarding_emails_enabled: false
+  pdf_reports_enabled: true
+  worker_enabled: true
+
+# --- Redis ---
+redis:
+  password: "${REDIS_PW}"
+
+# --- Admin Panel ---
+admin:
+  basic_auth_user: "${ADMIN_USER}"
+  basic_auth_password: "${ADMIN_PW}"
+
+# --- TLS ---
+tls:
+  mode: letsencrypt
+  letsencrypt_email: "${LE_EMAIL}"
+
+# --- Apple Auth / IAP ---
+apple_auth:
+  client_id: "com.tt.honeyDue"
+  team_id: "${APNS_TEAM_ID}"
+  iap_key_id: ""
+  iap_issuer_id: ""
+  iap_bundle_id: ""
+  iap_key_path: ""
+  iap_sandbox: true
+
+# --- Google Auth / IAP ---
+google_auth:
+  client_id: ""
+  android_client_id: ""
+  ios_client_id: ""
+  iap_package_name: ""
+  iap_service_account_path: ""
+YAML
+
+# --- Summary ---
+
+echo ""
+echo "============================================"
+echo "  Setup Complete"
+echo "============================================"
+echo ""
+echo "Generated:"
+echo "  config.yaml"
+echo "  secrets/secret_key.txt"
+echo "  secrets/postgres_password.txt"
+echo "  secrets/minio_root_password.txt"
+echo "  secrets/email_host_password.txt"
+echo "  secrets/fcm_server_key.txt"
+echo "  secrets/apns_auth_key.p8"
+echo ""
+echo "Admin panel credentials:"
+echo "  Username: ${ADMIN_USER}"
+echo "  Password: ${ADMIN_PW}"
+echo "  (save these — they won't be shown again)"
+echo ""
+echo "Next steps:"
+echo "  ./scripts/01-setup-k3s.sh"
+echo "  ./scripts/02-setup-secrets.sh"
+echo "  ./scripts/03-deploy.sh"
+echo "  ./scripts/04-verify.sh"
+echo ""
--- a/deploy-k3s-dev/scripts/01-setup-k3s.sh
+++ b/deploy-k3s-dev/scripts/01-setup-k3s.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_config.sh
+source "${SCRIPT_DIR}/_config.sh"
+
+log() { printf '[setup] %s\n' "$*"; }
+die() { printf '[setup][error] %s\n' "$*" >&2; exit 1; }
+
+# --- Local prerequisites ---
+
+command -v kubectl >/dev/null 2>&1 || die "Missing locally: kubectl (https://kubernetes.io/docs/tasks/tools/)"
+
+# --- Server connection ---
+
+SERVER_HOST="$(cfg_require server.host "Server IP or SSH alias")"
+SERVER_USER="$(cfg server.user)"
+SERVER_USER="${SERVER_USER:-root}"
+SSH_KEY="$(cfg server.ssh_key | sed "s|~|${HOME}|g")"
+
+SSH_OPTS=()
+if [[ -n "${SSH_KEY}" && -f "${SSH_KEY}" ]]; then
+  SSH_OPTS+=(-i "${SSH_KEY}")
+fi
+SSH_OPTS+=(-o StrictHostKeyChecking=accept-new)
+
+ssh_cmd() {
+  ssh "${SSH_OPTS[@]}" "${SERVER_USER}@${SERVER_HOST}" "$@"
+}
+
+log "Testing SSH connection to ${SERVER_USER}@${SERVER_HOST}..."
+ssh_cmd "echo 'SSH connection OK'" || die "Cannot SSH into ${SERVER_HOST}"
+
+# --- Server prerequisites ---
+
+log "Setting up server prerequisites..."
+
+ssh_cmd 'bash -s' <<'REMOTE_SETUP'
+set -euo pipefail
+
+log() { printf '[setup][remote] %s\n' "$*"; }
+
+# --- System updates ---
+log "Updating system packages..."
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get upgrade -y -qq
+
+# --- SSH hardening ---
+log "Hardening SSH..."
+sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
+sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
+systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true
+
+# --- fail2ban ---
+if ! command -v fail2ban-client >/dev/null 2>&1; then
+  log "Installing fail2ban..."
+  apt-get install -y -qq fail2ban
+  systemctl enable --now fail2ban
+else
+  log "fail2ban already installed"
+fi
+
+# --- Unattended security upgrades ---
+if ! dpkg -l | grep -q unattended-upgrades; then
+  log "Installing unattended-upgrades..."
+  apt-get install -y -qq unattended-upgrades
+  dpkg-reconfigure -plow unattended-upgrades
+else
+  log "unattended-upgrades already installed"
+fi
+
+# --- Firewall (ufw) ---
+if command -v ufw >/dev/null 2>&1; then
+  log "Configuring firewall..."
+  ufw default deny incoming
+  ufw default allow outgoing
+  ufw allow 22/tcp     # SSH
+  ufw allow 443/tcp    # HTTPS (Traefik)
+  ufw allow 6443/tcp   # K3s API
+  ufw allow 80/tcp     # HTTP (Let's Encrypt ACME challenge)
+  ufw --force enable
+else
+  log "Installing ufw..."
+  apt-get install -y -qq ufw
+  ufw default deny incoming
+  ufw default allow outgoing
+  ufw allow 22/tcp
+  ufw allow 443/tcp
+  ufw allow 6443/tcp
+  ufw allow 80/tcp
+  ufw --force enable
+fi
+
+log "Server prerequisites complete."
+REMOTE_SETUP
+
+# --- Install K3s ---
+
+log "Installing K3s on ${SERVER_HOST}..."
+log "  This takes about 1-2 minutes."
+
+ssh_cmd "curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='server --secrets-encryption' sh -"
+
+# --- Wait for K3s to be ready ---
+
+log "Waiting for K3s to be ready..."
+ssh_cmd "until kubectl get nodes >/dev/null 2>&1; do sleep 2; done"
+
+# --- Copy kubeconfig ---
+
+KUBECONFIG_PATH="${DEPLOY_DIR}/kubeconfig"
+
+log "Copying kubeconfig..."
+ssh_cmd "sudo cat /etc/rancher/k3s/k3s.yaml" > "${KUBECONFIG_PATH}"
+
+# Replace 127.0.0.1 with the server's actual IP/hostname
+# If SERVER_HOST is an SSH alias, resolve the actual IP
+ACTUAL_HOST="${SERVER_HOST}"
+if ! echo "${SERVER_HOST}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then
+  # Try to resolve from SSH config
+  RESOLVED="$(ssh -G "${SERVER_HOST}" 2>/dev/null | awk '/^hostname / {print $2}')"
+  if [[ -n "${RESOLVED}" && "${RESOLVED}" != "${SERVER_HOST}" ]]; then
+    ACTUAL_HOST="${RESOLVED}"
+  fi
+fi
+
+sed -i.bak "s|https://127.0.0.1:6443|https://${ACTUAL_HOST}:6443|g" "${KUBECONFIG_PATH}"
+rm -f "${KUBECONFIG_PATH}.bak"
+chmod 600 "${KUBECONFIG_PATH}"
+
+# --- Verify ---
+
+export KUBECONFIG="${KUBECONFIG_PATH}"
+log "Verifying cluster..."
+kubectl get nodes
+
+log ""
+log "K3s installed successfully on ${SERVER_HOST}."
+log "Server hardened: SSH key-only, fail2ban, ufw firewall, unattended-upgrades."
+log ""
+log "Next steps:"
+log "  export KUBECONFIG=${KUBECONFIG_PATH}"
+log "  kubectl get nodes"
+log "  ./scripts/02-setup-secrets.sh"
--- a/deploy-k3s-dev/scripts/02-setup-secrets.sh
+++ b/deploy-k3s-dev/scripts/02-setup-secrets.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_config.sh
+source "${SCRIPT_DIR}/_config.sh"
+
+SECRETS_DIR="${DEPLOY_DIR}/secrets"
+NAMESPACE="honeydue"
+
+log() { printf '[secrets] %s\n' "$*"; }
+warn() { printf '[secrets][warn] %s\n' "$*" >&2; }
+die() { printf '[secrets][error] %s\n' "$*" >&2; exit 1; }
+
+# --- Prerequisites ---
+
+command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl"
+
+kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 || {
+  log "Creating namespace ${NAMESPACE}..."
+  kubectl apply -f "${DEPLOY_DIR}/manifests/namespace.yaml"
+}
+
+# --- Validate secret files ---
+
+require_file() {
+  local path="$1" label="$2"
+  [[ -f "${path}" ]] || die "Missing: ${path} (${label})"
+  [[ -s "${path}" ]] || die "Empty: ${path} (${label})"
+}
+
+require_file "${SECRETS_DIR}/postgres_password.txt" "Postgres password"
+require_file "${SECRETS_DIR}/secret_key.txt" "SECRET_KEY"
+require_file "${SECRETS_DIR}/email_host_password.txt" "SMTP password"
+# FCM server key is optional (Android not yet ready)
+if [[ -f "${SECRETS_DIR}/fcm_server_key.txt" && -s "${SECRETS_DIR}/fcm_server_key.txt" ]]; then
+  FCM_CONTENT="$(tr -d '\r\n' < "${SECRETS_DIR}/fcm_server_key.txt")"
+  if [[ "${FCM_CONTENT}" == "PLACEHOLDER" ]]; then
+    warn "fcm_server_key.txt is a placeholder — FCM push disabled"
+    FCM_CONTENT=""
+  fi
+else
+  warn "fcm_server_key.txt not found — FCM push disabled"
+  FCM_CONTENT=""
+fi
+require_file "${SECRETS_DIR}/apns_auth_key.p8" "APNS private key"
+require_file "${SECRETS_DIR}/minio_root_password.txt" "MinIO root password"
+
+# Validate APNS key format
+if ! grep -q "BEGIN PRIVATE KEY" "${SECRETS_DIR}/apns_auth_key.p8"; then
+  die "APNS key file does not look like a private key: ${SECRETS_DIR}/apns_auth_key.p8"
+fi
+
+# Validate secret_key length (minimum 32 chars)
+SECRET_KEY_LEN="$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt" | wc -c | tr -d ' ')"
+if (( SECRET_KEY_LEN < 32 )); then
+  die "secret_key.txt must be at least 32 characters (got ${SECRET_KEY_LEN})."
+fi
+
+# Validate MinIO password length (minimum 8 chars)
+MINIO_PW_LEN="$(tr -d '\r\n' < "${SECRETS_DIR}/minio_root_password.txt" | wc -c | tr -d ' ')"
+if (( MINIO_PW_LEN < 8 )); then
+  die "minio_root_password.txt must be at least 8 characters (got ${MINIO_PW_LEN})."
+fi
+
+# --- Read optional config values ---
+
+REDIS_PASSWORD="$(cfg redis.password 2>/dev/null || true)"
+ADMIN_AUTH_USER="$(cfg admin.basic_auth_user 2>/dev/null || true)"
+ADMIN_AUTH_PASSWORD="$(cfg admin.basic_auth_password 2>/dev/null || true)"
+TLS_MODE="$(cfg tls.mode 2>/dev/null || echo "letsencrypt")"
+
+# --- Create app secrets ---
+
+log "Creating honeydue-secrets..."
+SECRET_ARGS=(
+  --namespace="${NAMESPACE}"
+  --from-literal="POSTGRES_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/postgres_password.txt")"
+  --from-literal="SECRET_KEY=$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt")"
+  --from-literal="EMAIL_HOST_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/email_host_password.txt")"
+  --from-literal="FCM_SERVER_KEY=${FCM_CONTENT}"
+  --from-literal="MINIO_ROOT_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/minio_root_password.txt")"
+)
+
+if [[ -n "${REDIS_PASSWORD}" ]]; then
+  log "  Including REDIS_PASSWORD in secrets"
+  SECRET_ARGS+=(--from-literal="REDIS_PASSWORD=${REDIS_PASSWORD}")
+fi
+
+kubectl create secret generic honeydue-secrets \
+  "${SECRET_ARGS[@]}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# --- Create APNS key secret ---
+
+log "Creating honeydue-apns-key..."
+kubectl create secret generic honeydue-apns-key \
+  --namespace="${NAMESPACE}" \
+  --from-file="apns_auth_key.p8=${SECRETS_DIR}/apns_auth_key.p8" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# --- Create GHCR registry credentials ---
+
+REGISTRY_SERVER="$(cfg registry.server)"
+REGISTRY_USER="$(cfg registry.username)"
+REGISTRY_TOKEN="$(cfg registry.token)"
+
+if [[ -n "${REGISTRY_SERVER}" && -n "${REGISTRY_USER}" && -n "${REGISTRY_TOKEN}" ]]; then
+  log "Creating ghcr-credentials..."
+  kubectl create secret docker-registry ghcr-credentials \
+    --namespace="${NAMESPACE}" \
+    --docker-server="${REGISTRY_SERVER}" \
+    --docker-username="${REGISTRY_USER}" \
+    --docker-password="${REGISTRY_TOKEN}" \
+    --dry-run=client -o yaml | kubectl apply -f -
+else
+  warn "Registry credentials incomplete in config.yaml — skipping ghcr-credentials."
+fi
+
+# --- Create Cloudflare origin cert (only if cloudflare mode) ---
+
+if [[ "${TLS_MODE}" == "cloudflare" ]]; then
+  require_file "${SECRETS_DIR}/cloudflare-origin.crt" "Cloudflare origin cert"
+  require_file "${SECRETS_DIR}/cloudflare-origin.key" "Cloudflare origin key"
+
+  log "Creating cloudflare-origin-cert..."
+  kubectl create secret tls cloudflare-origin-cert \
+    --namespace="${NAMESPACE}" \
+    --cert="${SECRETS_DIR}/cloudflare-origin.crt" \
+    --key="${SECRETS_DIR}/cloudflare-origin.key" \
+    --dry-run=client -o yaml | kubectl apply -f -
+fi
+
+# --- Create admin basic auth secret ---
+
+if [[ -n "${ADMIN_AUTH_USER}" && -n "${ADMIN_AUTH_PASSWORD}" ]]; then
+  command -v htpasswd >/dev/null 2>&1 || die "Missing: htpasswd (install apache2-utils)"
+  log "Creating admin-basic-auth secret..."
+  HTPASSWD="$(htpasswd -nb "${ADMIN_AUTH_USER}" "${ADMIN_AUTH_PASSWORD}")"
+  kubectl create secret generic admin-basic-auth \
+    --namespace="${NAMESPACE}" \
+    --from-literal=users="${HTPASSWD}" \
+    --dry-run=client -o yaml | kubectl apply -f -
+else
+  warn "admin.basic_auth_user/password not set in config.yaml — skipping admin-basic-auth."
+  warn "Admin panel will NOT have basic auth protection."
+fi
+
+# --- Done ---
+
+log ""
+log "All secrets created in namespace '${NAMESPACE}'."
+log "Verify: kubectl get secrets -n ${NAMESPACE}"
--- a/deploy-k3s-dev/scripts/03-deploy.sh
+++ b/deploy-k3s-dev/scripts/03-deploy.sh
@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_config.sh
+source "${SCRIPT_DIR}/_config.sh"
+
+REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)"
+NAMESPACE="honeydue"
+MANIFESTS="${DEPLOY_DIR}/manifests"
+
+log() { printf '[deploy] %s\n' "$*"; }
+warn() { printf '[deploy][warn] %s\n' "$*" >&2; }
+die() { printf '[deploy][error] %s\n' "$*" >&2; exit 1; }
+
+# --- Parse arguments ---
+
+SKIP_BUILD=false
+DEPLOY_TAG=""
+
+while (( $# > 0 )); do
+  case "$1" in
+    --skip-build) SKIP_BUILD=true; shift ;;
+    --tag)
+      [[ -n "${2:-}" ]] || die "--tag requires a value"
+      DEPLOY_TAG="$2"; shift 2 ;;
+    -h|--help)
+      cat <<'EOF'
+Usage: ./scripts/03-deploy.sh [OPTIONS]
+
+Options:
+  --skip-build       Skip Docker build/push, use existing images
+  --tag <tag>        Image tag (default: git short SHA)
+  -h, --help         Show this help
+EOF
+      exit 0 ;;
+    *) die "Unknown argument: $1" ;;
+  esac
+done
+
+# --- Prerequisites ---
+
+command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl"
+command -v docker >/dev/null 2>&1 || die "Missing: docker"
+
+if [[ -z "${DEPLOY_TAG}" ]]; then
+  DEPLOY_TAG="$(git -C "${REPO_DIR}" rev-parse --short HEAD 2>/dev/null || echo "latest")"
+fi
+
+# --- Read config ---
+
+REGISTRY_SERVER="$(cfg_require registry.server "Container registry server")"
+REGISTRY_NS="$(cfg_require registry.namespace "Registry namespace")"
+REGISTRY_USER="$(cfg_require registry.username "Registry username")"
+REGISTRY_TOKEN="$(cfg_require registry.token "Registry token")"
+TLS_MODE="$(cfg tls.mode 2>/dev/null || echo "letsencrypt")"
+API_DOMAIN="$(cfg_require domains.api "API domain")"
+ADMIN_DOMAIN="$(cfg_require domains.admin "Admin domain")"
+
+REGISTRY_PREFIX="${REGISTRY_SERVER%/}/${REGISTRY_NS#/}"
+API_IMAGE="${REGISTRY_PREFIX}/honeydue-api:${DEPLOY_TAG}"
+WORKER_IMAGE="${REGISTRY_PREFIX}/honeydue-worker:${DEPLOY_TAG}"
+ADMIN_IMAGE="${REGISTRY_PREFIX}/honeydue-admin:${DEPLOY_TAG}"
+
+# --- Build and push ---
+
+if [[ "${SKIP_BUILD}" == "false" ]]; then
+  log "Logging in to ${REGISTRY_SERVER}..."
+  printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_SERVER}" -u "${REGISTRY_USER}" --password-stdin >/dev/null
+
+  log "Building API image: ${API_IMAGE}"
+  docker build --target api -t "${API_IMAGE}" "${REPO_DIR}"
+
+  log "Building Worker image: ${WORKER_IMAGE}"
+  docker build --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}"
+
+  log "Building Admin image: ${ADMIN_IMAGE}"
+  docker build --target admin -t "${ADMIN_IMAGE}" "${REPO_DIR}"
+
+  log "Pushing images..."
+  docker push "${API_IMAGE}"
+  docker push "${WORKER_IMAGE}"
+  docker push "${ADMIN_IMAGE}"
+
+  # Also tag and push :latest
+  docker tag "${API_IMAGE}" "${REGISTRY_PREFIX}/honeydue-api:latest"
+  docker tag "${WORKER_IMAGE}" "${REGISTRY_PREFIX}/honeydue-worker:latest"
+  docker tag "${ADMIN_IMAGE}" "${REGISTRY_PREFIX}/honeydue-admin:latest"
+  docker push "${REGISTRY_PREFIX}/honeydue-api:latest"
+  docker push "${REGISTRY_PREFIX}/honeydue-worker:latest"
+  docker push "${REGISTRY_PREFIX}/honeydue-admin:latest"
+else
+  warn "Skipping build. Using images for tag: ${DEPLOY_TAG}"
+fi
+
+# --- Generate and apply ConfigMap from config.yaml ---
+
+log "Generating env from config.yaml..."
+ENV_FILE="$(mktemp)"
+trap 'rm -f "${ENV_FILE}"' EXIT
+generate_env > "${ENV_FILE}"
+
+log "Creating ConfigMap..."
+kubectl create configmap honeydue-config \
+  --namespace="${NAMESPACE}" \
+  --from-env-file="${ENV_FILE}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# --- Configure TLS ---
+
+if [[ "${TLS_MODE}" == "letsencrypt" ]]; then
+  LE_EMAIL="$(cfg_require tls.letsencrypt_email "Let's Encrypt email")"
+
+  log "Configuring Traefik with Let's Encrypt (${LE_EMAIL})..."
+  sed "s|LETSENCRYPT_EMAIL_PLACEHOLDER|${LE_EMAIL}|" \
+    "${MANIFESTS}/traefik/helmchartconfig.yaml" | kubectl apply -f -
+
+  TLS_SECRET="letsencrypt-cert"
+  TLS_ANNOTATION="traefik.ingress.kubernetes.io/router.tls.certresolver: letsencrypt"
+elif [[ "${TLS_MODE}" == "cloudflare" ]]; then
+  log "Using Cloudflare origin cert for TLS..."
+  TLS_SECRET="cloudflare-origin-cert"
+  TLS_ANNOTATION=""
+else
+  die "Unknown tls.mode: ${TLS_MODE} (expected: letsencrypt or cloudflare)"
+fi
+
+# --- Apply manifests ---
+
+log "Applying manifests..."
+
+kubectl apply -f "${MANIFESTS}/namespace.yaml"
+kubectl apply -f "${MANIFESTS}/rbac.yaml"
+kubectl apply -f "${MANIFESTS}/postgres/"
+kubectl apply -f "${MANIFESTS}/redis/"
+kubectl apply -f "${MANIFESTS}/minio/deployment.yaml"
+kubectl apply -f "${MANIFESTS}/minio/pvc.yaml"
+kubectl apply -f "${MANIFESTS}/minio/service.yaml"
+kubectl apply -f "${MANIFESTS}/ingress/middleware.yaml"
+
+# Apply ingress with domain and TLS substitution
+sed -e "s|API_DOMAIN_PLACEHOLDER|${API_DOMAIN}|g" \
+    -e "s|ADMIN_DOMAIN_PLACEHOLDER|${ADMIN_DOMAIN}|g" \
+    -e "s|TLS_SECRET_PLACEHOLDER|${TLS_SECRET}|g" \
+    -e "s|# TLS_ANNOTATIONS_PLACEHOLDER|${TLS_ANNOTATION}|g" \
+    "${MANIFESTS}/ingress/ingress.yaml" | kubectl apply -f -
+
+# Apply app deployments with image substitution
+sed "s|image: IMAGE_PLACEHOLDER|image: ${API_IMAGE}|" "${MANIFESTS}/api/deployment.yaml" | kubectl apply -f -
+kubectl apply -f "${MANIFESTS}/api/service.yaml"
+
+sed "s|image: IMAGE_PLACEHOLDER|image: ${WORKER_IMAGE}|" "${MANIFESTS}/worker/deployment.yaml" | kubectl apply -f -
+
+sed "s|image: IMAGE_PLACEHOLDER|image: ${ADMIN_IMAGE}|" "${MANIFESTS}/admin/deployment.yaml" | kubectl apply -f -
+kubectl apply -f "${MANIFESTS}/admin/service.yaml"
+
+# Apply network policies
+kubectl apply -f "${MANIFESTS}/network-policies.yaml"
+
+# --- Wait for infrastructure rollouts ---
+
+log "Waiting for infrastructure rollouts..."
+kubectl rollout status deployment/postgres -n "${NAMESPACE}" --timeout=120s
+kubectl rollout status deployment/redis -n "${NAMESPACE}" --timeout=120s
+kubectl rollout status deployment/minio -n "${NAMESPACE}" --timeout=120s
+
+# --- Create MinIO bucket ---
+
+log "Creating MinIO bucket..."
+# Delete previous job run if it exists (jobs are immutable)
+kubectl delete job minio-create-bucket -n "${NAMESPACE}" 2>/dev/null || true
+kubectl apply -f "${MANIFESTS}/minio/create-bucket-job.yaml"
+kubectl wait --for=condition=complete job/minio-create-bucket -n "${NAMESPACE}" --timeout=120s
+
+# --- Wait for app rollouts ---
+
+log "Waiting for app rollouts..."
+kubectl rollout status deployment/api -n "${NAMESPACE}" --timeout=300s
+kubectl rollout status deployment/worker -n "${NAMESPACE}" --timeout=300s
+kubectl rollout status deployment/admin -n "${NAMESPACE}" --timeout=300s
+
+# --- Done ---
+
+log ""
+log "Deploy completed successfully."
+log "Tag: ${DEPLOY_TAG}"
+log "TLS: ${TLS_MODE}"
+log "Images:"
+log "  API:    ${API_IMAGE}"
+log "  Worker: ${WORKER_IMAGE}"
+log "  Admin:  ${ADMIN_IMAGE}"
+log ""
+log "Run ./scripts/04-verify.sh to check cluster health."
--- a/deploy-k3s-dev/scripts/04-verify.sh
+++ b/deploy-k3s-dev/scripts/04-verify.sh
@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="honeydue"
+
+log() { printf '[verify] %s\n' "$*"; }
+sep() { printf '\n%s\n' "--- $1 ---"; }
+ok() { printf '[verify] ✓ %s\n' "$*"; }
+fail() { printf '[verify] ✗ %s\n' "$*"; }
+
+command -v kubectl >/dev/null 2>&1 || { echo "Missing: kubectl" >&2; exit 1; }
+
+sep "Node"
+kubectl get nodes -o wide
+
+sep "Pods"
+kubectl get pods -n "${NAMESPACE}" -o wide
+
+sep "Services"
+kubectl get svc -n "${NAMESPACE}"
+
+sep "Ingress"
+kubectl get ingress -n "${NAMESPACE}"
+
+sep "PVCs"
+kubectl get pvc -n "${NAMESPACE}"
+
+sep "Secrets (names only)"
+kubectl get secrets -n "${NAMESPACE}"
+
+sep "ConfigMap keys"
+kubectl get configmap honeydue-config -n "${NAMESPACE}" -o jsonpath='{.data}' 2>/dev/null | python3 -c "
+import json, sys
+try:
+    d = json.load(sys.stdin)
+    for k in sorted(d.keys()):
+        v = d[k]
+        if any(s in k.upper() for s in ['PASSWORD', 'SECRET', 'TOKEN', 'KEY']):
+            v = '***REDACTED***'
+        print(f'  {k}={v}')
+except:
+    print('  (could not parse)')
+" 2>/dev/null || log "ConfigMap not found or not parseable"
+
+sep "Warning Events (last 15 min)"
+kubectl get events -n "${NAMESPACE}" --field-selector type=Warning --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || log "No warning events"
+
+sep "Pod Restart Counts"
+kubectl get pods -n "${NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' 2>/dev/null || true
+
+# =============================================================================
+# Infrastructure Health
+# =============================================================================
+
+sep "PostgreSQL Health"
+PG_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+if [[ -n "${PG_POD}" ]]; then
+  kubectl exec -n "${NAMESPACE}" "${PG_POD}" -- pg_isready -U honeydue 2>/dev/null && ok "PostgreSQL is ready" || fail "PostgreSQL is NOT ready"
+else
+  fail "No PostgreSQL pod found"
+fi
+
+sep "Redis Health"
+REDIS_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+if [[ -n "${REDIS_POD}" ]]; then
+  kubectl exec -n "${NAMESPACE}" "${REDIS_POD}" -- sh -c 'if [ -n "$REDIS_PASSWORD" ]; then redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null; else redis-cli ping; fi' 2>/dev/null | grep -q PONG && ok "Redis is ready" || fail "Redis is NOT ready"
+else
+  fail "No Redis pod found"
+fi
+
+sep "MinIO Health"
+MINIO_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+if [[ -n "${MINIO_POD}" ]]; then
+  kubectl exec -n "${NAMESPACE}" "${MINIO_POD}" -- curl -sf http://localhost:9000/minio/health/ready 2>/dev/null && ok "MinIO is ready" || fail "MinIO is NOT ready"
+else
+  fail "No MinIO pod found"
+fi
+
+sep "API Health Check"
+API_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+if [[ -n "${API_POD}" ]]; then
+  kubectl exec -n "${NAMESPACE}" "${API_POD}" -- curl -sf http://localhost:8000/api/health/ 2>/dev/null && ok "API health check passed" || fail "API health check FAILED"
+else
+  fail "No API pod found"
+fi
+
+sep "Resource Usage"
+kubectl top pods -n "${NAMESPACE}" 2>/dev/null || log "Metrics server not available (install with: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml)"
+
+# =============================================================================
+# Security Verification
+# =============================================================================
+
+sep "Security: Network Policies"
+NP_COUNT="$(kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')"
+if (( NP_COUNT >= 5 )); then
+  ok "Found ${NP_COUNT} network policies"
+  kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do
+    echo "  ${line}"
+  done
+else
+  fail "Expected 5+ network policies, found ${NP_COUNT}"
+fi
+
+sep "Security: Service Accounts"
+SA_COUNT="$(kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | grep -cv default | tr -d ' ')"
+if (( SA_COUNT >= 6 )); then
+  ok "Found ${SA_COUNT} custom service accounts (api, worker, admin, redis, postgres, minio)"
+else
+  fail "Expected 6 custom service accounts, found ${SA_COUNT}"
+fi
+kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do
+  echo "  ${line}"
+done
+
+sep "Security: Pod Security Contexts"
+PODS_WITHOUT_SECURITY="$(kubectl get pods -n "${NAMESPACE}" -o json 2>/dev/null | python3 -c "
+import json, sys
+try:
+    data = json.load(sys.stdin)
+    issues = []
+    for pod in data.get('items', []):
+        name = pod['metadata']['name']
+        spec = pod['spec']
+        sc = spec.get('securityContext', {})
+        # Postgres is exempt from runAsNonRoot (entrypoint requirement)
+        is_postgres = any('postgres' in c.get('image', '') for c in spec.get('containers', []))
+        if not sc.get('runAsNonRoot') and not is_postgres:
+            issues.append(f'{name}: missing runAsNonRoot')
+        for c in spec.get('containers', []):
+            csc = c.get('securityContext', {})
+            if csc.get('allowPrivilegeEscalation', True):
+                issues.append(f'{name}/{c[\"name\"]}: allowPrivilegeEscalation not false')
+    if issues:
+        for i in issues:
+            print(i)
+    else:
+        print('OK')
+except Exception as e:
+    print(f'Error: {e}')
+" 2>/dev/null || echo "Error parsing pod specs")"
+
+if [[ "${PODS_WITHOUT_SECURITY}" == "OK" ]]; then
+  ok "All pods have proper security contexts"
+else
+  fail "Pod security context issues:"
+  echo "${PODS_WITHOUT_SECURITY}" | while read -r line; do
+    echo "    ${line}"
+  done
+fi
+
+sep "Security: Admin Basic Auth"
+ADMIN_AUTH="$(kubectl get secret admin-basic-auth -n "${NAMESPACE}" -o name 2>/dev/null || true)"
+if [[ -n "${ADMIN_AUTH}" ]]; then
+  ok "admin-basic-auth secret exists"
+else
+  fail "admin-basic-auth secret not found — admin panel has no additional auth layer"
+fi
+
+echo ""
+log "Verification complete."
--- a/deploy-k3s-dev/scripts/_config.sh
+++ b/deploy-k3s-dev/scripts/_config.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+# Shared config helper — sourced by all deploy scripts.
+# Provides cfg() to read values from config.yaml.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+CONFIG_FILE="${DEPLOY_DIR}/config.yaml"
+
+if [[ ! -f "${CONFIG_FILE}" ]]; then
+  if [[ -f "${CONFIG_FILE}.example" ]]; then
+    echo "[error] config.yaml not found. Run: cp config.yaml.example config.yaml" >&2
+  else
+    echo "[error] config.yaml not found." >&2
+  fi
+  exit 1
+fi
+
+# cfg "dotted.key.path" — reads a value from config.yaml
+cfg() {
+  python3 -c "
+import yaml, json, sys
+with open(sys.argv[1]) as f:
+    c = yaml.safe_load(f)
+keys = sys.argv[2].split('.')
+v = c
+for k in keys:
+    if isinstance(v, list):
+        v = v[int(k)]
+    else:
+        v = v[k]
+if isinstance(v, bool):
+    print(str(v).lower())
+elif isinstance(v, (dict, list)):
+    print(json.dumps(v))
+else:
+    print('' if v is None else v)
+" "${CONFIG_FILE}" "$1" 2>/dev/null
+}
+
+# cfg_require "key" "label" — reads value and dies if empty
+cfg_require() {
+  local val
+  val="$(cfg "$1")"
+  if [[ -z "${val}" ]]; then
+    echo "[error] Missing required config: $1 ($2)" >&2
+    exit 1
+  fi
+  printf '%s' "${val}"
+}
+
+# generate_env — writes the flat env file the app expects to stdout
+# Points DB at in-cluster PostgreSQL, storage at in-cluster MinIO
+generate_env() {
+  python3 -c "
+import yaml
+
+with open('${CONFIG_FILE}') as f:
+    c = yaml.safe_load(f)
+
+d = c['domains']
+db = c['database']
+em = c['email']
+ps = c['push']
+st = c['storage']
+wk = c['worker']
+ft = c['features']
+aa = c.get('apple_auth', {})
+ga = c.get('google_auth', {})
+rd = c.get('redis', {})
+
+def b(v):
+    return str(v).lower() if isinstance(v, bool) else str(v)
+
+def val(v):
+    return '' if v is None else str(v)
+
+lines = [
+    # API
+    'DEBUG=true',
+    f\"ALLOWED_HOSTS={d['api']},{d['base']},localhost\",
+    f\"CORS_ALLOWED_ORIGINS=https://{d['base']},https://{d['admin']}\",
+    'TIMEZONE=UTC',
+    f\"BASE_URL=https://{d['base']}\",
+    'PORT=8000',
+    # Admin
+    f\"NEXT_PUBLIC_API_URL=https://{d['api']}\",
+    f\"ADMIN_PANEL_URL=https://{d['admin']}\",
+    # Database (in-cluster PostgreSQL)
+    'DB_HOST=postgres.honeydue.svc.cluster.local',
+    'DB_PORT=5432',
+    f\"POSTGRES_USER={val(db['user'])}\",
+    f\"POSTGRES_DB={db['name']}\",
+    'DB_SSLMODE=disable',
+    f\"DB_MAX_OPEN_CONNS={db['max_open_conns']}\",
+    f\"DB_MAX_IDLE_CONNS={db['max_idle_conns']}\",
+    f\"DB_MAX_LIFETIME={db['max_lifetime']}\",
+    # Redis (in-cluster)
+    f\"REDIS_URL=redis://{':%s@' % val(rd.get('password')) if rd.get('password') else ''}redis.honeydue.svc.cluster.local:6379/0\",
+    'REDIS_DB=0',
+    # Email
+    f\"EMAIL_HOST={em['host']}\",
+    f\"EMAIL_PORT={em['port']}\",
+    f\"EMAIL_USE_TLS={b(em['use_tls'])}\",
+    f\"EMAIL_HOST_USER={val(em['user'])}\",
+    f\"DEFAULT_FROM_EMAIL={val(em['from'])}\",
+    # Push
+    'APNS_AUTH_KEY_PATH=/secrets/apns/apns_auth_key.p8',
+    f\"APNS_AUTH_KEY_ID={val(ps['apns_key_id'])}\",
+    f\"APNS_TEAM_ID={val(ps['apns_team_id'])}\",
+    f\"APNS_TOPIC={ps['apns_topic']}\",
+    f\"APNS_USE_SANDBOX={b(ps['apns_use_sandbox'])}\",
+    f\"APNS_PRODUCTION={b(ps['apns_production'])}\",
+    # Worker
+    f\"TASK_REMINDER_HOUR={wk['task_reminder_hour']}\",
+    f\"OVERDUE_REMINDER_HOUR={wk['overdue_reminder_hour']}\",
+    f\"DAILY_DIGEST_HOUR={wk['daily_digest_hour']}\",
+    # Storage (in-cluster MinIO — S3-compatible, same env vars as B2)
+    f\"B2_KEY_ID={val(st['minio_root_user'])}\",
+    # B2_APP_KEY injected from secret (MINIO_ROOT_PASSWORD)
+    f\"B2_BUCKET_NAME={val(st['bucket'])}\",
+    'B2_ENDPOINT=minio.honeydue.svc.cluster.local:9000',
+    'STORAGE_USE_SSL=false',
+    f\"STORAGE_MAX_FILE_SIZE={st['max_file_size']}\",
+    f\"STORAGE_ALLOWED_TYPES={st['allowed_types']}\",
+    # MinIO root user (for MinIO deployment + bucket init job)
+    f\"MINIO_ROOT_USER={val(st['minio_root_user'])}\",
+    # Features
+    f\"FEATURE_PUSH_ENABLED={b(ft['push_enabled'])}\",
+    f\"FEATURE_EMAIL_ENABLED={b(ft['email_enabled'])}\",
+    f\"FEATURE_WEBHOOKS_ENABLED={b(ft['webhooks_enabled'])}\",
+    f\"FEATURE_ONBOARDING_EMAILS_ENABLED={b(ft['onboarding_emails_enabled'])}\",
+    f\"FEATURE_PDF_REPORTS_ENABLED={b(ft['pdf_reports_enabled'])}\",
+    f\"FEATURE_WORKER_ENABLED={b(ft['worker_enabled'])}\",
+    # Apple auth/IAP
+    f\"APPLE_CLIENT_ID={val(aa.get('client_id'))}\",
+    f\"APPLE_TEAM_ID={val(aa.get('team_id'))}\",
+    f\"APPLE_IAP_KEY_ID={val(aa.get('iap_key_id'))}\",
+    f\"APPLE_IAP_ISSUER_ID={val(aa.get('iap_issuer_id'))}\",
+    f\"APPLE_IAP_BUNDLE_ID={val(aa.get('iap_bundle_id'))}\",
+    f\"APPLE_IAP_KEY_PATH={val(aa.get('iap_key_path'))}\",
+    f\"APPLE_IAP_SANDBOX={b(aa.get('iap_sandbox', True))}\",
+    # Google auth/IAP
+    f\"GOOGLE_CLIENT_ID={val(ga.get('client_id'))}\",
+    f\"GOOGLE_ANDROID_CLIENT_ID={val(ga.get('android_client_id'))}\",
+    f\"GOOGLE_IOS_CLIENT_ID={val(ga.get('ios_client_id'))}\",
+    f\"GOOGLE_IAP_PACKAGE_NAME={val(ga.get('iap_package_name'))}\",
+    f\"GOOGLE_IAP_SERVICE_ACCOUNT_PATH={val(ga.get('iap_service_account_path'))}\",
+]
+
+print('\n'.join(lines))
+"
+}
--- a/deploy-k3s-dev/scripts/rollback.sh
+++ b/deploy-k3s-dev/scripts/rollback.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+NAMESPACE="honeydue"
+
+log() { printf '[rollback] %s\n' "$*"; }
+die() { printf '[rollback][error] %s\n' "$*" >&2; exit 1; }
+
+command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl"
+
+DEPLOYMENTS=("api" "worker" "admin")
+
+# --- Show current state ---
+
+echo "=== Current Rollout History ==="
+for deploy in "${DEPLOYMENTS[@]}"; do
+  echo ""
+  echo "--- ${deploy} ---"
+  kubectl rollout history deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || echo "  (not found)"
+done
+
+echo ""
+echo "=== Current Images ==="
+for deploy in "${DEPLOYMENTS[@]}"; do
+  IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")"
+  echo "  ${deploy}: ${IMAGE}"
+done
+
+# --- Confirm ---
+
+echo ""
+read -rp "Roll back all deployments to previous revision? [y/N] " confirm
+if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then
+  log "Aborted."
+  exit 0
+fi
+
+# --- Rollback ---
+
+for deploy in "${DEPLOYMENTS[@]}"; do
+  log "Rolling back ${deploy}..."
+  kubectl rollout undo deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || log "Skipping ${deploy} (not found or no previous revision)"
+done
+
+# --- Wait ---
+
+log "Waiting for rollouts..."
+for deploy in "${DEPLOYMENTS[@]}"; do
+  kubectl rollout status deployment/"${deploy}" -n "${NAMESPACE}" --timeout=300s 2>/dev/null || true
+done
+
+# --- Verify ---
+
+echo ""
+echo "=== Post-Rollback Images ==="
+for deploy in "${DEPLOYMENTS[@]}"; do
+  IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")"
+  echo "  ${deploy}: ${IMAGE}"
+done
+
+log "Rollback complete. Run ./scripts/04-verify.sh to check health."