From 34553f3bec74d73109c445cb3554a65809ff7867 Mon Sep 17 00:00:00 2001 From: Trey t Date: Mon, 30 Mar 2026 21:30:39 -0500 Subject: [PATCH] Add K3s dev deployment setup for single-node VPS Mirrors the prod deploy-k3s/ setup but runs all services in-cluster on a single node: PostgreSQL (replaces Neon), MinIO S3-compatible storage (replaces B2), Redis, API, worker, and admin. Includes fully automated setup scripts (00-init through 04-verify), server hardening (SSH, fail2ban, ufw), Let's Encrypt TLS via Traefik, network policies, RBAC, and security contexts matching prod. Co-Authored-By: Claude Opus 4.6 (1M context) --- deploy-k3s-dev/.gitignore | 13 + deploy-k3s-dev/README.md | 78 ++ deploy-k3s-dev/config.yaml.example | 103 +++ .../manifests/admin/deployment.yaml | 94 ++ deploy-k3s-dev/manifests/admin/service.yaml | 16 + deploy-k3s-dev/manifests/ingress/ingress.yaml | 56 ++ .../manifests/ingress/middleware.yaml | 45 + .../manifests/minio/create-bucket-job.yaml | 81 ++ .../manifests/minio/deployment.yaml | 89 ++ deploy-k3s-dev/manifests/minio/pvc.yaml | 15 + deploy-k3s-dev/manifests/minio/service.yaml | 21 + deploy-k3s-dev/manifests/namespace.yaml | 6 + .../manifests/network-policies.yaml | 305 +++++++ .../manifests/postgres/deployment.yaml | 93 ++ deploy-k3s-dev/manifests/postgres/pvc.yaml | 15 + .../manifests/postgres/service.yaml | 16 + deploy-k3s-dev/manifests/rbac.yaml | 68 ++ .../manifests/redis/deployment.yaml | 105 +++ deploy-k3s-dev/manifests/redis/pvc.yaml | 15 + deploy-k3s-dev/manifests/redis/service.yaml | 16 + .../manifests/traefik/helmchartconfig.yaml | 16 + deploy-k3s-dev/scripts/00-init.sh | 235 +++++ deploy-k3s-dev/scripts/01-setup-k3s.sh | 146 ++++ deploy-k3s-dev/scripts/02-setup-secrets.sh | 153 ++++ deploy-k3s-dev/scripts/03-deploy.sh | 193 +++++ deploy-k3s-dev/scripts/04-verify.sh | 161 ++++ deploy-k3s-dev/scripts/_config.sh | 152 ++++ deploy-k3s-dev/scripts/rollback.sh | 61 ++ deploy-k3s-dev/secrets/README.md | 22 + deploy-k3s/.gitignore | 20 + deploy-k3s/README.md | 391 +++++++++ deploy-k3s/SECURITY.md | 813 ++++++++++++++++++ deploy-k3s/config.yaml.example | 118 +++ deploy-k3s/manifests/admin/deployment.yaml | 94 ++ deploy-k3s/manifests/admin/service.yaml | 16 + deploy-k3s/manifests/ingress/ingress.yaml | 54 ++ deploy-k3s/manifests/ingress/middleware.yaml | 82 ++ deploy-k3s/manifests/namespace.yaml | 6 + deploy-k3s/manifests/network-policies.yaml | 202 +++++ .../manifests/pod-disruption-budgets.yaml | 32 + deploy-k3s/manifests/rbac.yaml | 46 + deploy-k3s/manifests/redis/deployment.yaml | 106 +++ deploy-k3s/manifests/redis/pvc.yaml | 15 + deploy-k3s/manifests/redis/service.yaml | 16 + deploy-k3s/manifests/secrets.yaml.example | 47 + deploy-k3s/scripts/01-provision-cluster.sh | 124 +++ deploy-k3s/scripts/02-setup-secrets.sh | 131 +++ deploy-k3s/scripts/03-deploy.sh | 143 +++ deploy-k3s/scripts/04-verify.sh | 180 ++++ deploy-k3s/scripts/_config.sh | 214 +++++ deploy-k3s/scripts/rollback.sh | 61 ++ deploy-k3s/secrets/README.md | 19 + 52 files changed, 5319 insertions(+) create mode 100644 deploy-k3s-dev/.gitignore create mode 100644 deploy-k3s-dev/README.md create mode 100644 deploy-k3s-dev/config.yaml.example create mode 100644 deploy-k3s-dev/manifests/admin/deployment.yaml create mode 100644 deploy-k3s-dev/manifests/admin/service.yaml create mode 100644 deploy-k3s-dev/manifests/ingress/ingress.yaml create mode 100644 deploy-k3s-dev/manifests/ingress/middleware.yaml create mode 100644 deploy-k3s-dev/manifests/minio/create-bucket-job.yaml create mode 100644 deploy-k3s-dev/manifests/minio/deployment.yaml create mode 100644 deploy-k3s-dev/manifests/minio/pvc.yaml create mode 100644 deploy-k3s-dev/manifests/minio/service.yaml create mode 100644 deploy-k3s-dev/manifests/namespace.yaml create mode 100644 deploy-k3s-dev/manifests/network-policies.yaml create mode 100644 deploy-k3s-dev/manifests/postgres/deployment.yaml create mode 100644 deploy-k3s-dev/manifests/postgres/pvc.yaml create mode 100644 deploy-k3s-dev/manifests/postgres/service.yaml create mode 100644 deploy-k3s-dev/manifests/rbac.yaml create mode 100644 deploy-k3s-dev/manifests/redis/deployment.yaml create mode 100644 deploy-k3s-dev/manifests/redis/pvc.yaml create mode 100644 deploy-k3s-dev/manifests/redis/service.yaml create mode 100644 deploy-k3s-dev/manifests/traefik/helmchartconfig.yaml create mode 100755 deploy-k3s-dev/scripts/00-init.sh create mode 100755 deploy-k3s-dev/scripts/01-setup-k3s.sh create mode 100755 deploy-k3s-dev/scripts/02-setup-secrets.sh create mode 100755 deploy-k3s-dev/scripts/03-deploy.sh create mode 100755 deploy-k3s-dev/scripts/04-verify.sh create mode 100755 deploy-k3s-dev/scripts/_config.sh create mode 100755 deploy-k3s-dev/scripts/rollback.sh create mode 100644 deploy-k3s-dev/secrets/README.md create mode 100644 deploy-k3s/.gitignore create mode 100644 deploy-k3s/README.md create mode 100644 deploy-k3s/SECURITY.md create mode 100644 deploy-k3s/config.yaml.example create mode 100644 deploy-k3s/manifests/admin/deployment.yaml create mode 100644 deploy-k3s/manifests/admin/service.yaml create mode 100644 deploy-k3s/manifests/ingress/ingress.yaml create mode 100644 deploy-k3s/manifests/ingress/middleware.yaml create mode 100644 deploy-k3s/manifests/namespace.yaml create mode 100644 deploy-k3s/manifests/network-policies.yaml create mode 100644 deploy-k3s/manifests/pod-disruption-budgets.yaml create mode 100644 deploy-k3s/manifests/rbac.yaml create mode 100644 deploy-k3s/manifests/redis/deployment.yaml create mode 100644 deploy-k3s/manifests/redis/pvc.yaml create mode 100644 deploy-k3s/manifests/redis/service.yaml create mode 100644 deploy-k3s/manifests/secrets.yaml.example create mode 100755 deploy-k3s/scripts/01-provision-cluster.sh create mode 100755 deploy-k3s/scripts/02-setup-secrets.sh create mode 100755 deploy-k3s/scripts/03-deploy.sh create mode 100755 deploy-k3s/scripts/04-verify.sh create mode 100755 deploy-k3s/scripts/_config.sh create mode 100755 deploy-k3s/scripts/rollback.sh create mode 100644 deploy-k3s/secrets/README.md diff --git a/deploy-k3s-dev/.gitignore b/deploy-k3s-dev/.gitignore new file mode 100644 index 0000000..0696d08 --- /dev/null +++ b/deploy-k3s-dev/.gitignore @@ -0,0 +1,13 @@ +# Single config file (contains tokens and credentials) +config.yaml + +# Generated files +kubeconfig + +# Secret files +secrets/*.txt +secrets/*.p8 +secrets/*.pem +secrets/*.key +secrets/*.crt +!secrets/README.md diff --git a/deploy-k3s-dev/README.md b/deploy-k3s-dev/README.md new file mode 100644 index 0000000..ffc554a --- /dev/null +++ b/deploy-k3s-dev/README.md @@ -0,0 +1,78 @@ +# honeyDue — K3s Dev Deployment + +Single-node K3s dev environment that replicates the production setup with all services running locally. + +**Architecture**: 1-node K3s, in-cluster PostgreSQL + Redis + MinIO (S3-compatible), Let's Encrypt TLS. + +**Domains**: `devapi.myhoneydue.com`, `devadmin.myhoneydue.com` + +--- + +## Quick Start + +```bash +cd honeyDueAPI-go/deploy-k3s-dev + +# 1. Fill in config +cp config.yaml.example config.yaml +# Edit config.yaml — fill in ALL empty values + +# 2. Create secret files (see secrets/README.md) +echo "your-postgres-password" > secrets/postgres_password.txt +openssl rand -base64 48 > secrets/secret_key.txt +echo "your-smtp-password" > secrets/email_host_password.txt +echo "your-fcm-key" > secrets/fcm_server_key.txt +openssl rand -base64 24 > secrets/minio_root_password.txt +cp /path/to/AuthKey.p8 secrets/apns_auth_key.p8 + +# 3. Install K3s → Create secrets → Deploy +./scripts/01-setup-k3s.sh +./scripts/02-setup-secrets.sh +./scripts/03-deploy.sh + +# 4. Point DNS at the server IP, then verify +./scripts/04-verify.sh +curl https://devapi.myhoneydue.com/api/health/ +``` + +## Prod vs Dev + +| Component | Prod (`deploy-k3s/`) | Dev (`deploy-k3s-dev/`) | +|---|---|---| +| Nodes | 3x CX33 (HA etcd) | 1 node (any VPS) | +| PostgreSQL | Neon (managed) | In-cluster container | +| File storage | Backblaze B2 | MinIO (S3-compatible) | +| Redis | In-cluster | In-cluster (identical) | +| TLS | Cloudflare origin cert | Let's Encrypt (or Cloudflare) | +| Replicas | api=3, worker=2 | All 1 | +| HPA/PDB | Enabled | Not deployed | +| Network policies | Same | Same + postgres/minio rules | +| Security contexts | Same | Same (except postgres) | +| Deploy workflow | Same scripts | Same scripts | +| Docker images | Same | Same | + +## TLS Modes + +**Let's Encrypt** (default): Traefik auto-provisions certs. Set `tls.letsencrypt_email` in config.yaml. + +**Cloudflare**: Same as prod. Set `tls.mode: cloudflare`, add origin cert files to `secrets/`. + +## Storage Note + +MinIO provides the same S3-compatible API as Backblaze B2. The Go API uses the same env vars (`B2_KEY_ID`, `B2_APP_KEY`, `B2_BUCKET_NAME`, `B2_ENDPOINT`) — it connects to MinIO instead of B2 without code changes. + +An additional env var `STORAGE_USE_SSL=false` is set since MinIO runs in-cluster over HTTP. If the Go storage service hardcodes HTTPS, it may need a small change to respect this flag. + +## Monitoring + +```bash +stern -n honeydue . # All logs +kubectl logs -n honeydue deploy/api -f # API logs +kubectl top pods -n honeydue # Resource usage +``` + +## Rollback + +```bash +./scripts/rollback.sh +``` diff --git a/deploy-k3s-dev/config.yaml.example b/deploy-k3s-dev/config.yaml.example new file mode 100644 index 0000000..50fdf6b --- /dev/null +++ b/deploy-k3s-dev/config.yaml.example @@ -0,0 +1,103 @@ +# config.yaml — single source of truth for honeyDue K3s DEV deployment +# Copy to config.yaml, fill in all empty values, then run scripts in order. +# This file is gitignored — never commit it with real values. + +# --- Server --- +server: + host: "" # Server IP or SSH config alias + user: root # SSH user + ssh_key: ~/.ssh/id_ed25519 + +# --- Domains --- +domains: + api: devapi.myhoneydue.com + admin: devadmin.myhoneydue.com + base: dev.myhoneydue.com + +# --- Container Registry (GHCR) --- +registry: + server: ghcr.io + namespace: "" # GitHub username or org + username: "" # GitHub username + token: "" # PAT with read:packages, write:packages + +# --- Database (in-cluster PostgreSQL) --- +database: + name: honeydue_dev + user: honeydue + # password goes in secrets/postgres_password.txt + max_open_conns: 10 + max_idle_conns: 5 + max_lifetime: "600s" + +# --- Email (Fastmail) --- +email: + host: smtp.fastmail.com + port: 587 + user: "" # Fastmail email address + from: "honeyDue DEV " + use_tls: true + +# --- Push Notifications --- +push: + apns_key_id: "" + apns_team_id: "" + apns_topic: com.tt.honeyDue + apns_production: false + apns_use_sandbox: true # Sandbox for dev + +# --- Object Storage (in-cluster MinIO — S3-compatible, replaces B2) --- +storage: + minio_root_user: honeydue # MinIO access key + # minio_root_password goes in secrets/minio_root_password.txt + bucket: honeydue-dev + max_file_size: 10485760 + allowed_types: "image/jpeg,image/png,image/gif,image/webp,application/pdf" + +# --- Worker Schedules (UTC hours) --- +worker: + task_reminder_hour: 14 + overdue_reminder_hour: 15 + daily_digest_hour: 3 + +# --- Feature Flags --- +features: + push_enabled: true + email_enabled: false # Disabled for dev by default + webhooks_enabled: false + onboarding_emails_enabled: false + pdf_reports_enabled: true + worker_enabled: true + +# --- Redis --- +redis: + password: "" # Set a strong password + +# --- Admin Panel --- +admin: + basic_auth_user: "" # HTTP basic auth username + basic_auth_password: "" # HTTP basic auth password + +# --- TLS --- +tls: + mode: letsencrypt # "letsencrypt" or "cloudflare" + letsencrypt_email: "" # Required if mode=letsencrypt + # If mode=cloudflare, create secrets/cloudflare-origin.crt and .key + +# --- Apple Auth / IAP (optional) --- +apple_auth: + client_id: "" + team_id: "" + iap_key_id: "" + iap_issuer_id: "" + iap_bundle_id: "" + iap_key_path: "" + iap_sandbox: true + +# --- Google Auth / IAP (optional) --- +google_auth: + client_id: "" + android_client_id: "" + ios_client_id: "" + iap_package_name: "" + iap_service_account_path: "" diff --git a/deploy-k3s-dev/manifests/admin/deployment.yaml b/deploy-k3s-dev/manifests/admin/deployment.yaml new file mode 100644 index 0000000..4a33296 --- /dev/null +++ b/deploy-k3s-dev/manifests/admin/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: admin + template: + metadata: + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: admin + imagePullSecrets: + - name: ghcr-credentials + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: admin + image: IMAGE_PLACEHOLDER # Replaced by 03-deploy.sh + ports: + - containerPort: 3000 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: PORT + value: "3000" + - name: HOSTNAME + value: "0.0.0.0" + - name: NEXT_PUBLIC_API_URL + valueFrom: + configMapKeyRef: + name: honeydue-config + key: NEXT_PUBLIC_API_URL + volumeMounts: + - name: nextjs-cache + mountPath: /app/.next/cache + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + startupProbe: + httpGet: + path: /admin/ + port: 3000 + failureThreshold: 12 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /admin/ + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /admin/ + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + volumes: + - name: nextjs-cache + emptyDir: + sizeLimit: 256Mi + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/deploy-k3s-dev/manifests/admin/service.yaml b/deploy-k3s-dev/manifests/admin/service.yaml new file mode 100644 index 0000000..fa7a4ff --- /dev/null +++ b/deploy-k3s-dev/manifests/admin/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: admin + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP diff --git a/deploy-k3s-dev/manifests/ingress/ingress.yaml b/deploy-k3s-dev/manifests/ingress/ingress.yaml new file mode 100644 index 0000000..3bd9c57 --- /dev/null +++ b/deploy-k3s-dev/manifests/ingress/ingress.yaml @@ -0,0 +1,56 @@ +# API Ingress — TLS via Let's Encrypt (default) or Cloudflare origin cert +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: honeydue-api + namespace: honeydue + labels: + app.kubernetes.io/part-of: honeydue + annotations: + # TLS_ANNOTATIONS_PLACEHOLDER — replaced by 03-deploy.sh based on tls.mode + traefik.ingress.kubernetes.io/router.middlewares: honeydue-security-headers@kubernetescrd,honeydue-rate-limit@kubernetescrd +spec: + tls: + - hosts: + - API_DOMAIN_PLACEHOLDER + secretName: TLS_SECRET_PLACEHOLDER + rules: + - host: API_DOMAIN_PLACEHOLDER + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api + port: + number: 8000 + +--- +# Admin Ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: honeydue-admin + namespace: honeydue + labels: + app.kubernetes.io/part-of: honeydue + annotations: + # TLS_ANNOTATIONS_PLACEHOLDER — replaced by 03-deploy.sh based on tls.mode + traefik.ingress.kubernetes.io/router.middlewares: honeydue-security-headers@kubernetescrd,honeydue-rate-limit@kubernetescrd,honeydue-admin-auth@kubernetescrd +spec: + tls: + - hosts: + - ADMIN_DOMAIN_PLACEHOLDER + secretName: TLS_SECRET_PLACEHOLDER + rules: + - host: ADMIN_DOMAIN_PLACEHOLDER + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: admin + port: + number: 3000 diff --git a/deploy-k3s-dev/manifests/ingress/middleware.yaml b/deploy-k3s-dev/manifests/ingress/middleware.yaml new file mode 100644 index 0000000..4964663 --- /dev/null +++ b/deploy-k3s-dev/manifests/ingress/middleware.yaml @@ -0,0 +1,45 @@ +# Traefik CRD middleware for rate limiting +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: rate-limit + namespace: honeydue +spec: + rateLimit: + average: 100 + burst: 200 + period: 1m + +--- +# Security headers +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: honeydue +spec: + headers: + frameDeny: true + contentTypeNosniff: true + browserXssFilter: true + referrerPolicy: "strict-origin-when-cross-origin" + customResponseHeaders: + X-Content-Type-Options: "nosniff" + X-Frame-Options: "DENY" + Strict-Transport-Security: "max-age=31536000; includeSubDomains" + Content-Security-Policy: "default-src 'self'; frame-ancestors 'none'" + Permissions-Policy: "camera=(), microphone=(), geolocation=()" + X-Permitted-Cross-Domain-Policies: "none" + +--- +# Admin basic auth — additional auth layer for admin panel +# Secret created by 02-setup-secrets.sh from config.yaml credentials +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: admin-auth + namespace: honeydue +spec: + basicAuth: + secret: admin-basic-auth + realm: "honeyDue Admin" diff --git a/deploy-k3s-dev/manifests/minio/create-bucket-job.yaml b/deploy-k3s-dev/manifests/minio/create-bucket-job.yaml new file mode 100644 index 0000000..aa7c0a2 --- /dev/null +++ b/deploy-k3s-dev/manifests/minio/create-bucket-job.yaml @@ -0,0 +1,81 @@ +# One-shot job to create the default bucket in MinIO. +# Applied by 03-deploy.sh after MinIO is running. +# Re-running is safe — mc mb --ignore-existing is idempotent. +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-create-bucket + namespace: honeydue + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue +spec: + ttlSecondsAfterFinished: 300 + backoffLimit: 5 + template: + metadata: + labels: + app.kubernetes.io/name: minio-init + app.kubernetes.io/part-of: honeydue + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: mc + image: minio/mc:latest + command: + - sh + - -c + - | + echo "Waiting for MinIO to be ready..." + until mc alias set honeydue http://minio.honeydue.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do + sleep 2 + done + echo "Creating bucket: $BUCKET_NAME" + mc mb --ignore-existing "honeydue/$BUCKET_NAME" + echo "Bucket ready." + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: MINIO_ROOT_USER + valueFrom: + configMapKeyRef: + name: honeydue-config + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: honeydue-secrets + key: MINIO_ROOT_PASSWORD + - name: BUCKET_NAME + valueFrom: + configMapKeyRef: + name: honeydue-config + key: B2_BUCKET_NAME + volumeMounts: + - name: tmp + mountPath: /tmp + - name: mc-config + mountPath: /.mc + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 64Mi + volumes: + - name: tmp + emptyDir: + sizeLimit: 16Mi + - name: mc-config + emptyDir: + sizeLimit: 16Mi + restartPolicy: OnFailure diff --git a/deploy-k3s-dev/manifests/minio/deployment.yaml b/deploy-k3s-dev/manifests/minio/deployment.yaml new file mode 100644 index 0000000..fb9e1bf --- /dev/null +++ b/deploy-k3s-dev/manifests/minio/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: honeydue + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: Recreate # ReadWriteOnce PVC — can't attach to two pods + selector: + matchLabels: + app.kubernetes.io/name: minio + template: + metadata: + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: minio + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: minio + image: minio/minio:latest + args: ["server", "/data", "--console-address", ":9001"] + ports: + - name: api + containerPort: 9000 + protocol: TCP + - name: console + containerPort: 9001 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: MINIO_ROOT_USER + valueFrom: + configMapKeyRef: + name: honeydue-config + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: honeydue-secrets + key: MINIO_ROOT_PASSWORD + volumeMounts: + - name: minio-data + mountPath: /data + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-data + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/deploy-k3s-dev/manifests/minio/pvc.yaml b/deploy-k3s-dev/manifests/minio/pvc.yaml new file mode 100644 index 0000000..9f6102d --- /dev/null +++ b/deploy-k3s-dev/manifests/minio/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: honeydue + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi diff --git a/deploy-k3s-dev/manifests/minio/service.yaml b/deploy-k3s-dev/manifests/minio/service.yaml new file mode 100644 index 0000000..a492b53 --- /dev/null +++ b/deploy-k3s-dev/manifests/minio/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: honeydue + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: minio + ports: + - name: api + port: 9000 + targetPort: 9000 + protocol: TCP + - name: console + port: 9001 + targetPort: 9001 + protocol: TCP diff --git a/deploy-k3s-dev/manifests/namespace.yaml b/deploy-k3s-dev/manifests/namespace.yaml new file mode 100644 index 0000000..76f4908 --- /dev/null +++ b/deploy-k3s-dev/manifests/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: honeydue + labels: + app.kubernetes.io/part-of: honeydue diff --git a/deploy-k3s-dev/manifests/network-policies.yaml b/deploy-k3s-dev/manifests/network-policies.yaml new file mode 100644 index 0000000..bd016fb --- /dev/null +++ b/deploy-k3s-dev/manifests/network-policies.yaml @@ -0,0 +1,305 @@ +# Network Policies — default-deny with explicit allows +# Same pattern as prod, with added rules for in-cluster postgres and minio. + +# --- Default deny all ingress and egress --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: honeydue +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + +--- +# --- Allow DNS for all pods (required for service discovery) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns + namespace: honeydue +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: [] + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + +--- +# --- API: allow ingress from Traefik (kube-system namespace) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-api + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: api + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: TCP + port: 8000 + +--- +# --- Admin: allow ingress from Traefik (kube-system namespace) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-admin + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: admin + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: TCP + port: 3000 + +--- +# --- Redis: allow ingress ONLY from api + worker pods --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-redis + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: redis + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + - podSelector: + matchLabels: + app.kubernetes.io/name: worker + ports: + - protocol: TCP + port: 6379 + +--- +# --- PostgreSQL: allow ingress ONLY from api + worker pods --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-postgres + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + - podSelector: + matchLabels: + app.kubernetes.io/name: worker + ports: + - protocol: TCP + port: 5432 + +--- +# --- MinIO: allow ingress from api + worker + minio-init job pods --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-minio + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: minio + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + - podSelector: + matchLabels: + app.kubernetes.io/name: worker + - podSelector: + matchLabels: + app.kubernetes.io/name: minio-init + ports: + - protocol: TCP + port: 9000 + - protocol: TCP + port: 9001 + +--- +# --- API: allow egress to Redis, PostgreSQL, MinIO, external services --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-api + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: api + policyTypes: + - Egress + egress: + # Redis (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + # PostgreSQL (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: postgres + ports: + - protocol: TCP + port: 5432 + # MinIO (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + ports: + - protocol: TCP + port: 9000 + # External services: SMTP (587), HTTPS (443 — APNs, FCM, PostHog) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 587 + - protocol: TCP + port: 443 + +--- +# --- Worker: allow egress to Redis, PostgreSQL, MinIO, external services --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-worker + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: worker + policyTypes: + - Egress + egress: + # Redis (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + # PostgreSQL (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: postgres + ports: + - protocol: TCP + port: 5432 + # MinIO (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + ports: + - protocol: TCP + port: 9000 + # External services: SMTP (587), HTTPS (443 — APNs, FCM) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 587 + - protocol: TCP + port: 443 + +--- +# --- Admin: allow egress to API (internal) for SSR --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-admin + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: admin + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + ports: + - protocol: TCP + port: 8000 + +--- +# --- MinIO init job: allow egress to MinIO --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-minio-init + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: minio-init + policyTypes: + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + ports: + - protocol: TCP + port: 9000 diff --git a/deploy-k3s-dev/manifests/postgres/deployment.yaml b/deploy-k3s-dev/manifests/postgres/deployment.yaml new file mode 100644 index 0000000..c0fb514 --- /dev/null +++ b/deploy-k3s-dev/manifests/postgres/deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: honeydue + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: Recreate # ReadWriteOnce PVC — can't attach to two pods + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: postgres + # Note: postgres image entrypoint requires root initially to set up + # permissions, then drops to the postgres user. runAsNonRoot is not set + # here because of this requirement. This differs from prod which uses + # managed Neon PostgreSQL (no container to secure). + securityContext: + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: postgres + image: postgres:17-alpine + ports: + - containerPort: 5432 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: honeydue-config + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: honeydue-config + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: honeydue-secrets + key: POSTGRES_PASSWORD + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + - name: run + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + exec: + command: ["pg_isready", "-U", "honeydue"] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["pg_isready", "-U", "honeydue"] + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data + - name: run + emptyDir: {} + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/deploy-k3s-dev/manifests/postgres/pvc.yaml b/deploy-k3s-dev/manifests/postgres/pvc.yaml new file mode 100644 index 0000000..398be23 --- /dev/null +++ b/deploy-k3s-dev/manifests/postgres/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data + namespace: honeydue + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: honeydue +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi diff --git a/deploy-k3s-dev/manifests/postgres/service.yaml b/deploy-k3s-dev/manifests/postgres/service.yaml new file mode 100644 index 0000000..8058a63 --- /dev/null +++ b/deploy-k3s-dev/manifests/postgres/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: honeydue + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: postgres + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/deploy-k3s-dev/manifests/rbac.yaml b/deploy-k3s-dev/manifests/rbac.yaml new file mode 100644 index 0000000..2f3fcb8 --- /dev/null +++ b/deploy-k3s-dev/manifests/rbac.yaml @@ -0,0 +1,68 @@ +# RBAC — Dedicated service accounts with no K8s API access +# Each pod gets its own SA with automountServiceAccountToken: false, +# so a compromised pod cannot query the Kubernetes API. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: api + namespace: honeydue + labels: + app.kubernetes.io/name: api + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: worker + namespace: honeydue + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgres + namespace: honeydue + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minio + namespace: honeydue + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false diff --git a/deploy-k3s-dev/manifests/redis/deployment.yaml b/deploy-k3s-dev/manifests/redis/deployment.yaml new file mode 100644 index 0000000..509eff1 --- /dev/null +++ b/deploy-k3s-dev/manifests/redis/deployment.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: Recreate # ReadWriteOnce PVC — can't attach to two pods + selector: + matchLabels: + app.kubernetes.io/name: redis + template: + metadata: + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: redis + # No nodeSelector — single node dev cluster + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: redis + image: redis:7-alpine + command: + - sh + - -c + - | + ARGS="--appendonly yes --appendfsync everysec --maxmemory 256mb --maxmemory-policy noeviction" + if [ -n "$REDIS_PASSWORD" ]; then + ARGS="$ARGS --requirepass $REDIS_PASSWORD" + fi + exec redis-server $ARGS + ports: + - containerPort: 6379 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: honeydue-secrets + key: REDIS_PASSWORD + optional: true + volumeMounts: + - name: redis-data + mountPath: /data + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + exec: + command: + - sh + - -c + - | + if [ -n "$REDIS_PASSWORD" ]; then + redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null | grep -q PONG + else + redis-cli ping | grep -q PONG + fi + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - sh + - -c + - | + if [ -n "$REDIS_PASSWORD" ]; then + redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null | grep -q PONG + else + redis-cli ping | grep -q PONG + fi + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 64Mi diff --git a/deploy-k3s-dev/manifests/redis/pvc.yaml b/deploy-k3s-dev/manifests/redis/pvc.yaml new file mode 100644 index 0000000..dbddee5 --- /dev/null +++ b/deploy-k3s-dev/manifests/redis/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 5Gi diff --git a/deploy-k3s-dev/manifests/redis/service.yaml b/deploy-k3s-dev/manifests/redis/service.yaml new file mode 100644 index 0000000..9bed2ea --- /dev/null +++ b/deploy-k3s-dev/manifests/redis/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: redis + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP diff --git a/deploy-k3s-dev/manifests/traefik/helmchartconfig.yaml b/deploy-k3s-dev/manifests/traefik/helmchartconfig.yaml new file mode 100644 index 0000000..96dc9b3 --- /dev/null +++ b/deploy-k3s-dev/manifests/traefik/helmchartconfig.yaml @@ -0,0 +1,16 @@ +# Configure K3s's built-in Traefik with Let's Encrypt ACME. +# Applied by 03-deploy.sh only when tls.mode=letsencrypt. +# The email placeholder is replaced by the deploy script. +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + additionalArguments: + - "--certificatesresolvers.letsencrypt.acme.email=LETSENCRYPT_EMAIL_PLACEHOLDER" + - "--certificatesresolvers.letsencrypt.acme.storage=/data/acme.json" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + persistence: + enabled: true diff --git a/deploy-k3s-dev/scripts/00-init.sh b/deploy-k3s-dev/scripts/00-init.sh new file mode 100755 index 0000000..0689c07 --- /dev/null +++ b/deploy-k3s-dev/scripts/00-init.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +SECRETS_DIR="${DEPLOY_DIR}/secrets" +CONFIG_FILE="${DEPLOY_DIR}/config.yaml" + +log() { printf '[init] %s\n' "$*"; } +warn() { printf '[init][warn] %s\n' "$*" >&2; } +die() { printf '[init][error] %s\n' "$*" >&2; exit 1; } + +# --- Prerequisites --- + +command -v openssl >/dev/null 2>&1 || die "Missing: openssl" +command -v python3 >/dev/null 2>&1 || die "Missing: python3" + +echo "" +echo "============================================" +echo " honeyDue Dev Server — Initial Setup" +echo "============================================" +echo "" +echo "This script will:" +echo " 1. Generate any missing random secrets" +echo " 2. Ask for anything not already filled in" +echo " 3. Create config.yaml with everything filled in" +echo "" + +mkdir -p "${SECRETS_DIR}" + +# --- Generate random secrets (skip if already exist) --- + +generate_if_missing() { + local file="$1" label="$2" cmd="$3" + if [[ -f "${file}" && -s "${file}" ]]; then + log " ${label} — already exists, keeping" + else + eval "${cmd}" > "${file}" + log " ${label} — generated" + fi +} + +log "Checking secrets..." +generate_if_missing "${SECRETS_DIR}/secret_key.txt" "secrets/secret_key.txt" "openssl rand -base64 48" +generate_if_missing "${SECRETS_DIR}/postgres_password.txt" "secrets/postgres_password.txt" "openssl rand -base64 24" +generate_if_missing "${SECRETS_DIR}/minio_root_password.txt" "secrets/minio_root_password.txt" "openssl rand -base64 24" +generate_if_missing "${SECRETS_DIR}/email_host_password.txt" "secrets/email_host_password.txt" "echo PLACEHOLDER" +log " secrets/fcm_server_key.txt — skipped (Android not ready)" +generate_if_missing "${SECRETS_DIR}/apns_auth_key.p8" "secrets/apns_auth_key.p8" "echo ''" + +REDIS_PW="$(openssl rand -base64 24)" +log " Redis password — generated" + +# --- Collect only what's missing --- + +ask() { + local var_name="$1" prompt="$2" default="${3:-}" + local val + if [[ -n "${default}" ]]; then + read -rp "${prompt} [${default}]: " val + val="${val:-${default}}" + else + read -rp "${prompt}: " val + fi + eval "${var_name}='${val}'" +} + +echo "" +echo "--- Server ---" +ask SERVER_HOST "Server IP or SSH alias" "honeyDueDevUpdate" +[[ -n "${SERVER_HOST}" ]] || die "Server host is required" +ask SERVER_USER "SSH user" "root" +ask SSH_KEY "SSH key path" "~/.ssh/id_ed25519" + +echo "" +echo "--- Container Registry (GHCR) ---" +ask GHCR_USER "GitHub username" "treytartt" +[[ -n "${GHCR_USER}" ]] || die "GitHub username is required" +ask GHCR_TOKEN "GitHub PAT (read:packages, write:packages)" "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" +[[ -n "${GHCR_TOKEN}" ]] || die "GitHub PAT is required" + +echo "" +echo "--- TLS ---" +ask LE_EMAIL "Let's Encrypt email" "treytartt@fastmail.com" + +echo "" +echo "--- Admin Panel ---" +ask ADMIN_USER "Admin basic auth username" "admin" +ADMIN_PW="$(openssl rand -base64 16)" + +# --- Known values from existing Dokku setup --- + +EMAIL_USER="treytartt@fastmail.com" +APNS_KEY_ID="9R5Q7ZX874" +APNS_TEAM_ID="V3PF3M6B6U" + +log "" +log "Pre-filled from existing dev server:" +log " Email user: ${EMAIL_USER}" +log " APNS Key ID: ${APNS_KEY_ID}" +log " APNS Team ID: ${APNS_TEAM_ID}" + +# --- Generate config.yaml --- + +log "Generating config.yaml..." + +cat > "${CONFIG_FILE}" <" + use_tls: true + +# --- Push Notifications --- +push: + apns_key_id: "${APNS_KEY_ID}" + apns_team_id: "${APNS_TEAM_ID}" + apns_topic: com.tt.honeyDue + apns_production: false + apns_use_sandbox: true + +# --- Object Storage (in-cluster MinIO) --- +storage: + minio_root_user: honeydue + bucket: honeydue-dev + max_file_size: 10485760 + allowed_types: "image/jpeg,image/png,image/gif,image/webp,application/pdf" + +# --- Worker Schedules (UTC hours) --- +worker: + task_reminder_hour: 14 + overdue_reminder_hour: 15 + daily_digest_hour: 3 + +# --- Feature Flags --- +features: + push_enabled: true + email_enabled: false + webhooks_enabled: false + onboarding_emails_enabled: false + pdf_reports_enabled: true + worker_enabled: true + +# --- Redis --- +redis: + password: "${REDIS_PW}" + +# --- Admin Panel --- +admin: + basic_auth_user: "${ADMIN_USER}" + basic_auth_password: "${ADMIN_PW}" + +# --- TLS --- +tls: + mode: letsencrypt + letsencrypt_email: "${LE_EMAIL}" + +# --- Apple Auth / IAP --- +apple_auth: + client_id: "com.tt.honeyDue" + team_id: "${APNS_TEAM_ID}" + iap_key_id: "" + iap_issuer_id: "" + iap_bundle_id: "" + iap_key_path: "" + iap_sandbox: true + +# --- Google Auth / IAP --- +google_auth: + client_id: "" + android_client_id: "" + ios_client_id: "" + iap_package_name: "" + iap_service_account_path: "" +YAML + +# --- Summary --- + +echo "" +echo "============================================" +echo " Setup Complete" +echo "============================================" +echo "" +echo "Generated:" +echo " config.yaml" +echo " secrets/secret_key.txt" +echo " secrets/postgres_password.txt" +echo " secrets/minio_root_password.txt" +echo " secrets/email_host_password.txt" +echo " secrets/fcm_server_key.txt" +echo " secrets/apns_auth_key.p8" +echo "" +echo "Admin panel credentials:" +echo " Username: ${ADMIN_USER}" +echo " Password: ${ADMIN_PW}" +echo " (save these — they won't be shown again)" +echo "" +echo "Next steps:" +echo " ./scripts/01-setup-k3s.sh" +echo " ./scripts/02-setup-secrets.sh" +echo " ./scripts/03-deploy.sh" +echo " ./scripts/04-verify.sh" +echo "" diff --git a/deploy-k3s-dev/scripts/01-setup-k3s.sh b/deploy-k3s-dev/scripts/01-setup-k3s.sh new file mode 100755 index 0000000..2c0ebc1 --- /dev/null +++ b/deploy-k3s-dev/scripts/01-setup-k3s.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +log() { printf '[setup] %s\n' "$*"; } +die() { printf '[setup][error] %s\n' "$*" >&2; exit 1; } + +# --- Local prerequisites --- + +command -v kubectl >/dev/null 2>&1 || die "Missing locally: kubectl (https://kubernetes.io/docs/tasks/tools/)" + +# --- Server connection --- + +SERVER_HOST="$(cfg_require server.host "Server IP or SSH alias")" +SERVER_USER="$(cfg server.user)" +SERVER_USER="${SERVER_USER:-root}" +SSH_KEY="$(cfg server.ssh_key | sed "s|~|${HOME}|g")" + +SSH_OPTS=() +if [[ -n "${SSH_KEY}" && -f "${SSH_KEY}" ]]; then + SSH_OPTS+=(-i "${SSH_KEY}") +fi +SSH_OPTS+=(-o StrictHostKeyChecking=accept-new) + +ssh_cmd() { + ssh "${SSH_OPTS[@]}" "${SERVER_USER}@${SERVER_HOST}" "$@" +} + +log "Testing SSH connection to ${SERVER_USER}@${SERVER_HOST}..." +ssh_cmd "echo 'SSH connection OK'" || die "Cannot SSH into ${SERVER_HOST}" + +# --- Server prerequisites --- + +log "Setting up server prerequisites..." + +ssh_cmd 'bash -s' <<'REMOTE_SETUP' +set -euo pipefail + +log() { printf '[setup][remote] %s\n' "$*"; } + +# --- System updates --- +log "Updating system packages..." +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get upgrade -y -qq + +# --- SSH hardening --- +log "Hardening SSH..." +sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config +sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config +systemctl reload sshd 2>/dev/null || systemctl reload ssh 2>/dev/null || true + +# --- fail2ban --- +if ! command -v fail2ban-client >/dev/null 2>&1; then + log "Installing fail2ban..." + apt-get install -y -qq fail2ban + systemctl enable --now fail2ban +else + log "fail2ban already installed" +fi + +# --- Unattended security upgrades --- +if ! dpkg -l | grep -q unattended-upgrades; then + log "Installing unattended-upgrades..." + apt-get install -y -qq unattended-upgrades + dpkg-reconfigure -plow unattended-upgrades +else + log "unattended-upgrades already installed" +fi + +# --- Firewall (ufw) --- +if command -v ufw >/dev/null 2>&1; then + log "Configuring firewall..." + ufw default deny incoming + ufw default allow outgoing + ufw allow 22/tcp # SSH + ufw allow 443/tcp # HTTPS (Traefik) + ufw allow 6443/tcp # K3s API + ufw allow 80/tcp # HTTP (Let's Encrypt ACME challenge) + ufw --force enable +else + log "Installing ufw..." + apt-get install -y -qq ufw + ufw default deny incoming + ufw default allow outgoing + ufw allow 22/tcp + ufw allow 443/tcp + ufw allow 6443/tcp + ufw allow 80/tcp + ufw --force enable +fi + +log "Server prerequisites complete." +REMOTE_SETUP + +# --- Install K3s --- + +log "Installing K3s on ${SERVER_HOST}..." +log " This takes about 1-2 minutes." + +ssh_cmd "curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='server --secrets-encryption' sh -" + +# --- Wait for K3s to be ready --- + +log "Waiting for K3s to be ready..." +ssh_cmd "until kubectl get nodes >/dev/null 2>&1; do sleep 2; done" + +# --- Copy kubeconfig --- + +KUBECONFIG_PATH="${DEPLOY_DIR}/kubeconfig" + +log "Copying kubeconfig..." +ssh_cmd "sudo cat /etc/rancher/k3s/k3s.yaml" > "${KUBECONFIG_PATH}" + +# Replace 127.0.0.1 with the server's actual IP/hostname +# If SERVER_HOST is an SSH alias, resolve the actual IP +ACTUAL_HOST="${SERVER_HOST}" +if ! echo "${SERVER_HOST}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then + # Try to resolve from SSH config + RESOLVED="$(ssh -G "${SERVER_HOST}" 2>/dev/null | awk '/^hostname / {print $2}')" + if [[ -n "${RESOLVED}" && "${RESOLVED}" != "${SERVER_HOST}" ]]; then + ACTUAL_HOST="${RESOLVED}" + fi +fi + +sed -i.bak "s|https://127.0.0.1:6443|https://${ACTUAL_HOST}:6443|g" "${KUBECONFIG_PATH}" +rm -f "${KUBECONFIG_PATH}.bak" +chmod 600 "${KUBECONFIG_PATH}" + +# --- Verify --- + +export KUBECONFIG="${KUBECONFIG_PATH}" +log "Verifying cluster..." +kubectl get nodes + +log "" +log "K3s installed successfully on ${SERVER_HOST}." +log "Server hardened: SSH key-only, fail2ban, ufw firewall, unattended-upgrades." +log "" +log "Next steps:" +log " export KUBECONFIG=${KUBECONFIG_PATH}" +log " kubectl get nodes" +log " ./scripts/02-setup-secrets.sh" diff --git a/deploy-k3s-dev/scripts/02-setup-secrets.sh b/deploy-k3s-dev/scripts/02-setup-secrets.sh new file mode 100755 index 0000000..191da44 --- /dev/null +++ b/deploy-k3s-dev/scripts/02-setup-secrets.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +SECRETS_DIR="${DEPLOY_DIR}/secrets" +NAMESPACE="honeydue" + +log() { printf '[secrets] %s\n' "$*"; } +warn() { printf '[secrets][warn] %s\n' "$*" >&2; } +die() { printf '[secrets][error] %s\n' "$*" >&2; exit 1; } + +# --- Prerequisites --- + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" + +kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 || { + log "Creating namespace ${NAMESPACE}..." + kubectl apply -f "${DEPLOY_DIR}/manifests/namespace.yaml" +} + +# --- Validate secret files --- + +require_file() { + local path="$1" label="$2" + [[ -f "${path}" ]] || die "Missing: ${path} (${label})" + [[ -s "${path}" ]] || die "Empty: ${path} (${label})" +} + +require_file "${SECRETS_DIR}/postgres_password.txt" "Postgres password" +require_file "${SECRETS_DIR}/secret_key.txt" "SECRET_KEY" +require_file "${SECRETS_DIR}/email_host_password.txt" "SMTP password" +# FCM server key is optional (Android not yet ready) +if [[ -f "${SECRETS_DIR}/fcm_server_key.txt" && -s "${SECRETS_DIR}/fcm_server_key.txt" ]]; then + FCM_CONTENT="$(tr -d '\r\n' < "${SECRETS_DIR}/fcm_server_key.txt")" + if [[ "${FCM_CONTENT}" == "PLACEHOLDER" ]]; then + warn "fcm_server_key.txt is a placeholder — FCM push disabled" + FCM_CONTENT="" + fi +else + warn "fcm_server_key.txt not found — FCM push disabled" + FCM_CONTENT="" +fi +require_file "${SECRETS_DIR}/apns_auth_key.p8" "APNS private key" +require_file "${SECRETS_DIR}/minio_root_password.txt" "MinIO root password" + +# Validate APNS key format +if ! grep -q "BEGIN PRIVATE KEY" "${SECRETS_DIR}/apns_auth_key.p8"; then + die "APNS key file does not look like a private key: ${SECRETS_DIR}/apns_auth_key.p8" +fi + +# Validate secret_key length (minimum 32 chars) +SECRET_KEY_LEN="$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt" | wc -c | tr -d ' ')" +if (( SECRET_KEY_LEN < 32 )); then + die "secret_key.txt must be at least 32 characters (got ${SECRET_KEY_LEN})." +fi + +# Validate MinIO password length (minimum 8 chars) +MINIO_PW_LEN="$(tr -d '\r\n' < "${SECRETS_DIR}/minio_root_password.txt" | wc -c | tr -d ' ')" +if (( MINIO_PW_LEN < 8 )); then + die "minio_root_password.txt must be at least 8 characters (got ${MINIO_PW_LEN})." +fi + +# --- Read optional config values --- + +REDIS_PASSWORD="$(cfg redis.password 2>/dev/null || true)" +ADMIN_AUTH_USER="$(cfg admin.basic_auth_user 2>/dev/null || true)" +ADMIN_AUTH_PASSWORD="$(cfg admin.basic_auth_password 2>/dev/null || true)" +TLS_MODE="$(cfg tls.mode 2>/dev/null || echo "letsencrypt")" + +# --- Create app secrets --- + +log "Creating honeydue-secrets..." +SECRET_ARGS=( + --namespace="${NAMESPACE}" + --from-literal="POSTGRES_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/postgres_password.txt")" + --from-literal="SECRET_KEY=$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt")" + --from-literal="EMAIL_HOST_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/email_host_password.txt")" + --from-literal="FCM_SERVER_KEY=${FCM_CONTENT}" + --from-literal="MINIO_ROOT_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/minio_root_password.txt")" +) + +if [[ -n "${REDIS_PASSWORD}" ]]; then + log " Including REDIS_PASSWORD in secrets" + SECRET_ARGS+=(--from-literal="REDIS_PASSWORD=${REDIS_PASSWORD}") +fi + +kubectl create secret generic honeydue-secrets \ + "${SECRET_ARGS[@]}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Create APNS key secret --- + +log "Creating honeydue-apns-key..." +kubectl create secret generic honeydue-apns-key \ + --namespace="${NAMESPACE}" \ + --from-file="apns_auth_key.p8=${SECRETS_DIR}/apns_auth_key.p8" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Create GHCR registry credentials --- + +REGISTRY_SERVER="$(cfg registry.server)" +REGISTRY_USER="$(cfg registry.username)" +REGISTRY_TOKEN="$(cfg registry.token)" + +if [[ -n "${REGISTRY_SERVER}" && -n "${REGISTRY_USER}" && -n "${REGISTRY_TOKEN}" ]]; then + log "Creating ghcr-credentials..." + kubectl create secret docker-registry ghcr-credentials \ + --namespace="${NAMESPACE}" \ + --docker-server="${REGISTRY_SERVER}" \ + --docker-username="${REGISTRY_USER}" \ + --docker-password="${REGISTRY_TOKEN}" \ + --dry-run=client -o yaml | kubectl apply -f - +else + warn "Registry credentials incomplete in config.yaml — skipping ghcr-credentials." +fi + +# --- Create Cloudflare origin cert (only if cloudflare mode) --- + +if [[ "${TLS_MODE}" == "cloudflare" ]]; then + require_file "${SECRETS_DIR}/cloudflare-origin.crt" "Cloudflare origin cert" + require_file "${SECRETS_DIR}/cloudflare-origin.key" "Cloudflare origin key" + + log "Creating cloudflare-origin-cert..." + kubectl create secret tls cloudflare-origin-cert \ + --namespace="${NAMESPACE}" \ + --cert="${SECRETS_DIR}/cloudflare-origin.crt" \ + --key="${SECRETS_DIR}/cloudflare-origin.key" \ + --dry-run=client -o yaml | kubectl apply -f - +fi + +# --- Create admin basic auth secret --- + +if [[ -n "${ADMIN_AUTH_USER}" && -n "${ADMIN_AUTH_PASSWORD}" ]]; then + command -v htpasswd >/dev/null 2>&1 || die "Missing: htpasswd (install apache2-utils)" + log "Creating admin-basic-auth secret..." + HTPASSWD="$(htpasswd -nb "${ADMIN_AUTH_USER}" "${ADMIN_AUTH_PASSWORD}")" + kubectl create secret generic admin-basic-auth \ + --namespace="${NAMESPACE}" \ + --from-literal=users="${HTPASSWD}" \ + --dry-run=client -o yaml | kubectl apply -f - +else + warn "admin.basic_auth_user/password not set in config.yaml — skipping admin-basic-auth." + warn "Admin panel will NOT have basic auth protection." +fi + +# --- Done --- + +log "" +log "All secrets created in namespace '${NAMESPACE}'." +log "Verify: kubectl get secrets -n ${NAMESPACE}" diff --git a/deploy-k3s-dev/scripts/03-deploy.sh b/deploy-k3s-dev/scripts/03-deploy.sh new file mode 100755 index 0000000..32bb516 --- /dev/null +++ b/deploy-k3s-dev/scripts/03-deploy.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)" +NAMESPACE="honeydue" +MANIFESTS="${DEPLOY_DIR}/manifests" + +log() { printf '[deploy] %s\n' "$*"; } +warn() { printf '[deploy][warn] %s\n' "$*" >&2; } +die() { printf '[deploy][error] %s\n' "$*" >&2; exit 1; } + +# --- Parse arguments --- + +SKIP_BUILD=false +DEPLOY_TAG="" + +while (( $# > 0 )); do + case "$1" in + --skip-build) SKIP_BUILD=true; shift ;; + --tag) + [[ -n "${2:-}" ]] || die "--tag requires a value" + DEPLOY_TAG="$2"; shift 2 ;; + -h|--help) + cat <<'EOF' +Usage: ./scripts/03-deploy.sh [OPTIONS] + +Options: + --skip-build Skip Docker build/push, use existing images + --tag Image tag (default: git short SHA) + -h, --help Show this help +EOF + exit 0 ;; + *) die "Unknown argument: $1" ;; + esac +done + +# --- Prerequisites --- + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" +command -v docker >/dev/null 2>&1 || die "Missing: docker" + +if [[ -z "${DEPLOY_TAG}" ]]; then + DEPLOY_TAG="$(git -C "${REPO_DIR}" rev-parse --short HEAD 2>/dev/null || echo "latest")" +fi + +# --- Read config --- + +REGISTRY_SERVER="$(cfg_require registry.server "Container registry server")" +REGISTRY_NS="$(cfg_require registry.namespace "Registry namespace")" +REGISTRY_USER="$(cfg_require registry.username "Registry username")" +REGISTRY_TOKEN="$(cfg_require registry.token "Registry token")" +TLS_MODE="$(cfg tls.mode 2>/dev/null || echo "letsencrypt")" +API_DOMAIN="$(cfg_require domains.api "API domain")" +ADMIN_DOMAIN="$(cfg_require domains.admin "Admin domain")" + +REGISTRY_PREFIX="${REGISTRY_SERVER%/}/${REGISTRY_NS#/}" +API_IMAGE="${REGISTRY_PREFIX}/honeydue-api:${DEPLOY_TAG}" +WORKER_IMAGE="${REGISTRY_PREFIX}/honeydue-worker:${DEPLOY_TAG}" +ADMIN_IMAGE="${REGISTRY_PREFIX}/honeydue-admin:${DEPLOY_TAG}" + +# --- Build and push --- + +if [[ "${SKIP_BUILD}" == "false" ]]; then + log "Logging in to ${REGISTRY_SERVER}..." + printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_SERVER}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + + log "Building API image: ${API_IMAGE}" + docker build --target api -t "${API_IMAGE}" "${REPO_DIR}" + + log "Building Worker image: ${WORKER_IMAGE}" + docker build --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}" + + log "Building Admin image: ${ADMIN_IMAGE}" + docker build --target admin -t "${ADMIN_IMAGE}" "${REPO_DIR}" + + log "Pushing images..." + docker push "${API_IMAGE}" + docker push "${WORKER_IMAGE}" + docker push "${ADMIN_IMAGE}" + + # Also tag and push :latest + docker tag "${API_IMAGE}" "${REGISTRY_PREFIX}/honeydue-api:latest" + docker tag "${WORKER_IMAGE}" "${REGISTRY_PREFIX}/honeydue-worker:latest" + docker tag "${ADMIN_IMAGE}" "${REGISTRY_PREFIX}/honeydue-admin:latest" + docker push "${REGISTRY_PREFIX}/honeydue-api:latest" + docker push "${REGISTRY_PREFIX}/honeydue-worker:latest" + docker push "${REGISTRY_PREFIX}/honeydue-admin:latest" +else + warn "Skipping build. Using images for tag: ${DEPLOY_TAG}" +fi + +# --- Generate and apply ConfigMap from config.yaml --- + +log "Generating env from config.yaml..." +ENV_FILE="$(mktemp)" +trap 'rm -f "${ENV_FILE}"' EXIT +generate_env > "${ENV_FILE}" + +log "Creating ConfigMap..." +kubectl create configmap honeydue-config \ + --namespace="${NAMESPACE}" \ + --from-env-file="${ENV_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Configure TLS --- + +if [[ "${TLS_MODE}" == "letsencrypt" ]]; then + LE_EMAIL="$(cfg_require tls.letsencrypt_email "Let's Encrypt email")" + + log "Configuring Traefik with Let's Encrypt (${LE_EMAIL})..." + sed "s|LETSENCRYPT_EMAIL_PLACEHOLDER|${LE_EMAIL}|" \ + "${MANIFESTS}/traefik/helmchartconfig.yaml" | kubectl apply -f - + + TLS_SECRET="letsencrypt-cert" + TLS_ANNOTATION="traefik.ingress.kubernetes.io/router.tls.certresolver: letsencrypt" +elif [[ "${TLS_MODE}" == "cloudflare" ]]; then + log "Using Cloudflare origin cert for TLS..." + TLS_SECRET="cloudflare-origin-cert" + TLS_ANNOTATION="" +else + die "Unknown tls.mode: ${TLS_MODE} (expected: letsencrypt or cloudflare)" +fi + +# --- Apply manifests --- + +log "Applying manifests..." + +kubectl apply -f "${MANIFESTS}/namespace.yaml" +kubectl apply -f "${MANIFESTS}/rbac.yaml" +kubectl apply -f "${MANIFESTS}/postgres/" +kubectl apply -f "${MANIFESTS}/redis/" +kubectl apply -f "${MANIFESTS}/minio/deployment.yaml" +kubectl apply -f "${MANIFESTS}/minio/pvc.yaml" +kubectl apply -f "${MANIFESTS}/minio/service.yaml" +kubectl apply -f "${MANIFESTS}/ingress/middleware.yaml" + +# Apply ingress with domain and TLS substitution +sed -e "s|API_DOMAIN_PLACEHOLDER|${API_DOMAIN}|g" \ + -e "s|ADMIN_DOMAIN_PLACEHOLDER|${ADMIN_DOMAIN}|g" \ + -e "s|TLS_SECRET_PLACEHOLDER|${TLS_SECRET}|g" \ + -e "s|# TLS_ANNOTATIONS_PLACEHOLDER|${TLS_ANNOTATION}|g" \ + "${MANIFESTS}/ingress/ingress.yaml" | kubectl apply -f - + +# Apply app deployments with image substitution +sed "s|image: IMAGE_PLACEHOLDER|image: ${API_IMAGE}|" "${MANIFESTS}/api/deployment.yaml" | kubectl apply -f - +kubectl apply -f "${MANIFESTS}/api/service.yaml" + +sed "s|image: IMAGE_PLACEHOLDER|image: ${WORKER_IMAGE}|" "${MANIFESTS}/worker/deployment.yaml" | kubectl apply -f - + +sed "s|image: IMAGE_PLACEHOLDER|image: ${ADMIN_IMAGE}|" "${MANIFESTS}/admin/deployment.yaml" | kubectl apply -f - +kubectl apply -f "${MANIFESTS}/admin/service.yaml" + +# Apply network policies +kubectl apply -f "${MANIFESTS}/network-policies.yaml" + +# --- Wait for infrastructure rollouts --- + +log "Waiting for infrastructure rollouts..." +kubectl rollout status deployment/postgres -n "${NAMESPACE}" --timeout=120s +kubectl rollout status deployment/redis -n "${NAMESPACE}" --timeout=120s +kubectl rollout status deployment/minio -n "${NAMESPACE}" --timeout=120s + +# --- Create MinIO bucket --- + +log "Creating MinIO bucket..." +# Delete previous job run if it exists (jobs are immutable) +kubectl delete job minio-create-bucket -n "${NAMESPACE}" 2>/dev/null || true +kubectl apply -f "${MANIFESTS}/minio/create-bucket-job.yaml" +kubectl wait --for=condition=complete job/minio-create-bucket -n "${NAMESPACE}" --timeout=120s + +# --- Wait for app rollouts --- + +log "Waiting for app rollouts..." +kubectl rollout status deployment/api -n "${NAMESPACE}" --timeout=300s +kubectl rollout status deployment/worker -n "${NAMESPACE}" --timeout=300s +kubectl rollout status deployment/admin -n "${NAMESPACE}" --timeout=300s + +# --- Done --- + +log "" +log "Deploy completed successfully." +log "Tag: ${DEPLOY_TAG}" +log "TLS: ${TLS_MODE}" +log "Images:" +log " API: ${API_IMAGE}" +log " Worker: ${WORKER_IMAGE}" +log " Admin: ${ADMIN_IMAGE}" +log "" +log "Run ./scripts/04-verify.sh to check cluster health." diff --git a/deploy-k3s-dev/scripts/04-verify.sh b/deploy-k3s-dev/scripts/04-verify.sh new file mode 100755 index 0000000..3de59d8 --- /dev/null +++ b/deploy-k3s-dev/scripts/04-verify.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="honeydue" + +log() { printf '[verify] %s\n' "$*"; } +sep() { printf '\n%s\n' "--- $1 ---"; } +ok() { printf '[verify] ✓ %s\n' "$*"; } +fail() { printf '[verify] ✗ %s\n' "$*"; } + +command -v kubectl >/dev/null 2>&1 || { echo "Missing: kubectl" >&2; exit 1; } + +sep "Node" +kubectl get nodes -o wide + +sep "Pods" +kubectl get pods -n "${NAMESPACE}" -o wide + +sep "Services" +kubectl get svc -n "${NAMESPACE}" + +sep "Ingress" +kubectl get ingress -n "${NAMESPACE}" + +sep "PVCs" +kubectl get pvc -n "${NAMESPACE}" + +sep "Secrets (names only)" +kubectl get secrets -n "${NAMESPACE}" + +sep "ConfigMap keys" +kubectl get configmap honeydue-config -n "${NAMESPACE}" -o jsonpath='{.data}' 2>/dev/null | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + for k in sorted(d.keys()): + v = d[k] + if any(s in k.upper() for s in ['PASSWORD', 'SECRET', 'TOKEN', 'KEY']): + v = '***REDACTED***' + print(f' {k}={v}') +except: + print(' (could not parse)') +" 2>/dev/null || log "ConfigMap not found or not parseable" + +sep "Warning Events (last 15 min)" +kubectl get events -n "${NAMESPACE}" --field-selector type=Warning --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || log "No warning events" + +sep "Pod Restart Counts" +kubectl get pods -n "${NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' 2>/dev/null || true + +# ============================================================================= +# Infrastructure Health +# ============================================================================= + +sep "PostgreSQL Health" +PG_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -n "${PG_POD}" ]]; then + kubectl exec -n "${NAMESPACE}" "${PG_POD}" -- pg_isready -U honeydue 2>/dev/null && ok "PostgreSQL is ready" || fail "PostgreSQL is NOT ready" +else + fail "No PostgreSQL pod found" +fi + +sep "Redis Health" +REDIS_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -n "${REDIS_POD}" ]]; then + kubectl exec -n "${NAMESPACE}" "${REDIS_POD}" -- sh -c 'if [ -n "$REDIS_PASSWORD" ]; then redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null; else redis-cli ping; fi' 2>/dev/null | grep -q PONG && ok "Redis is ready" || fail "Redis is NOT ready" +else + fail "No Redis pod found" +fi + +sep "MinIO Health" +MINIO_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -n "${MINIO_POD}" ]]; then + kubectl exec -n "${NAMESPACE}" "${MINIO_POD}" -- curl -sf http://localhost:9000/minio/health/ready 2>/dev/null && ok "MinIO is ready" || fail "MinIO is NOT ready" +else + fail "No MinIO pod found" +fi + +sep "API Health Check" +API_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -n "${API_POD}" ]]; then + kubectl exec -n "${NAMESPACE}" "${API_POD}" -- curl -sf http://localhost:8000/api/health/ 2>/dev/null && ok "API health check passed" || fail "API health check FAILED" +else + fail "No API pod found" +fi + +sep "Resource Usage" +kubectl top pods -n "${NAMESPACE}" 2>/dev/null || log "Metrics server not available (install with: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml)" + +# ============================================================================= +# Security Verification +# ============================================================================= + +sep "Security: Network Policies" +NP_COUNT="$(kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')" +if (( NP_COUNT >= 5 )); then + ok "Found ${NP_COUNT} network policies" + kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do + echo " ${line}" + done +else + fail "Expected 5+ network policies, found ${NP_COUNT}" +fi + +sep "Security: Service Accounts" +SA_COUNT="$(kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | grep -cv default | tr -d ' ')" +if (( SA_COUNT >= 6 )); then + ok "Found ${SA_COUNT} custom service accounts (api, worker, admin, redis, postgres, minio)" +else + fail "Expected 6 custom service accounts, found ${SA_COUNT}" +fi +kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do + echo " ${line}" +done + +sep "Security: Pod Security Contexts" +PODS_WITHOUT_SECURITY="$(kubectl get pods -n "${NAMESPACE}" -o json 2>/dev/null | python3 -c " +import json, sys +try: + data = json.load(sys.stdin) + issues = [] + for pod in data.get('items', []): + name = pod['metadata']['name'] + spec = pod['spec'] + sc = spec.get('securityContext', {}) + # Postgres is exempt from runAsNonRoot (entrypoint requirement) + is_postgres = any('postgres' in c.get('image', '') for c in spec.get('containers', [])) + if not sc.get('runAsNonRoot') and not is_postgres: + issues.append(f'{name}: missing runAsNonRoot') + for c in spec.get('containers', []): + csc = c.get('securityContext', {}) + if csc.get('allowPrivilegeEscalation', True): + issues.append(f'{name}/{c[\"name\"]}: allowPrivilegeEscalation not false') + if issues: + for i in issues: + print(i) + else: + print('OK') +except Exception as e: + print(f'Error: {e}') +" 2>/dev/null || echo "Error parsing pod specs")" + +if [[ "${PODS_WITHOUT_SECURITY}" == "OK" ]]; then + ok "All pods have proper security contexts" +else + fail "Pod security context issues:" + echo "${PODS_WITHOUT_SECURITY}" | while read -r line; do + echo " ${line}" + done +fi + +sep "Security: Admin Basic Auth" +ADMIN_AUTH="$(kubectl get secret admin-basic-auth -n "${NAMESPACE}" -o name 2>/dev/null || true)" +if [[ -n "${ADMIN_AUTH}" ]]; then + ok "admin-basic-auth secret exists" +else + fail "admin-basic-auth secret not found — admin panel has no additional auth layer" +fi + +echo "" +log "Verification complete." diff --git a/deploy-k3s-dev/scripts/_config.sh b/deploy-k3s-dev/scripts/_config.sh new file mode 100755 index 0000000..e9583c0 --- /dev/null +++ b/deploy-k3s-dev/scripts/_config.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Shared config helper — sourced by all deploy scripts. +# Provides cfg() to read values from config.yaml. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_FILE="${DEPLOY_DIR}/config.yaml" + +if [[ ! -f "${CONFIG_FILE}" ]]; then + if [[ -f "${CONFIG_FILE}.example" ]]; then + echo "[error] config.yaml not found. Run: cp config.yaml.example config.yaml" >&2 + else + echo "[error] config.yaml not found." >&2 + fi + exit 1 +fi + +# cfg "dotted.key.path" — reads a value from config.yaml +cfg() { + python3 -c " +import yaml, json, sys +with open(sys.argv[1]) as f: + c = yaml.safe_load(f) +keys = sys.argv[2].split('.') +v = c +for k in keys: + if isinstance(v, list): + v = v[int(k)] + else: + v = v[k] +if isinstance(v, bool): + print(str(v).lower()) +elif isinstance(v, (dict, list)): + print(json.dumps(v)) +else: + print('' if v is None else v) +" "${CONFIG_FILE}" "$1" 2>/dev/null +} + +# cfg_require "key" "label" — reads value and dies if empty +cfg_require() { + local val + val="$(cfg "$1")" + if [[ -z "${val}" ]]; then + echo "[error] Missing required config: $1 ($2)" >&2 + exit 1 + fi + printf '%s' "${val}" +} + +# generate_env — writes the flat env file the app expects to stdout +# Points DB at in-cluster PostgreSQL, storage at in-cluster MinIO +generate_env() { + python3 -c " +import yaml + +with open('${CONFIG_FILE}') as f: + c = yaml.safe_load(f) + +d = c['domains'] +db = c['database'] +em = c['email'] +ps = c['push'] +st = c['storage'] +wk = c['worker'] +ft = c['features'] +aa = c.get('apple_auth', {}) +ga = c.get('google_auth', {}) +rd = c.get('redis', {}) + +def b(v): + return str(v).lower() if isinstance(v, bool) else str(v) + +def val(v): + return '' if v is None else str(v) + +lines = [ + # API + 'DEBUG=true', + f\"ALLOWED_HOSTS={d['api']},{d['base']},localhost\", + f\"CORS_ALLOWED_ORIGINS=https://{d['base']},https://{d['admin']}\", + 'TIMEZONE=UTC', + f\"BASE_URL=https://{d['base']}\", + 'PORT=8000', + # Admin + f\"NEXT_PUBLIC_API_URL=https://{d['api']}\", + f\"ADMIN_PANEL_URL=https://{d['admin']}\", + # Database (in-cluster PostgreSQL) + 'DB_HOST=postgres.honeydue.svc.cluster.local', + 'DB_PORT=5432', + f\"POSTGRES_USER={val(db['user'])}\", + f\"POSTGRES_DB={db['name']}\", + 'DB_SSLMODE=disable', + f\"DB_MAX_OPEN_CONNS={db['max_open_conns']}\", + f\"DB_MAX_IDLE_CONNS={db['max_idle_conns']}\", + f\"DB_MAX_LIFETIME={db['max_lifetime']}\", + # Redis (in-cluster) + f\"REDIS_URL=redis://{':%s@' % val(rd.get('password')) if rd.get('password') else ''}redis.honeydue.svc.cluster.local:6379/0\", + 'REDIS_DB=0', + # Email + f\"EMAIL_HOST={em['host']}\", + f\"EMAIL_PORT={em['port']}\", + f\"EMAIL_USE_TLS={b(em['use_tls'])}\", + f\"EMAIL_HOST_USER={val(em['user'])}\", + f\"DEFAULT_FROM_EMAIL={val(em['from'])}\", + # Push + 'APNS_AUTH_KEY_PATH=/secrets/apns/apns_auth_key.p8', + f\"APNS_AUTH_KEY_ID={val(ps['apns_key_id'])}\", + f\"APNS_TEAM_ID={val(ps['apns_team_id'])}\", + f\"APNS_TOPIC={ps['apns_topic']}\", + f\"APNS_USE_SANDBOX={b(ps['apns_use_sandbox'])}\", + f\"APNS_PRODUCTION={b(ps['apns_production'])}\", + # Worker + f\"TASK_REMINDER_HOUR={wk['task_reminder_hour']}\", + f\"OVERDUE_REMINDER_HOUR={wk['overdue_reminder_hour']}\", + f\"DAILY_DIGEST_HOUR={wk['daily_digest_hour']}\", + # Storage (in-cluster MinIO — S3-compatible, same env vars as B2) + f\"B2_KEY_ID={val(st['minio_root_user'])}\", + # B2_APP_KEY injected from secret (MINIO_ROOT_PASSWORD) + f\"B2_BUCKET_NAME={val(st['bucket'])}\", + 'B2_ENDPOINT=minio.honeydue.svc.cluster.local:9000', + 'STORAGE_USE_SSL=false', + f\"STORAGE_MAX_FILE_SIZE={st['max_file_size']}\", + f\"STORAGE_ALLOWED_TYPES={st['allowed_types']}\", + # MinIO root user (for MinIO deployment + bucket init job) + f\"MINIO_ROOT_USER={val(st['minio_root_user'])}\", + # Features + f\"FEATURE_PUSH_ENABLED={b(ft['push_enabled'])}\", + f\"FEATURE_EMAIL_ENABLED={b(ft['email_enabled'])}\", + f\"FEATURE_WEBHOOKS_ENABLED={b(ft['webhooks_enabled'])}\", + f\"FEATURE_ONBOARDING_EMAILS_ENABLED={b(ft['onboarding_emails_enabled'])}\", + f\"FEATURE_PDF_REPORTS_ENABLED={b(ft['pdf_reports_enabled'])}\", + f\"FEATURE_WORKER_ENABLED={b(ft['worker_enabled'])}\", + # Apple auth/IAP + f\"APPLE_CLIENT_ID={val(aa.get('client_id'))}\", + f\"APPLE_TEAM_ID={val(aa.get('team_id'))}\", + f\"APPLE_IAP_KEY_ID={val(aa.get('iap_key_id'))}\", + f\"APPLE_IAP_ISSUER_ID={val(aa.get('iap_issuer_id'))}\", + f\"APPLE_IAP_BUNDLE_ID={val(aa.get('iap_bundle_id'))}\", + f\"APPLE_IAP_KEY_PATH={val(aa.get('iap_key_path'))}\", + f\"APPLE_IAP_SANDBOX={b(aa.get('iap_sandbox', True))}\", + # Google auth/IAP + f\"GOOGLE_CLIENT_ID={val(ga.get('client_id'))}\", + f\"GOOGLE_ANDROID_CLIENT_ID={val(ga.get('android_client_id'))}\", + f\"GOOGLE_IOS_CLIENT_ID={val(ga.get('ios_client_id'))}\", + f\"GOOGLE_IAP_PACKAGE_NAME={val(ga.get('iap_package_name'))}\", + f\"GOOGLE_IAP_SERVICE_ACCOUNT_PATH={val(ga.get('iap_service_account_path'))}\", +] + +print('\n'.join(lines)) +" +} diff --git a/deploy-k3s-dev/scripts/rollback.sh b/deploy-k3s-dev/scripts/rollback.sh new file mode 100755 index 0000000..4d3f7ab --- /dev/null +++ b/deploy-k3s-dev/scripts/rollback.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="honeydue" + +log() { printf '[rollback] %s\n' "$*"; } +die() { printf '[rollback][error] %s\n' "$*" >&2; exit 1; } + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" + +DEPLOYMENTS=("api" "worker" "admin") + +# --- Show current state --- + +echo "=== Current Rollout History ===" +for deploy in "${DEPLOYMENTS[@]}"; do + echo "" + echo "--- ${deploy} ---" + kubectl rollout history deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || echo " (not found)" +done + +echo "" +echo "=== Current Images ===" +for deploy in "${DEPLOYMENTS[@]}"; do + IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")" + echo " ${deploy}: ${IMAGE}" +done + +# --- Confirm --- + +echo "" +read -rp "Roll back all deployments to previous revision? [y/N] " confirm +if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then + log "Aborted." + exit 0 +fi + +# --- Rollback --- + +for deploy in "${DEPLOYMENTS[@]}"; do + log "Rolling back ${deploy}..." + kubectl rollout undo deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || log "Skipping ${deploy} (not found or no previous revision)" +done + +# --- Wait --- + +log "Waiting for rollouts..." +for deploy in "${DEPLOYMENTS[@]}"; do + kubectl rollout status deployment/"${deploy}" -n "${NAMESPACE}" --timeout=300s 2>/dev/null || true +done + +# --- Verify --- + +echo "" +echo "=== Post-Rollback Images ===" +for deploy in "${DEPLOYMENTS[@]}"; do + IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")" + echo " ${deploy}: ${IMAGE}" +done + +log "Rollback complete. Run ./scripts/04-verify.sh to check health." diff --git a/deploy-k3s-dev/secrets/README.md b/deploy-k3s-dev/secrets/README.md new file mode 100644 index 0000000..c19757d --- /dev/null +++ b/deploy-k3s-dev/secrets/README.md @@ -0,0 +1,22 @@ +# Secrets Directory + +Create these files before running `scripts/02-setup-secrets.sh`: + +| File | Purpose | +|------|---------| +| `postgres_password.txt` | In-cluster PostgreSQL password | +| `secret_key.txt` | App signing secret (minimum 32 characters) | +| `email_host_password.txt` | SMTP password (Fastmail app password) | +| `fcm_server_key.txt` | Firebase Cloud Messaging server key (optional — Android not yet ready) | +| `apns_auth_key.p8` | Apple Push Notification private key | +| `minio_root_password.txt` | MinIO root password (minimum 8 characters) | + +Optional (only if `tls.mode: cloudflare` in config.yaml): + +| File | Purpose | +|------|---------| +| `cloudflare-origin.crt` | Cloudflare origin certificate (PEM) | +| `cloudflare-origin.key` | Cloudflare origin certificate key (PEM) | + +All string config (registry token, domains, etc.) goes in `config.yaml` instead. +These files are gitignored and should never be committed. diff --git a/deploy-k3s/.gitignore b/deploy-k3s/.gitignore new file mode 100644 index 0000000..31d21e4 --- /dev/null +++ b/deploy-k3s/.gitignore @@ -0,0 +1,20 @@ +# Single config file (contains tokens and credentials) +config.yaml + +# Generated files +kubeconfig +cluster-config.yaml +prod.env + +# Secret files +secrets/*.txt +secrets/*.p8 +secrets/*.pem +secrets/*.key +secrets/*.crt +!secrets/README.md + +# Terraform / Hetzner state +*.tfstate +*.tfstate.backup +.terraform/ diff --git a/deploy-k3s/README.md b/deploy-k3s/README.md new file mode 100644 index 0000000..3a9768c --- /dev/null +++ b/deploy-k3s/README.md @@ -0,0 +1,391 @@ +# honeyDue — K3s Production Deployment + +Production Kubernetes deployment for honeyDue on Hetzner Cloud using K3s. + +**Architecture**: 3-node HA K3s cluster (CX33), Neon Postgres, Redis (in-cluster), Backblaze B2 (uploads), Cloudflare CDN/TLS. + +**Domains**: `api.myhoneydue.com`, `admin.myhoneydue.com` + +--- + +## Quick Start + +```bash +cd honeyDueAPI-go/deploy-k3s + +# 1. Fill in the single config file +cp config.yaml.example config.yaml +# Edit config.yaml — fill in ALL empty values + +# 2. Create secret files +# See secrets/README.md for the full list +echo "your-neon-password" > secrets/postgres_password.txt +openssl rand -base64 48 > secrets/secret_key.txt +echo "your-smtp-password" > secrets/email_host_password.txt +echo "your-fcm-key" > secrets/fcm_server_key.txt +cp /path/to/AuthKey.p8 secrets/apns_auth_key.p8 +cp /path/to/origin.pem secrets/cloudflare-origin.crt +cp /path/to/origin-key.pem secrets/cloudflare-origin.key + +# 3. Provision → Secrets → Deploy +./scripts/01-provision-cluster.sh +./scripts/02-setup-secrets.sh +./scripts/03-deploy.sh + +# 4. Set up Hetzner LB + Cloudflare DNS (see sections below) + +# 5. Verify +./scripts/04-verify.sh +curl https://api.myhoneydue.com/api/health/ +``` + +That's it. Everything reads from `config.yaml` + `secrets/`. + +--- + +## Table of Contents + +1. [Prerequisites](#1-prerequisites) +2. [Configuration](#2-configuration) +3. [Provision Cluster](#3-provision-cluster) +4. [Create Secrets](#4-create-secrets) +5. [Deploy](#5-deploy) +6. [Configure Load Balancer & DNS](#6-configure-load-balancer--dns) +7. [Verify](#7-verify) +8. [Monitoring & Logs](#8-monitoring--logs) +9. [Scaling](#9-scaling) +10. [Rollback](#10-rollback) +11. [Backup & DR](#11-backup--dr) +12. [Security Checklist](#12-security-checklist) +13. [Troubleshooting](#13-troubleshooting) + +--- + +## 1. Prerequisites + +| Tool | Install | Purpose | +|------|---------|---------| +| `hetzner-k3s` | `gem install hetzner-k3s` | Cluster provisioning | +| `kubectl` | https://kubernetes.io/docs/tasks/tools/ | Cluster management | +| `helm` | https://helm.sh/docs/intro/install/ | Optional: Prometheus/Grafana | +| `stern` | `brew install stern` | Multi-pod log tailing | +| `docker` | https://docs.docker.com/get-docker/ | Image building | +| `python3` | Pre-installed on macOS | Config parsing | +| `htpasswd` | `brew install httpd` or `apt install apache2-utils` | Admin basic auth secret | + +Verify: + +```bash +hetzner-k3s version && kubectl version --client && docker version && python3 --version +``` + +## 2. Configuration + +There are two things to fill in: + +### config.yaml — all string configuration + +```bash +cp config.yaml.example config.yaml +``` + +Open `config.yaml` and fill in every empty `""` value: + +| Section | What to fill in | +|---------|----------------| +| `cluster.hcloud_token` | Hetzner API token (Read/Write) — generate at console.hetzner.cloud | +| `registry.*` | GHCR credentials (same as Docker Swarm setup) | +| `database.host`, `database.user` | Neon PostgreSQL connection info | +| `email.user` | Fastmail email address | +| `push.apns_key_id`, `push.apns_team_id` | Apple Push Notification identifiers | +| `storage.b2_*` | Backblaze B2 bucket and credentials | +| `redis.password` | Strong password for Redis authentication (required for production) | +| `admin.basic_auth_user` | HTTP basic auth username for admin panel | +| `admin.basic_auth_password` | HTTP basic auth password for admin panel | + +Everything else has sensible defaults. `config.yaml` is gitignored. + +### secrets/ — file-based secrets + +These are binary or multi-line files that can't go in YAML: + +| File | Source | +|------|--------| +| `secrets/postgres_password.txt` | Your Neon database password | +| `secrets/secret_key.txt` | `openssl rand -base64 48` (min 32 chars) | +| `secrets/email_host_password.txt` | Fastmail app password | +| `secrets/fcm_server_key.txt` | Firebase console → Project Settings → Cloud Messaging | +| `secrets/apns_auth_key.p8` | Apple Developer → Keys → APNs key | +| `secrets/cloudflare-origin.crt` | Cloudflare → SSL/TLS → Origin Server → Create Certificate | +| `secrets/cloudflare-origin.key` | (saved with the certificate above) | + +## 3. Provision Cluster + +```bash +export KUBECONFIG=$(pwd)/kubeconfig +./scripts/01-provision-cluster.sh +``` + +This script: +1. Reads cluster config from `config.yaml` +2. Generates `cluster-config.yaml` for hetzner-k3s +3. Provisions 3x CX33 nodes with HA etcd (5-10 minutes) +4. Writes node IPs back into `config.yaml` +5. Labels the Redis node + +After provisioning: + +```bash +kubectl get nodes +``` + +## 4. Create Secrets + +```bash +./scripts/02-setup-secrets.sh +``` + +This reads `config.yaml` for registry credentials and creates all Kubernetes Secrets from the `secrets/` files: +- `honeydue-secrets` — DB password, app secret, email password, FCM key, Redis password (if configured) +- `honeydue-apns-key` — APNS .p8 key (mounted as volume in pods) +- `ghcr-credentials` — GHCR image pull credentials +- `cloudflare-origin-cert` — TLS certificate for Ingress +- `admin-basic-auth` — htpasswd secret for admin panel basic auth (if configured) + +## 5. Deploy + +**Full deploy** (build + push + apply): + +```bash +./scripts/03-deploy.sh +``` + +**Deploy pre-built images** (skip build): + +```bash +./scripts/03-deploy.sh --skip-build --tag abc1234 +``` + +The script: +1. Reads registry config from `config.yaml` +2. Builds and pushes 3 Docker images to GHCR +3. Generates a Kubernetes ConfigMap from `config.yaml` (converts to flat env vars) +4. Applies all manifests with image tag substitution +5. Waits for all rollouts to complete + +## 6. Configure Load Balancer & DNS + +### Hetzner Load Balancer + +1. [Hetzner Console](https://console.hetzner.cloud/) → **Load Balancers → Create** +2. Location: **fsn1**, add all 3 nodes as targets +3. Service: TCP 443 → 443, health check on TCP 443 +4. Note the LB IP and update `load_balancer_ip` in `config.yaml` + +### Cloudflare DNS + +1. [Cloudflare Dashboard](https://dash.cloudflare.com/) → `myhoneydue.com` → **DNS** + + | Type | Name | Content | Proxy | + |------|------|---------|-------| + | A | `api` | `` | Proxied (orange cloud) | + | A | `admin` | `` | Proxied (orange cloud) | + +2. **SSL/TLS → Overview** → Set mode to **Full (Strict)** + +3. If you haven't generated the origin cert yet: + **SSL/TLS → Origin Server → Create Certificate** + - Hostnames: `*.myhoneydue.com`, `myhoneydue.com` + - Validity: 15 years + - Save to `secrets/cloudflare-origin.crt` and `secrets/cloudflare-origin.key` + - Re-run `./scripts/02-setup-secrets.sh` + +## 7. Verify + +```bash +# Automated cluster health check +./scripts/04-verify.sh + +# External health check (after DNS propagation) +curl -v https://api.myhoneydue.com/api/health/ +``` + +Expected: `{"status": "ok"}` with HTTP 200. + +## 8. Monitoring & Logs + +### Logs with stern + +```bash +stern -n honeydue api # All API pod logs +stern -n honeydue worker # All worker logs +stern -n honeydue . # Everything +stern -n honeydue api | grep ERROR # Filter +``` + +### kubectl logs + +```bash +kubectl logs -n honeydue deployment/api -f +kubectl logs -n honeydue --previous # Crashed container +``` + +### Resource usage + +```bash +kubectl top pods -n honeydue +kubectl top nodes +``` + +### Optional: Prometheus + Grafana + +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +helm install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring --create-namespace \ + --set grafana.adminPassword=your-password + +# Access Grafana +kubectl port-forward -n monitoring svc/monitoring-grafana 3001:80 +# Open http://localhost:3001 +``` + +## 9. Scaling + +### Manual + +```bash +kubectl scale deployment/api -n honeydue --replicas=5 +kubectl scale deployment/worker -n honeydue --replicas=3 +``` + +### HPA (auto-scaling) + +API auto-scales 3→6 replicas on CPU > 70% or memory > 80%: + +```bash +kubectl get hpa -n honeydue +kubectl describe hpa api -n honeydue +``` + +### Adding nodes + +Edit `config.yaml` to add nodes, then re-run provisioning: + +```bash +./scripts/01-provision-cluster.sh +``` + +## 10. Rollback + +```bash +./scripts/rollback.sh +``` + +Shows rollout history, asks for confirmation, rolls back all deployments to previous revision. + +Single deployment rollback: + +```bash +kubectl rollout undo deployment/api -n honeydue +``` + +## 11. Backup & DR + +| Component | Strategy | Action Required | +|-----------|----------|-----------------| +| PostgreSQL | Neon PITR (automatic) | None | +| Redis | Reconstructible cache + Asynq queue | None | +| etcd | K3s auto-snapshots (12h, keeps 5) | None | +| B2 Storage | B2 versioning + lifecycle rules | Enable in B2 settings | +| Secrets | Local `secrets/` + `config.yaml` | Keep secure offline backup | + +**Disaster recovery**: Re-provision → re-create secrets → re-deploy. Database recovers via Neon PITR. + +## 12. Security + +See **[SECURITY.md](SECURITY.md)** for the comprehensive hardening guide, incident response playbooks, and full compliance checklist. + +### Summary of deployed security controls + +| Control | Status | Manifests | +|---------|--------|-----------| +| Pod security contexts (non-root, read-only FS, no caps) | Applied | All `deployment.yaml` | +| Network policies (default-deny + explicit allows) | Applied | `manifests/network-policies.yaml` | +| RBAC (dedicated SAs, no K8s API access) | Applied | `manifests/rbac.yaml` | +| Pod disruption budgets | Applied | `manifests/pod-disruption-budgets.yaml` | +| Redis authentication | Applied (if `redis.password` set) | `redis/deployment.yaml` | +| Cloudflare-only origin lockdown | Applied | `ingress/ingress.yaml` | +| Admin basic auth | Applied (if `admin.*` set) | `ingress/middleware.yaml` | +| Security headers (HSTS, CSP, Permissions-Policy) | Applied | `ingress/middleware.yaml` | +| Secret encryption at rest | K3s config | `--secrets-encryption` | + +### Quick checklist + +- [ ] Hetzner Firewall: allow only 22, 443, 6443 from your IP +- [ ] SSH: key-only auth (`PasswordAuthentication no`) +- [ ] `redis.password` set in `config.yaml` +- [ ] `admin.basic_auth_user` and `admin.basic_auth_password` set in `config.yaml` +- [ ] `kubeconfig`: `chmod 600 kubeconfig`, never commit +- [ ] `config.yaml`: contains tokens — never commit, keep secure backup +- [ ] Image scanning: `trivy image` or `docker scout cves` before deploy +- [ ] Run `./scripts/04-verify.sh` — includes automated security checks + +## 13. Troubleshooting + +### ImagePullBackOff + +```bash +kubectl describe pod -n honeydue +# Check: image name, GHCR credentials, image exists +``` + +Fix: verify `registry.*` in config.yaml, re-run `02-setup-secrets.sh`. + +### CrashLoopBackOff + +```bash +kubectl logs -n honeydue --previous +# Common: missing env vars, DB connection failure, invalid APNS key +``` + +### Redis connection refused / NOAUTH + +```bash +kubectl get pods -n honeydue -l app.kubernetes.io/name=redis + +# If redis.password is set, you must authenticate: +kubectl exec -it deploy/redis -n honeydue -- redis-cli -a "$REDIS_PASSWORD" ping +# Without -a: (error) NOAUTH Authentication required. +``` + +### Health check failures + +```bash +kubectl exec -it deploy/api -n honeydue -- curl -v http://localhost:8000/api/health/ +kubectl exec -it deploy/api -n honeydue -- env | sort +``` + +### Pods stuck in Pending + +```bash +kubectl describe pod -n honeydue +# For Redis: ensure a node has label honeydue/redis=true +kubectl get nodes --show-labels | grep redis +``` + +### DNS not resolving + +```bash +dig api.myhoneydue.com +short +# Verify LB IP matches what's in config.yaml +``` + +### Certificate / TLS errors + +```bash +kubectl get secret cloudflare-origin-cert -n honeydue +kubectl describe ingress honeydue -n honeydue +curl -vk --resolve api.myhoneydue.com:443: https://api.myhoneydue.com/api/health/ +``` diff --git a/deploy-k3s/SECURITY.md b/deploy-k3s/SECURITY.md new file mode 100644 index 0000000..e71190e --- /dev/null +++ b/deploy-k3s/SECURITY.md @@ -0,0 +1,813 @@ +# honeyDue — Production Security Hardening Guide + +Comprehensive security documentation for the honeyDue K3s deployment. Covers every layer from cloud provider to application. + +**Last updated**: 2026-03-28 + +--- + +## Table of Contents + +1. [Threat Model](#1-threat-model) +2. [Hetzner Cloud (Host)](#2-hetzner-cloud-host) +3. [K3s Cluster](#3-k3s-cluster) +4. [Pod Security](#4-pod-security) +5. [Network Segmentation](#5-network-segmentation) +6. [Redis](#6-redis) +7. [PostgreSQL (Neon)](#7-postgresql-neon) +8. [Cloudflare](#8-cloudflare) +9. [Container Images](#9-container-images) +10. [Secrets Management](#10-secrets-management) +11. [B2 Object Storage](#11-b2-object-storage) +12. [Monitoring & Alerting](#12-monitoring--alerting) +13. [Incident Response](#13-incident-response) +14. [Compliance Checklist](#14-compliance-checklist) + +--- + +## 1. Threat Model + +### What We're Protecting + +| Asset | Impact if Compromised | +|-------|----------------------| +| User credentials (bcrypt hashes) | Account takeover, password reuse attacks | +| Auth tokens | Session hijacking | +| Personal data (email, name, residences) | Privacy violation, regulatory exposure | +| Push notification keys (APNs, FCM) | Spam push to all users, key revocation | +| Cloudflare origin cert | Direct TLS impersonation | +| Database credentials | Full data exfiltration | +| Redis data | Session replay, job queue manipulation | +| B2 storage keys | Document theft or deletion | + +### Attack Surface + +``` +Internet + │ + ▼ +Cloudflare (WAF, DDoS protection, TLS termination) + │ + ▼ (origin cert, Full Strict) +Hetzner Cloud Firewall (ports 22, 443, 6443) + │ + ▼ +K3s Traefik Ingress (Cloudflare-only IP allowlist) + │ + ├──► API pods (Go) ──► Neon PostgreSQL (external, TLS) + │ ──► Redis (internal, authenticated) + │ ──► APNs/FCM (external, TLS) + │ ──► B2 Storage (external, TLS) + │ ──► SMTP (external, TLS) + │ + ├──► Admin pods (Next.js) ──► API pods (internal) + │ + └──► Worker pods (Go) ──► same as API +``` + +### Trust Boundaries + +1. **Internet → Cloudflare**: Untrusted. Cloudflare handles DDoS, WAF, TLS. +2. **Cloudflare → Origin**: Semi-trusted. Origin cert validates, IP allowlist enforces. +3. **Ingress → Pods**: Trusted network, but segmented by NetworkPolicy. +4. **Pods → External Services**: Outbound only, TLS required, credentials scoped. +5. **Pods → K8s API**: Denied. Service accounts have no permissions. + +--- + +## 2. Hetzner Cloud (Host) + +### Firewall Rules + +Only three ports should be open on the Hetzner Cloud Firewall: + +| Port | Protocol | Source | Purpose | +|------|----------|--------|---------| +| 22 | TCP | Your IP(s) only | SSH management | +| 443 | TCP | Cloudflare IPs only | HTTPS traffic | +| 6443 | TCP | Your IP(s) only | K3s API (kubectl) | + +```bash +# Verify Hetzner firewall rules (Hetzner CLI) +hcloud firewall describe honeydue-fw +``` + +### SSH Hardening + +- **Key-only authentication** — password auth disabled in `/etc/ssh/sshd_config` +- **Root login disabled** — `PermitRootLogin no` +- **fail2ban active** — auto-bans IPs after 5 failed SSH attempts + +```bash +# Verify SSH config on each node +ssh user@NODE_IP "grep -E 'PasswordAuthentication|PermitRootLogin' /etc/ssh/sshd_config" +# Expected: PasswordAuthentication no, PermitRootLogin no + +# Check fail2ban status +ssh user@NODE_IP "sudo fail2ban-client status sshd" +``` + +### OS Updates + +```bash +# Enable unattended security updates (Ubuntu 24.04) +ssh user@NODE_IP "sudo apt install unattended-upgrades && sudo dpkg-reconfigure -plow unattended-upgrades" +``` + +--- + +## 3. K3s Cluster + +### Secret Encryption at Rest + +K3s is configured with `secrets-encryption: true` in the server config. This encrypts all Secret resources in etcd using AES-CBC. + +```bash +# Verify encryption is active +k3s secrets-encrypt status +# Expected: Encryption Status: Enabled + +# Rotate encryption keys (do periodically) +k3s secrets-encrypt rotate-keys +k3s secrets-encrypt reencrypt +``` + +### RBAC + +Each workload has a dedicated ServiceAccount with `automountServiceAccountToken: false`: + +| ServiceAccount | Used By | K8s API Access | +|---------------|---------|----------------| +| `api` | API deployment | None | +| `worker` | Worker deployment | None | +| `admin` | Admin deployment | None | +| `redis` | Redis deployment | None | + +No Roles or RoleBindings are created — pods have zero K8s API access. + +```bash +# Verify service accounts exist +kubectl get sa -n honeydue + +# Verify no roles are bound +kubectl get rolebindings -n honeydue +kubectl get clusterrolebindings | grep honeydue +# Expected: no results +``` + +### Pod Disruption Budgets + +Prevent node maintenance from taking down all replicas: + +| Workload | Replicas | minAvailable | +|----------|----------|-------------| +| API | 3 | 2 | +| Worker | 2 | 1 | + +```bash +# Verify PDBs +kubectl get pdb -n honeydue +``` + +### Audit Logging (Optional Enhancement) + +K3s supports audit logging for API server requests: + +```yaml +# Add to K3s server config for detailed audit logging +# /etc/rancher/k3s/audit-policy.yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: + - level: Metadata + resources: + - group: "" + resources: ["secrets", "configmaps"] + - level: RequestResponse + users: ["system:anonymous"] + - level: None + resources: + - group: "" + resources: ["events"] +``` + +### WireGuard (Optional Enhancement) + +K3s supports WireGuard for encrypting inter-node traffic: + +```bash +# Enable WireGuard on K3s (add to server args) +# --flannel-backend=wireguard-native +``` + +--- + +## 4. Pod Security + +### Security Contexts + +Every pod runs with these security restrictions: + +**Pod-level:** +```yaml +securityContext: + runAsNonRoot: true + runAsUser: # 1000 (api/worker), 1001 (admin), 999 (redis) + runAsGroup: + fsGroup: + seccompProfile: + type: RuntimeDefault # Linux kernel syscall filtering +``` + +**Container-level:** +```yaml +securityContext: + allowPrivilegeEscalation: false # Cannot gain more privileges than parent + readOnlyRootFilesystem: true # Filesystem is immutable + capabilities: + drop: ["ALL"] # No Linux capabilities +``` + +### Writable Directories + +With `readOnlyRootFilesystem: true`, writable paths use emptyDir volumes: + +| Pod | Path | Purpose | Backing | +|-----|------|---------|---------| +| API | `/tmp` | Temp files | emptyDir (64Mi) | +| Worker | `/tmp` | Temp files | emptyDir (64Mi) | +| Admin | `/app/.next/cache` | Next.js ISR cache | emptyDir (256Mi) | +| Admin | `/tmp` | Temp files | emptyDir (64Mi) | +| Redis | `/data` | Persistence | PVC (5Gi) | +| Redis | `/tmp` | AOF rewrite temp | emptyDir tmpfs (64Mi) | + +### User IDs + +| Container | UID:GID | Source | +|-----------|---------|--------| +| API | 1000:1000 | Dockerfile `app` user | +| Worker | 1000:1000 | Dockerfile `app` user | +| Admin | 1001:1001 | Dockerfile `nextjs` user | +| Redis | 999:999 | Alpine `redis` user | + +```bash +# Verify all pods run as non-root +kubectl get pods -n honeydue -o jsonpath='{range .items[*]}{.metadata.name}{" runAsNonRoot="}{.spec.securityContext.runAsNonRoot}{"\n"}{end}' +``` + +--- + +## 5. Network Segmentation + +### Default-Deny Policy + +All ingress and egress traffic in the `honeydue` namespace is denied by default. Explicit NetworkPolicy rules allow only necessary traffic. + +### Allowed Traffic + +``` + ┌─────────────┐ + │ Traefik │ + │ (kube-system)│ + └──────┬──────┘ + │ + ┌──────────┼──────────┐ + │ │ │ + ▼ ▼ │ + ┌────────┐ ┌────────┐ │ + │ API │ │ Admin │ │ + │ :8000 │ │ :3000 │ │ + └───┬────┘ └────┬───┘ │ + │ │ │ + ┌───────┤ │ │ + │ │ │ │ + ▼ ▼ ▼ │ + ┌───────┐ ┌────────┐ ┌────────┐ │ + │ Redis │ │External│ │ API │ │ + │ :6379 │ │Services│ │(in-clr)│ │ + └───────┘ └────────┘ └────────┘ │ + ▲ │ + │ ┌────────┐ │ + └───────│ Worker │────────────┘ + └────────┘ +``` + +| Policy | From | To | Ports | +|--------|------|----|-------| +| `default-deny-all` | all | all | none | +| `allow-dns` | all pods | kube-dns | 53 UDP/TCP | +| `allow-ingress-to-api` | Traefik (kube-system) | API pods | 8000 | +| `allow-ingress-to-admin` | Traefik (kube-system) | Admin pods | 3000 | +| `allow-ingress-to-redis` | API + Worker pods | Redis | 6379 | +| `allow-egress-from-api` | API pods | Redis, external (443, 5432, 587) | various | +| `allow-egress-from-worker` | Worker pods | Redis, external (443, 5432, 587) | various | +| `allow-egress-from-admin` | Admin pods | API pods (in-cluster) | 8000 | + +**Key restrictions:** +- Redis is reachable ONLY from API and Worker pods +- Admin can ONLY reach the API service (no direct DB/Redis access) +- No pod can reach private IP ranges except in-cluster services +- External egress limited to specific ports (443, 5432, 587) + +```bash +# Verify network policies +kubectl get networkpolicy -n honeydue + +# Test: admin pod should NOT be able to reach Redis +kubectl exec -n honeydue deploy/admin -- nc -zv redis.honeydue.svc.cluster.local 6379 +# Expected: timeout/refused +``` + +--- + +## 6. Redis + +### Authentication + +Redis requires a password when `redis.password` is set in `config.yaml`: + +- Password passed via `REDIS_PASSWORD` environment variable from `honeydue-secrets` +- Redis starts with `--requirepass $REDIS_PASSWORD` +- Health probes authenticate with `-a $REDIS_PASSWORD` +- Go API connects via `redis://:PASSWORD@redis.honeydue.svc.cluster.local:6379/0` + +### Network Isolation + +- Redis has **no Ingress** — not exposed outside the cluster +- NetworkPolicy restricts access to API and Worker pods only +- Admin pods cannot reach Redis + +### Memory Limits + +- `--maxmemory 256mb` — hard cap on Redis memory +- `--maxmemory-policy noeviction` — returns errors rather than silently evicting data +- K8s resource limit: 512Mi (headroom for AOF rewrite) + +### Dangerous Command Renaming (Optional Enhancement) + +For additional protection, rename dangerous commands in a custom `redis.conf`: + +``` +rename-command FLUSHDB "" +rename-command FLUSHALL "" +rename-command DEBUG "" +rename-command CONFIG "HONEYDUE_CONFIG_a7f3b" +``` + +```bash +# Verify Redis auth is required +kubectl exec -n honeydue deploy/redis -- redis-cli ping +# Expected: (error) NOAUTH Authentication required. + +kubectl exec -n honeydue deploy/redis -- redis-cli -a "$REDIS_PASSWORD" ping +# Expected: PONG +``` + +--- + +## 7. PostgreSQL (Neon) + +### Connection Security + +- **SSL required**: `sslmode=require` in connection string +- **Connection limits**: `max_open_conns=25`, `max_idle_conns=10` +- **Scoped credentials**: Database user has access only to `honeydue` database +- **Password rotation**: Change in Neon dashboard, update `secrets/postgres_password.txt`, re-run `02-setup-secrets.sh` + +### Access Control + +- Only API and Worker pods have egress to port 5432 (NetworkPolicy enforced) +- Admin pods cannot reach the database directly +- Redis pods have no external egress + +```bash +# Verify only API/Worker can reach Neon +kubectl exec -n honeydue deploy/admin -- nc -zv ep-xxx.us-east-2.aws.neon.tech 5432 +# Expected: timeout (blocked by network policy) +``` + +### Query Safety + +- GORM uses parameterized queries (SQL injection prevention) +- No raw SQL in handlers — all queries go through repositories +- Decimal fields use `shopspring/decimal` (no floating-point errors) + +--- + +## 8. Cloudflare + +### TLS Configuration + +- **Mode**: Full (Strict) — Cloudflare validates the origin certificate +- **Origin cert**: Stored as K8s Secret `cloudflare-origin-cert` +- **Minimum TLS**: 1.2 (set in Cloudflare dashboard) +- **HSTS**: Enabled via security headers middleware + +### Origin Lockdown + +The `cloudflare-only` Traefik middleware restricts all ingress to Cloudflare IP ranges only. Direct requests to the origin IP are rejected with 403. + +```bash +# Test: direct request to origin should fail +curl -k https://ORIGIN_IP/api/health/ +# Expected: 403 Forbidden + +# Test: request through Cloudflare should work +curl https://api.myhoneydue.com/api/health/ +# Expected: 200 OK +``` + +### Cloudflare IP Range Updates + +Cloudflare IP ranges change infrequently but should be checked periodically: + +```bash +# Compare current ranges with deployed middleware +diff <(curl -s https://www.cloudflare.com/ips-v4; curl -s https://www.cloudflare.com/ips-v6) \ + <(kubectl get middleware cloudflare-only -n honeydue -o jsonpath='{.spec.ipAllowList.sourceRange[*]}' | tr ' ' '\n') +``` + +### WAF & Rate Limiting + +- **Cloudflare WAF**: Enable managed rulesets in dashboard (OWASP Core, Cloudflare Specials) +- **Rate limiting**: Traefik middleware (100 req/min, burst 200) + Go API auth rate limiting +- **Bot management**: Enable in Cloudflare dashboard for API routes + +### Security Headers + +Applied via Traefik middleware to all responses: + +| Header | Value | +|--------|-------| +| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` | +| `X-Frame-Options` | `DENY` | +| `X-Content-Type-Options` | `nosniff` | +| `X-XSS-Protection` | `1; mode=block` | +| `Referrer-Policy` | `strict-origin-when-cross-origin` | +| `Content-Security-Policy` | `default-src 'self'; frame-ancestors 'none'` | +| `Permissions-Policy` | `camera=(), microphone=(), geolocation=()` | +| `X-Permitted-Cross-Domain-Policies` | `none` | + +--- + +## 9. Container Images + +### Build Security + +- **Multi-stage builds**: Build stage discarded, only runtime artifacts copied +- **Alpine base**: Minimal attack surface (~5MB base) +- **Non-root users**: `app:1000` (Go), `nextjs:1001` (admin) +- **Stripped binaries**: Go binaries built with `-ldflags "-s -w"` (no debug symbols) +- **No shell in final image** (Go containers): Only the binary + CA certs + +### Image Scanning (Recommended) + +Add image scanning to CI/CD before pushing to GHCR: + +```bash +# Trivy scan (run in CI) +trivy image --severity HIGH,CRITICAL --exit-code 1 ghcr.io/NAMESPACE/honeydue-api:latest + +# Grype alternative +grype ghcr.io/NAMESPACE/honeydue-api:latest --fail-on high +``` + +### Version Pinning + +- Redis image: `redis:7-alpine` (pin to specific tag in production, e.g., `redis:7.4.2-alpine`) +- Go base: pinned in Dockerfile +- Node base: pinned in admin Dockerfile + +--- + +## 10. Secrets Management + +### At-Rest Encryption + +K3s encrypts all Secret resources in etcd with AES-CBC (`--secrets-encryption` flag). + +### Secret Inventory + +| Secret | Contains | Rotation Procedure | +|--------|----------|--------------------| +| `honeydue-secrets` | DB password, SECRET_KEY, SMTP password, FCM key, Redis password | Update source files + re-run `02-setup-secrets.sh` | +| `honeydue-apns-key` | APNs .p8 private key | Replace file + re-run `02-setup-secrets.sh` | +| `cloudflare-origin-cert` | TLS cert + key | Regenerate in Cloudflare + re-run `02-setup-secrets.sh` | +| `ghcr-credentials` | Registry PAT | Regenerate GitHub PAT + re-run `02-setup-secrets.sh` | +| `admin-basic-auth` | htpasswd hash | Update config.yaml + re-run `02-setup-secrets.sh` | + +### Rotation Procedure + +```bash +# 1. Update the secret source (file or config.yaml value) +# 2. Re-run the secrets script +./scripts/02-setup-secrets.sh + +# 3. Restart affected pods to pick up new secret values +kubectl rollout restart deployment/api deployment/worker -n honeydue + +# 4. Verify pods are healthy +kubectl get pods -n honeydue -w +``` + +### Secret Hygiene + +- `secrets/` directory is gitignored — never committed +- `config.yaml` is gitignored — never committed +- Scripts validate secret files exist and aren't empty before creating K8s secrets +- `SECRET_KEY` requires minimum 32 characters +- ConfigMap redacts sensitive values in `04-verify.sh` output + +--- + +## 11. B2 Object Storage + +### Access Control + +- **Scoped application key**: Create a B2 key with access to only the `honeydue` bucket +- **Permissions**: Read + Write only (no `deleteFiles`, no `listAllBucketNames`) +- **Bucket-only**: Key cannot access other buckets in the account + +```bash +# Create scoped B2 key (Backblaze CLI) +b2 create-key --bucket BUCKET_NAME honeydue-api readFiles,writeFiles,listFiles +``` + +### Upload Validation (Go API) + +- File size limit: `STORAGE_MAX_FILE_SIZE` (10MB default) +- Allowed MIME types: `STORAGE_ALLOWED_TYPES` (images + PDF only) +- Path traversal protection in upload handler +- Files served via authenticated proxy (`media_handler`) — no direct B2 URLs exposed to clients + +### Versioning + +Enable B2 bucket versioning to protect against accidental deletion: + +```bash +# Enable versioning on the B2 bucket +b2 update-bucket --versioning enabled BUCKET_NAME +``` + +--- + +## 12. Monitoring & Alerting + +### Log Aggregation + +K3s logs are available via `kubectl logs`. For persistent log aggregation: + +```bash +# View API logs +kubectl logs -n honeydue -l app.kubernetes.io/name=api --tail=100 -f + +# View worker logs +kubectl logs -n honeydue -l app.kubernetes.io/name=worker --tail=100 -f + +# View all warning events +kubectl get events -n honeydue --field-selector type=Warning --sort-by='.lastTimestamp' +``` + +**Recommended**: Deploy Loki + Grafana for persistent log search and alerting. + +### Health Monitoring + +```bash +# Continuous health monitoring +watch -n 10 "kubectl get pods -n honeydue -o wide && echo && kubectl top pods -n honeydue 2>/dev/null" + +# Check pod restart counts (indicator of crashes) +kubectl get pods -n honeydue -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' +``` + +### Alerting Thresholds + +| Metric | Warning | Critical | Check Command | +|--------|---------|----------|---------------| +| Pod restarts | > 3 in 1h | > 10 in 1h | `kubectl get pods` | +| API response time | > 500ms p95 | > 2s p95 | Cloudflare Analytics | +| Memory usage | > 80% limit | > 95% limit | `kubectl top pods` | +| Redis memory | > 200MB | > 250MB | `redis-cli info memory` | +| Disk (PVC) | > 80% | > 95% | `kubectl exec ... df -h` | +| Certificate expiry | < 30 days | < 7 days | Cloudflare dashboard | + +### Audit Trail + +- **K8s events**: `kubectl get events -n honeydue` (auto-pruned after 1h) +- **Go API**: zerolog structured logging with credential masking +- **Cloudflare**: Access logs, WAF logs, rate limiting logs in dashboard +- **Hetzner**: SSH auth logs in `/var/log/auth.log` + +--- + +## 13. Incident Response + +### Playbook: Compromised API Token + +```bash +# 1. Rotate SECRET_KEY to invalidate ALL tokens +echo "$(openssl rand -hex 32)" > secrets/secret_key.txt +./scripts/02-setup-secrets.sh +kubectl rollout restart deployment/api deployment/worker -n honeydue + +# 2. All users will need to re-authenticate +``` + +### Playbook: Compromised Database Credentials + +```bash +# 1. Rotate password in Neon dashboard +# 2. Update local secret file +echo "NEW_PASSWORD" > secrets/postgres_password.txt +./scripts/02-setup-secrets.sh +kubectl rollout restart deployment/api deployment/worker -n honeydue + +# 3. Monitor for connection errors +kubectl logs -n honeydue -l app.kubernetes.io/name=api --tail=50 -f +``` + +### Playbook: Compromised Push Notification Keys + +```bash +# APNs: Revoke key in Apple Developer Console, generate new .p8 +cp new_key.p8 secrets/apns_auth_key.p8 +./scripts/02-setup-secrets.sh +kubectl rollout restart deployment/api deployment/worker -n honeydue + +# FCM: Rotate server key in Firebase Console +echo "NEW_FCM_KEY" > secrets/fcm_server_key.txt +./scripts/02-setup-secrets.sh +kubectl rollout restart deployment/api deployment/worker -n honeydue +``` + +### Playbook: Suspicious Pod Behavior + +```bash +# 1. Isolate the pod (remove from service) +kubectl label pod SUSPICIOUS_POD -n honeydue app.kubernetes.io/name- + +# 2. Capture state for investigation +kubectl logs SUSPICIOUS_POD -n honeydue > /tmp/suspicious-logs.txt +kubectl describe pod SUSPICIOUS_POD -n honeydue > /tmp/suspicious-describe.txt + +# 3. Delete and let deployment recreate +kubectl delete pod SUSPICIOUS_POD -n honeydue +``` + +### Communication Plan + +1. **Internal**: Document incident timeline in a private channel +2. **Users**: If data breach — notify affected users within 72 hours +3. **Vendors**: Revoke/rotate all potentially compromised credentials +4. **Post-mortem**: Document root cause, timeline, remediation, prevention + +--- + +## 14. Compliance Checklist + +Run through this checklist before production launch and periodically thereafter. + +### Infrastructure + +- [ ] Hetzner firewall allows only ports 22, 443, 6443 +- [ ] SSH password auth disabled on all nodes +- [ ] fail2ban active on all nodes +- [ ] OS security updates enabled (unattended-upgrades) + +```bash +# Verify +hcloud firewall describe honeydue-fw +ssh user@NODE "grep PasswordAuthentication /etc/ssh/sshd_config" +ssh user@NODE "sudo fail2ban-client status sshd" +``` + +### K3s Cluster + +- [ ] Secret encryption enabled +- [ ] Service accounts created with no API access +- [ ] Pod disruption budgets deployed +- [ ] No default service account used by workloads + +```bash +# Verify +k3s secrets-encrypt status +kubectl get sa -n honeydue +kubectl get pdb -n honeydue +kubectl get pods -n honeydue -o jsonpath='{range .items[*]}{.metadata.name}{" sa="}{.spec.serviceAccountName}{"\n"}{end}' +``` + +### Pod Security + +- [ ] All pods: `runAsNonRoot: true` +- [ ] All containers: `allowPrivilegeEscalation: false` +- [ ] All containers: `readOnlyRootFilesystem: true` +- [ ] All containers: `capabilities.drop: ["ALL"]` +- [ ] All pods: `seccompProfile.type: RuntimeDefault` + +```bash +# Verify (automated check in 04-verify.sh) +./scripts/04-verify.sh +``` + +### Network + +- [ ] Default-deny NetworkPolicy applied +- [ ] 8+ explicit allow policies deployed +- [ ] Redis only reachable from API + Worker +- [ ] Admin only reaches API service +- [ ] Cloudflare-only middleware applied to all ingress + +```bash +# Verify +kubectl get networkpolicy -n honeydue +kubectl get ingress -n honeydue -o yaml | grep cloudflare-only +``` + +### Authentication & Authorization + +- [ ] Redis requires password +- [ ] Admin panel has basic auth layer +- [ ] API uses bcrypt for passwords +- [ ] Auth tokens have expiration +- [ ] Rate limiting on auth endpoints + +```bash +# Verify Redis auth +kubectl exec -n honeydue deploy/redis -- redis-cli ping +# Expected: NOAUTH error + +# Verify admin auth +kubectl get secret admin-basic-auth -n honeydue +``` + +### Secrets + +- [ ] All secrets stored as K8s Secrets (not ConfigMap) +- [ ] Secrets encrypted at rest (K3s) +- [ ] No secrets in git history +- [ ] SECRET_KEY >= 32 characters +- [ ] Secret rotation documented + +```bash +# Verify no secrets in ConfigMap +kubectl get configmap honeydue-config -n honeydue -o yaml | grep -iE 'password|secret|token|key' +# Should show only non-sensitive config keys (EMAIL_HOST, APNS_KEY_ID, etc.) +``` + +### TLS & Headers + +- [ ] Cloudflare Full (Strict) mode enabled +- [ ] Origin cert valid and not expired +- [ ] HSTS header present with includeSubDomains +- [ ] CSP header: `default-src 'self'; frame-ancestors 'none'` +- [ ] Permissions-Policy blocks camera/mic/geo +- [ ] X-Frame-Options: DENY + +```bash +# Verify headers (via Cloudflare) +curl -sI https://api.myhoneydue.com/api/health/ | grep -iE 'strict-transport|content-security|permissions-policy|x-frame' +``` + +### Container Images + +- [ ] Multi-stage Dockerfile (no build tools in runtime) +- [ ] Non-root user in all images +- [ ] Alpine base (minimal surface) +- [ ] No secrets baked into images + +```bash +# Verify non-root +kubectl get pods -n honeydue -o jsonpath='{range .items[*]}{.metadata.name}{" uid="}{.spec.securityContext.runAsUser}{"\n"}{end}' +``` + +### External Services + +- [ ] PostgreSQL: `sslmode=require` +- [ ] B2: Scoped application key (single bucket) +- [ ] APNs: .p8 key (not .p12 certificate) +- [ ] SMTP: TLS enabled (`use_tls: true`) + +--- + +## Quick Reference Commands + +```bash +# Full security verification +./scripts/04-verify.sh + +# Rotate all secrets +./scripts/02-setup-secrets.sh && \ +kubectl rollout restart deployment/api deployment/worker deployment/admin -n honeydue + +# Check for security events +kubectl get events -n honeydue --field-selector type=Warning + +# Emergency: scale down everything +kubectl scale deployment --all -n honeydue --replicas=0 + +# Emergency: restore +kubectl scale deployment api -n honeydue --replicas=3 +kubectl scale deployment worker -n honeydue --replicas=2 +kubectl scale deployment admin -n honeydue --replicas=1 +kubectl scale deployment redis -n honeydue --replicas=1 +``` diff --git a/deploy-k3s/config.yaml.example b/deploy-k3s/config.yaml.example new file mode 100644 index 0000000..97d9314 --- /dev/null +++ b/deploy-k3s/config.yaml.example @@ -0,0 +1,118 @@ +# config.yaml — single source of truth for honeyDue K3s deployment +# Copy to config.yaml, fill in all empty values, then run scripts in order. +# This file is gitignored — never commit it with real values. + +# --- Hetzner Cloud --- +cluster: + hcloud_token: "" # Hetzner API token (Read/Write) + ssh_public_key: ~/.ssh/id_ed25519.pub + ssh_private_key: ~/.ssh/id_ed25519 + k3s_version: v1.31.4+k3s1 + location: fsn1 # Hetzner datacenter + instance_type: cx33 # 4 vCPU, 16GB RAM + +# Filled by 01-provision-cluster.sh, or manually after creating servers +nodes: + - name: honeydue-master1 + ip: "" + roles: [master, redis] # 'redis' = pin Redis PVC here + - name: honeydue-master2 + ip: "" + roles: [master] + - name: honeydue-master3 + ip: "" + roles: [master] + +# Hetzner Load Balancer IP (created in console after provisioning) +load_balancer_ip: "" + +# --- Domains --- +domains: + api: api.myhoneydue.com + admin: admin.myhoneydue.com + base: myhoneydue.com + +# --- Container Registry (GHCR) --- +registry: + server: ghcr.io + namespace: "" # GitHub username or org + username: "" # GitHub username + token: "" # PAT with read:packages, write:packages + +# --- Database (Neon PostgreSQL) --- +database: + host: "" # e.g. ep-xxx.us-east-2.aws.neon.tech + port: 5432 + user: "" + name: honeydue + sslmode: require + max_open_conns: 25 + max_idle_conns: 10 + max_lifetime: "600s" + +# --- Email (Fastmail) --- +email: + host: smtp.fastmail.com + port: 587 + user: "" # Fastmail email address + from: "honeyDue " + use_tls: true + +# --- Push Notifications --- +push: + apns_key_id: "" + apns_team_id: "" + apns_topic: com.tt.honeyDue + apns_production: true + apns_use_sandbox: false + +# --- B2 Object Storage --- +storage: + b2_key_id: "" + b2_app_key: "" + b2_bucket: "" + b2_endpoint: "" # e.g. s3.us-west-004.backblazeb2.com + max_file_size: 10485760 + allowed_types: "image/jpeg,image/png,image/gif,image/webp,application/pdf" + +# --- Worker Schedules (UTC hours) --- +worker: + task_reminder_hour: 14 + overdue_reminder_hour: 15 + daily_digest_hour: 3 + +# --- Feature Flags --- +features: + push_enabled: true + email_enabled: true + webhooks_enabled: true + onboarding_emails_enabled: true + pdf_reports_enabled: true + worker_enabled: true + +# --- Redis --- +redis: + password: "" # Set a strong password; leave empty for no auth (NOT recommended for production) + +# --- Admin Panel --- +admin: + basic_auth_user: "" # HTTP basic auth username for admin panel + basic_auth_password: "" # HTTP basic auth password for admin panel + +# --- Apple Auth / IAP (optional, leave empty if unused) --- +apple_auth: + client_id: "" + team_id: "" + iap_key_id: "" + iap_issuer_id: "" + iap_bundle_id: "" + iap_key_path: "" + iap_sandbox: false + +# --- Google Auth / IAP (optional, leave empty if unused) --- +google_auth: + client_id: "" + android_client_id: "" + ios_client_id: "" + iap_package_name: "" + iap_service_account_path: "" diff --git a/deploy-k3s/manifests/admin/deployment.yaml b/deploy-k3s/manifests/admin/deployment.yaml new file mode 100644 index 0000000..4a33296 --- /dev/null +++ b/deploy-k3s/manifests/admin/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: admin + template: + metadata: + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: admin + imagePullSecrets: + - name: ghcr-credentials + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: admin + image: IMAGE_PLACEHOLDER # Replaced by 03-deploy.sh + ports: + - containerPort: 3000 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: PORT + value: "3000" + - name: HOSTNAME + value: "0.0.0.0" + - name: NEXT_PUBLIC_API_URL + valueFrom: + configMapKeyRef: + name: honeydue-config + key: NEXT_PUBLIC_API_URL + volumeMounts: + - name: nextjs-cache + mountPath: /app/.next/cache + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + startupProbe: + httpGet: + path: /admin/ + port: 3000 + failureThreshold: 12 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /admin/ + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /admin/ + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + volumes: + - name: nextjs-cache + emptyDir: + sizeLimit: 256Mi + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/deploy-k3s/manifests/admin/service.yaml b/deploy-k3s/manifests/admin/service.yaml new file mode 100644 index 0000000..fa7a4ff --- /dev/null +++ b/deploy-k3s/manifests/admin/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: admin + ports: + - port: 3000 + targetPort: 3000 + protocol: TCP diff --git a/deploy-k3s/manifests/ingress/ingress.yaml b/deploy-k3s/manifests/ingress/ingress.yaml new file mode 100644 index 0000000..e30bf31 --- /dev/null +++ b/deploy-k3s/manifests/ingress/ingress.yaml @@ -0,0 +1,54 @@ +# API Ingress — Cloudflare-only + security headers + rate limiting +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: honeydue-api + namespace: honeydue + labels: + app.kubernetes.io/part-of: honeydue + annotations: + traefik.ingress.kubernetes.io/router.middlewares: honeydue-cloudflare-only@kubernetescrd,honeydue-security-headers@kubernetescrd,honeydue-rate-limit@kubernetescrd +spec: + tls: + - hosts: + - api.myhoneydue.com + secretName: cloudflare-origin-cert + rules: + - host: api.myhoneydue.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api + port: + number: 8000 + +--- +# Admin Ingress — Cloudflare-only + security headers + rate limiting + basic auth +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: honeydue-admin + namespace: honeydue + labels: + app.kubernetes.io/part-of: honeydue + annotations: + traefik.ingress.kubernetes.io/router.middlewares: honeydue-cloudflare-only@kubernetescrd,honeydue-security-headers@kubernetescrd,honeydue-rate-limit@kubernetescrd,honeydue-admin-auth@kubernetescrd +spec: + tls: + - hosts: + - admin.myhoneydue.com + secretName: cloudflare-origin-cert + rules: + - host: admin.myhoneydue.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: admin + port: + number: 3000 diff --git a/deploy-k3s/manifests/ingress/middleware.yaml b/deploy-k3s/manifests/ingress/middleware.yaml new file mode 100644 index 0000000..5ea56b3 --- /dev/null +++ b/deploy-k3s/manifests/ingress/middleware.yaml @@ -0,0 +1,82 @@ +# Traefik CRD middleware for rate limiting +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: rate-limit + namespace: honeydue +spec: + rateLimit: + average: 100 + burst: 200 + period: 1m + +--- +# Security headers +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: security-headers + namespace: honeydue +spec: + headers: + frameDeny: true + contentTypeNosniff: true + browserXssFilter: true + referrerPolicy: "strict-origin-when-cross-origin" + customResponseHeaders: + X-Content-Type-Options: "nosniff" + X-Frame-Options: "DENY" + Strict-Transport-Security: "max-age=31536000; includeSubDomains" + Content-Security-Policy: "default-src 'self'; frame-ancestors 'none'" + Permissions-Policy: "camera=(), microphone=(), geolocation=()" + X-Permitted-Cross-Domain-Policies: "none" + +--- +# Cloudflare IP allowlist (restrict origin to Cloudflare only) +# https://www.cloudflare.com/ips-v4 and /ips-v6 +# Update periodically: curl -s https://www.cloudflare.com/ips-v4 && curl -s https://www.cloudflare.com/ips-v6 +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: cloudflare-only + namespace: honeydue +spec: + ipAllowList: + sourceRange: + # Cloudflare IPv4 ranges + - 173.245.48.0/20 + - 103.21.244.0/22 + - 103.22.200.0/22 + - 103.31.4.0/22 + - 141.101.64.0/18 + - 108.162.192.0/18 + - 190.93.240.0/20 + - 188.114.96.0/20 + - 197.234.240.0/22 + - 198.41.128.0/17 + - 162.158.0.0/15 + - 104.16.0.0/13 + - 104.24.0.0/14 + - 172.64.0.0/13 + - 131.0.72.0/22 + # Cloudflare IPv6 ranges + - 2400:cb00::/32 + - 2606:4700::/32 + - 2803:f800::/32 + - 2405:b500::/32 + - 2405:8100::/32 + - 2a06:98c0::/29 + - 2c0f:f248::/32 + +--- +# Admin basic auth — additional auth layer for admin panel +# Secret created by 02-setup-secrets.sh from config.yaml credentials +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: admin-auth + namespace: honeydue +spec: + basicAuth: + secret: admin-basic-auth + realm: "honeyDue Admin" diff --git a/deploy-k3s/manifests/namespace.yaml b/deploy-k3s/manifests/namespace.yaml new file mode 100644 index 0000000..76f4908 --- /dev/null +++ b/deploy-k3s/manifests/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: honeydue + labels: + app.kubernetes.io/part-of: honeydue diff --git a/deploy-k3s/manifests/network-policies.yaml b/deploy-k3s/manifests/network-policies.yaml new file mode 100644 index 0000000..cf5e08c --- /dev/null +++ b/deploy-k3s/manifests/network-policies.yaml @@ -0,0 +1,202 @@ +# Network Policies — default-deny with explicit allows +# Apply AFTER namespace and deployments are created. +# Verify: kubectl get networkpolicy -n honeydue + +# --- Default deny all ingress and egress --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: honeydue +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + +--- +# --- Allow DNS for all pods (required for service discovery) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns + namespace: honeydue +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: [] + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + +--- +# --- API: allow ingress from Traefik (kube-system namespace) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-api + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: api + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: TCP + port: 8000 + +--- +# --- Admin: allow ingress from Traefik (kube-system namespace) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-admin + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: admin + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: TCP + port: 3000 + +--- +# --- Redis: allow ingress ONLY from api + worker pods --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-ingress-to-redis + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: redis + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + - podSelector: + matchLabels: + app.kubernetes.io/name: worker + ports: + - protocol: TCP + port: 6379 + +--- +# --- API: allow egress to Redis, external services (Neon DB, APNs, FCM, B2, SMTP) --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-api + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: api + policyTypes: + - Egress + egress: + # Redis (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + # External services: Neon DB (5432), SMTP (587), HTTPS (443 — APNs, FCM, B2, PostHog) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 587 + - protocol: TCP + port: 443 + +--- +# --- Worker: allow egress to Redis, external services --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-worker + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: worker + policyTypes: + - Egress + egress: + # Redis (in-cluster) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + # External services: Neon DB (5432), SMTP (587), HTTPS (443 — APNs, FCM, B2) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - protocol: TCP + port: 5432 + - protocol: TCP + port: 587 + - protocol: TCP + port: 443 + +--- +# --- Admin: allow egress to API (internal) for SSR --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-from-admin + namespace: honeydue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: admin + policyTypes: + - Egress + egress: + # API service (in-cluster, for server-side API calls) + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: api + ports: + - protocol: TCP + port: 8000 diff --git a/deploy-k3s/manifests/pod-disruption-budgets.yaml b/deploy-k3s/manifests/pod-disruption-budgets.yaml new file mode 100644 index 0000000..e1abea8 --- /dev/null +++ b/deploy-k3s/manifests/pod-disruption-budgets.yaml @@ -0,0 +1,32 @@ +# Pod Disruption Budgets — prevent node maintenance from killing all replicas +# API: at least 2 of 3 replicas must stay up during voluntary disruptions +# Worker: at least 1 of 2 replicas must stay up + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: api-pdb + namespace: honeydue + labels: + app.kubernetes.io/name: api + app.kubernetes.io/part-of: honeydue +spec: + minAvailable: 2 + selector: + matchLabels: + app.kubernetes.io/name: api + +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: worker-pdb + namespace: honeydue + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: honeydue +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: worker diff --git a/deploy-k3s/manifests/rbac.yaml b/deploy-k3s/manifests/rbac.yaml new file mode 100644 index 0000000..69e0860 --- /dev/null +++ b/deploy-k3s/manifests/rbac.yaml @@ -0,0 +1,46 @@ +# RBAC — Dedicated service accounts with no K8s API access +# Each pod gets its own SA with automountServiceAccountToken: false, +# so a compromised pod cannot query the Kubernetes API. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: api + namespace: honeydue + labels: + app.kubernetes.io/name: api + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: worker + namespace: honeydue + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin + namespace: honeydue + labels: + app.kubernetes.io/name: admin + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +automountServiceAccountToken: false diff --git a/deploy-k3s/manifests/redis/deployment.yaml b/deploy-k3s/manifests/redis/deployment.yaml new file mode 100644 index 0000000..7660c2a --- /dev/null +++ b/deploy-k3s/manifests/redis/deployment.yaml @@ -0,0 +1,106 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + replicas: 1 + strategy: + type: Recreate # ReadWriteOnce PVC — can't attach to two pods + selector: + matchLabels: + app.kubernetes.io/name: redis + template: + metadata: + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue + spec: + serviceAccountName: redis + nodeSelector: + honeydue/redis: "true" + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: redis + image: redis:7-alpine + command: + - sh + - -c + - | + ARGS="--appendonly yes --appendfsync everysec --maxmemory 256mb --maxmemory-policy noeviction" + if [ -n "$REDIS_PASSWORD" ]; then + ARGS="$ARGS --requirepass $REDIS_PASSWORD" + fi + exec redis-server $ARGS + ports: + - containerPort: 6379 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: honeydue-secrets + key: REDIS_PASSWORD + optional: true + volumeMounts: + - name: redis-data + mountPath: /data + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + exec: + command: + - sh + - -c + - | + if [ -n "$REDIS_PASSWORD" ]; then + redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null | grep -q PONG + else + redis-cli ping | grep -q PONG + fi + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - sh + - -c + - | + if [ -n "$REDIS_PASSWORD" ]; then + redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null | grep -q PONG + else + redis-cli ping | grep -q PONG + fi + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + volumes: + - name: redis-data + persistentVolumeClaim: + claimName: redis-data + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 64Mi diff --git a/deploy-k3s/manifests/redis/pvc.yaml b/deploy-k3s/manifests/redis/pvc.yaml new file mode 100644 index 0000000..dbddee5 --- /dev/null +++ b/deploy-k3s/manifests/redis/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 5Gi diff --git a/deploy-k3s/manifests/redis/service.yaml b/deploy-k3s/manifests/redis/service.yaml new file mode 100644 index 0000000..9bed2ea --- /dev/null +++ b/deploy-k3s/manifests/redis/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: honeydue + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: honeydue +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: redis + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP diff --git a/deploy-k3s/manifests/secrets.yaml.example b/deploy-k3s/manifests/secrets.yaml.example new file mode 100644 index 0000000..3d72b85 --- /dev/null +++ b/deploy-k3s/manifests/secrets.yaml.example @@ -0,0 +1,47 @@ +# EXAMPLE ONLY — never commit real values. +# Secrets are created by scripts/02-setup-secrets.sh. +# This file shows the expected structure for reference. + +--- +apiVersion: v1 +kind: Secret +metadata: + name: honeydue-secrets + namespace: honeydue +type: Opaque +stringData: + POSTGRES_PASSWORD: "CHANGEME" + SECRET_KEY: "CHANGEME_MIN_32_CHARS" + EMAIL_HOST_PASSWORD: "CHANGEME" + FCM_SERVER_KEY: "CHANGEME" + +--- +apiVersion: v1 +kind: Secret +metadata: + name: honeydue-apns-key + namespace: honeydue +type: Opaque +data: + apns_auth_key.p8: "" # base64-encoded .p8 file contents + +--- +apiVersion: v1 +kind: Secret +metadata: + name: ghcr-credentials + namespace: honeydue +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: "" # base64-encoded Docker config + +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloudflare-origin-cert + namespace: honeydue +type: kubernetes.io/tls +data: + tls.crt: "" # base64-encoded origin certificate + tls.key: "" # base64-encoded origin private key diff --git a/deploy-k3s/scripts/01-provision-cluster.sh b/deploy-k3s/scripts/01-provision-cluster.sh new file mode 100755 index 0000000..d85c2b1 --- /dev/null +++ b/deploy-k3s/scripts/01-provision-cluster.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +log() { printf '[provision] %s\n' "$*"; } +die() { printf '[provision][error] %s\n' "$*" >&2; exit 1; } + +# --- Prerequisites --- + +command -v hetzner-k3s >/dev/null 2>&1 || die "Missing: hetzner-k3s CLI. Install: https://github.com/vitobotta/hetzner-k3s" +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" + +HCLOUD_TOKEN="$(cfg_require cluster.hcloud_token "Hetzner API token")" +export HCLOUD_TOKEN + +# Validate SSH keys +SSH_PUB="$(cfg cluster.ssh_public_key | sed "s|~|${HOME}|g")" +SSH_PRIV="$(cfg cluster.ssh_private_key | sed "s|~|${HOME}|g")" +[[ -f "${SSH_PUB}" ]] || die "SSH public key not found: ${SSH_PUB}" +[[ -f "${SSH_PRIV}" ]] || die "SSH private key not found: ${SSH_PRIV}" + +# --- Generate hetzner-k3s cluster config from config.yaml --- + +CLUSTER_CONFIG="${DEPLOY_DIR}/cluster-config.yaml" +log "Generating cluster-config.yaml from config.yaml..." +generate_cluster_config > "${CLUSTER_CONFIG}" + +# --- Provision --- + +INSTANCE_TYPE="$(cfg cluster.instance_type)" +LOCATION="$(cfg cluster.location)" +NODE_COUNT="$(node_count)" + +log "Provisioning K3s cluster on Hetzner Cloud..." +log " Nodes: ${NODE_COUNT}x ${INSTANCE_TYPE} in ${LOCATION}" +log " This takes about 5-10 minutes." +echo "" + +hetzner-k3s create --config "${CLUSTER_CONFIG}" + +KUBECONFIG_PATH="${DEPLOY_DIR}/kubeconfig" + +if [[ ! -f "${KUBECONFIG_PATH}" ]]; then + die "Provisioning completed but kubeconfig not found. Check hetzner-k3s output." +fi + +# --- Write node IPs back to config.yaml --- + +log "Querying node IPs..." +export KUBECONFIG="${KUBECONFIG_PATH}" + +python3 -c " +import yaml, subprocess, json + +# Get node info from kubectl +result = subprocess.run( + ['kubectl', 'get', 'nodes', '-o', 'json'], + capture_output=True, text=True +) +nodes_json = json.loads(result.stdout) + +# Build name → IP map +ip_map = {} +for node in nodes_json.get('items', []): + name = node['metadata']['name'] + for addr in node.get('status', {}).get('addresses', []): + if addr['type'] == 'ExternalIP': + ip_map[name] = addr['address'] + break + else: + for addr in node.get('status', {}).get('addresses', []): + if addr['type'] == 'InternalIP': + ip_map[name] = addr['address'] + break + +# Update config.yaml with IPs +with open('${CONFIG_FILE}') as f: + config = yaml.safe_load(f) + +updated = 0 +for i, node in enumerate(config.get('nodes', [])): + for real_name, ip in ip_map.items(): + if node['name'] in real_name or real_name in node['name']: + config['nodes'][i]['ip'] = ip + config['nodes'][i]['name'] = real_name + updated += 1 + break + +if updated == 0 and ip_map: + # Names didn't match — assign by index + for i, (name, ip) in enumerate(sorted(ip_map.items())): + if i < len(config['nodes']): + config['nodes'][i]['name'] = name + config['nodes'][i]['ip'] = ip + updated += 1 + +with open('${CONFIG_FILE}', 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + +print(f'Updated {updated} node IPs in config.yaml') +for name, ip in sorted(ip_map.items()): + print(f' {name}: {ip}') +" + +# --- Label Redis node --- + +REDIS_NODE="$(nodes_with_role redis | head -1)" +if [[ -n "${REDIS_NODE}" ]]; then + # Find the actual K8s node name that matches + ACTUAL_NODE="$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | head -1)" + log "Labeling node ${ACTUAL_NODE} for Redis..." + kubectl label node "${ACTUAL_NODE}" honeydue/redis=true --overwrite +fi + +log "" +log "Cluster provisioned successfully." +log "" +log "Next steps:" +log " export KUBECONFIG=${KUBECONFIG_PATH}" +log " kubectl get nodes" +log " ./scripts/02-setup-secrets.sh" diff --git a/deploy-k3s/scripts/02-setup-secrets.sh b/deploy-k3s/scripts/02-setup-secrets.sh new file mode 100755 index 0000000..5e9c3a7 --- /dev/null +++ b/deploy-k3s/scripts/02-setup-secrets.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +SECRETS_DIR="${DEPLOY_DIR}/secrets" +NAMESPACE="honeydue" + +log() { printf '[secrets] %s\n' "$*"; } +warn() { printf '[secrets][warn] %s\n' "$*" >&2; } +die() { printf '[secrets][error] %s\n' "$*" >&2; exit 1; } + +# --- Prerequisites --- + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" + +kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 || { + log "Creating namespace ${NAMESPACE}..." + kubectl apply -f "${DEPLOY_DIR}/manifests/namespace.yaml" +} + +# --- Validate secret files --- + +require_file() { + local path="$1" label="$2" + [[ -f "${path}" ]] || die "Missing: ${path} (${label})" + [[ -s "${path}" ]] || die "Empty: ${path} (${label})" +} + +require_file "${SECRETS_DIR}/postgres_password.txt" "Postgres password" +require_file "${SECRETS_DIR}/secret_key.txt" "SECRET_KEY" +require_file "${SECRETS_DIR}/email_host_password.txt" "SMTP password" +require_file "${SECRETS_DIR}/fcm_server_key.txt" "FCM server key" +require_file "${SECRETS_DIR}/apns_auth_key.p8" "APNS private key" +require_file "${SECRETS_DIR}/cloudflare-origin.crt" "Cloudflare origin cert" +require_file "${SECRETS_DIR}/cloudflare-origin.key" "Cloudflare origin key" + +# Validate APNS key format +if ! grep -q "BEGIN PRIVATE KEY" "${SECRETS_DIR}/apns_auth_key.p8"; then + die "APNS key file does not look like a private key: ${SECRETS_DIR}/apns_auth_key.p8" +fi + +# Validate secret_key length (minimum 32 chars) +SECRET_KEY_LEN="$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt" | wc -c | tr -d ' ')" +if (( SECRET_KEY_LEN < 32 )); then + die "secret_key.txt must be at least 32 characters (got ${SECRET_KEY_LEN})." +fi + +# --- Read optional config values --- + +REDIS_PASSWORD="$(cfg redis.password 2>/dev/null || true)" +ADMIN_AUTH_USER="$(cfg admin.basic_auth_user 2>/dev/null || true)" +ADMIN_AUTH_PASSWORD="$(cfg admin.basic_auth_password 2>/dev/null || true)" + +# --- Create app secrets --- + +log "Creating honeydue-secrets..." +SECRET_ARGS=( + --namespace="${NAMESPACE}" + --from-literal="POSTGRES_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/postgres_password.txt")" + --from-literal="SECRET_KEY=$(tr -d '\r\n' < "${SECRETS_DIR}/secret_key.txt")" + --from-literal="EMAIL_HOST_PASSWORD=$(tr -d '\r\n' < "${SECRETS_DIR}/email_host_password.txt")" + --from-literal="FCM_SERVER_KEY=$(tr -d '\r\n' < "${SECRETS_DIR}/fcm_server_key.txt")" +) + +if [[ -n "${REDIS_PASSWORD}" ]]; then + log " Including REDIS_PASSWORD in secrets" + SECRET_ARGS+=(--from-literal="REDIS_PASSWORD=${REDIS_PASSWORD}") +fi + +kubectl create secret generic honeydue-secrets \ + "${SECRET_ARGS[@]}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Create APNS key secret --- + +log "Creating honeydue-apns-key..." +kubectl create secret generic honeydue-apns-key \ + --namespace="${NAMESPACE}" \ + --from-file="apns_auth_key.p8=${SECRETS_DIR}/apns_auth_key.p8" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Create GHCR registry credentials --- + +REGISTRY_SERVER="$(cfg registry.server)" +REGISTRY_USER="$(cfg registry.username)" +REGISTRY_TOKEN="$(cfg registry.token)" + +if [[ -n "${REGISTRY_SERVER}" && -n "${REGISTRY_USER}" && -n "${REGISTRY_TOKEN}" ]]; then + log "Creating ghcr-credentials..." + kubectl create secret docker-registry ghcr-credentials \ + --namespace="${NAMESPACE}" \ + --docker-server="${REGISTRY_SERVER}" \ + --docker-username="${REGISTRY_USER}" \ + --docker-password="${REGISTRY_TOKEN}" \ + --dry-run=client -o yaml | kubectl apply -f - +else + warn "Registry credentials incomplete in config.yaml — skipping ghcr-credentials." +fi + +# --- Create Cloudflare origin cert --- + +log "Creating cloudflare-origin-cert..." +kubectl create secret tls cloudflare-origin-cert \ + --namespace="${NAMESPACE}" \ + --cert="${SECRETS_DIR}/cloudflare-origin.crt" \ + --key="${SECRETS_DIR}/cloudflare-origin.key" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Create admin basic auth secret --- + +if [[ -n "${ADMIN_AUTH_USER}" && -n "${ADMIN_AUTH_PASSWORD}" ]]; then + command -v htpasswd >/dev/null 2>&1 || die "Missing: htpasswd (install apache2-utils)" + log "Creating admin-basic-auth secret..." + HTPASSWD="$(htpasswd -nb "${ADMIN_AUTH_USER}" "${ADMIN_AUTH_PASSWORD}")" + kubectl create secret generic admin-basic-auth \ + --namespace="${NAMESPACE}" \ + --from-literal=users="${HTPASSWD}" \ + --dry-run=client -o yaml | kubectl apply -f - +else + warn "admin.basic_auth_user/password not set in config.yaml — skipping admin-basic-auth." + warn "Admin panel will NOT have basic auth protection." +fi + +# --- Done --- + +log "" +log "All secrets created in namespace '${NAMESPACE}'." +log "Verify: kubectl get secrets -n ${NAMESPACE}" diff --git a/deploy-k3s/scripts/03-deploy.sh b/deploy-k3s/scripts/03-deploy.sh new file mode 100755 index 0000000..7cef4b4 --- /dev/null +++ b/deploy-k3s/scripts/03-deploy.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=_config.sh +source "${SCRIPT_DIR}/_config.sh" + +REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)" +NAMESPACE="honeydue" +MANIFESTS="${DEPLOY_DIR}/manifests" + +log() { printf '[deploy] %s\n' "$*"; } +warn() { printf '[deploy][warn] %s\n' "$*" >&2; } +die() { printf '[deploy][error] %s\n' "$*" >&2; exit 1; } + +# --- Parse arguments --- + +SKIP_BUILD=false +DEPLOY_TAG="" + +while (( $# > 0 )); do + case "$1" in + --skip-build) SKIP_BUILD=true; shift ;; + --tag) + [[ -n "${2:-}" ]] || die "--tag requires a value" + DEPLOY_TAG="$2"; shift 2 ;; + -h|--help) + cat <<'EOF' +Usage: ./scripts/03-deploy.sh [OPTIONS] + +Options: + --skip-build Skip Docker build/push, use existing images + --tag Image tag (default: git short SHA) + -h, --help Show this help +EOF + exit 0 ;; + *) die "Unknown argument: $1" ;; + esac +done + +# --- Prerequisites --- + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" +command -v docker >/dev/null 2>&1 || die "Missing: docker" + +if [[ -z "${DEPLOY_TAG}" ]]; then + DEPLOY_TAG="$(git -C "${REPO_DIR}" rev-parse --short HEAD 2>/dev/null || echo "latest")" +fi + +# --- Read registry config --- + +REGISTRY_SERVER="$(cfg_require registry.server "Container registry server")" +REGISTRY_NS="$(cfg_require registry.namespace "Registry namespace")" +REGISTRY_USER="$(cfg_require registry.username "Registry username")" +REGISTRY_TOKEN="$(cfg_require registry.token "Registry token")" + +REGISTRY_PREFIX="${REGISTRY_SERVER%/}/${REGISTRY_NS#/}" +API_IMAGE="${REGISTRY_PREFIX}/honeydue-api:${DEPLOY_TAG}" +WORKER_IMAGE="${REGISTRY_PREFIX}/honeydue-worker:${DEPLOY_TAG}" +ADMIN_IMAGE="${REGISTRY_PREFIX}/honeydue-admin:${DEPLOY_TAG}" + +# --- Build and push --- + +if [[ "${SKIP_BUILD}" == "false" ]]; then + log "Logging in to ${REGISTRY_SERVER}..." + printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_SERVER}" -u "${REGISTRY_USER}" --password-stdin >/dev/null + + log "Building API image: ${API_IMAGE}" + docker build --target api -t "${API_IMAGE}" "${REPO_DIR}" + + log "Building Worker image: ${WORKER_IMAGE}" + docker build --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}" + + log "Building Admin image: ${ADMIN_IMAGE}" + docker build --target admin -t "${ADMIN_IMAGE}" "${REPO_DIR}" + + log "Pushing images..." + docker push "${API_IMAGE}" + docker push "${WORKER_IMAGE}" + docker push "${ADMIN_IMAGE}" + + # Also tag and push :latest + docker tag "${API_IMAGE}" "${REGISTRY_PREFIX}/honeydue-api:latest" + docker tag "${WORKER_IMAGE}" "${REGISTRY_PREFIX}/honeydue-worker:latest" + docker tag "${ADMIN_IMAGE}" "${REGISTRY_PREFIX}/honeydue-admin:latest" + docker push "${REGISTRY_PREFIX}/honeydue-api:latest" + docker push "${REGISTRY_PREFIX}/honeydue-worker:latest" + docker push "${REGISTRY_PREFIX}/honeydue-admin:latest" +else + warn "Skipping build. Using images for tag: ${DEPLOY_TAG}" +fi + +# --- Generate and apply ConfigMap from config.yaml --- + +log "Generating env from config.yaml..." +ENV_FILE="$(mktemp)" +trap 'rm -f "${ENV_FILE}"' EXIT +generate_env > "${ENV_FILE}" + +log "Creating ConfigMap..." +kubectl create configmap honeydue-config \ + --namespace="${NAMESPACE}" \ + --from-env-file="${ENV_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# --- Apply manifests --- + +log "Applying manifests..." + +kubectl apply -f "${MANIFESTS}/namespace.yaml" +kubectl apply -f "${MANIFESTS}/redis/" +kubectl apply -f "${MANIFESTS}/ingress/" + +# Apply deployments with image substitution +sed "s|image: IMAGE_PLACEHOLDER|image: ${API_IMAGE}|" "${MANIFESTS}/api/deployment.yaml" | kubectl apply -f - +kubectl apply -f "${MANIFESTS}/api/service.yaml" +kubectl apply -f "${MANIFESTS}/api/hpa.yaml" + +sed "s|image: IMAGE_PLACEHOLDER|image: ${WORKER_IMAGE}|" "${MANIFESTS}/worker/deployment.yaml" | kubectl apply -f - + +sed "s|image: IMAGE_PLACEHOLDER|image: ${ADMIN_IMAGE}|" "${MANIFESTS}/admin/deployment.yaml" | kubectl apply -f - +kubectl apply -f "${MANIFESTS}/admin/service.yaml" + +# --- Wait for rollouts --- + +log "Waiting for rollouts..." + +kubectl rollout status deployment/redis -n "${NAMESPACE}" --timeout=120s +kubectl rollout status deployment/api -n "${NAMESPACE}" --timeout=300s +kubectl rollout status deployment/worker -n "${NAMESPACE}" --timeout=300s +kubectl rollout status deployment/admin -n "${NAMESPACE}" --timeout=300s + +# --- Done --- + +log "" +log "Deploy completed successfully." +log "Tag: ${DEPLOY_TAG}" +log "Images:" +log " API: ${API_IMAGE}" +log " Worker: ${WORKER_IMAGE}" +log " Admin: ${ADMIN_IMAGE}" +log "" +log "Run ./scripts/04-verify.sh to check cluster health." diff --git a/deploy-k3s/scripts/04-verify.sh b/deploy-k3s/scripts/04-verify.sh new file mode 100755 index 0000000..f7a6288 --- /dev/null +++ b/deploy-k3s/scripts/04-verify.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="honeydue" + +log() { printf '[verify] %s\n' "$*"; } +sep() { printf '\n%s\n' "--- $1 ---"; } +ok() { printf '[verify] ✓ %s\n' "$*"; } +fail() { printf '[verify] ✗ %s\n' "$*"; } + +command -v kubectl >/dev/null 2>&1 || { echo "Missing: kubectl" >&2; exit 1; } + +sep "Nodes" +kubectl get nodes -o wide + +sep "Pods" +kubectl get pods -n "${NAMESPACE}" -o wide + +sep "Services" +kubectl get svc -n "${NAMESPACE}" + +sep "Ingress" +kubectl get ingress -n "${NAMESPACE}" + +sep "HPA" +kubectl get hpa -n "${NAMESPACE}" + +sep "PVCs" +kubectl get pvc -n "${NAMESPACE}" + +sep "Secrets (names only)" +kubectl get secrets -n "${NAMESPACE}" + +sep "ConfigMap keys" +kubectl get configmap honeydue-config -n "${NAMESPACE}" -o jsonpath='{.data}' 2>/dev/null | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + for k in sorted(d.keys()): + v = d[k] + if any(s in k.upper() for s in ['PASSWORD', 'SECRET', 'TOKEN', 'KEY']): + v = '***REDACTED***' + print(f' {k}={v}') +except: + print(' (could not parse)') +" 2>/dev/null || log "ConfigMap not found or not parseable" + +sep "Warning Events (last 15 min)" +kubectl get events -n "${NAMESPACE}" --field-selector type=Warning --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || log "No warning events" + +sep "Pod Restart Counts" +kubectl get pods -n "${NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' 2>/dev/null || true + +sep "In-Cluster Health Check" +API_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" +if [[ -n "${API_POD}" ]]; then + log "Running health check from pod ${API_POD}..." + kubectl exec -n "${NAMESPACE}" "${API_POD}" -- curl -sf http://localhost:8000/api/health/ 2>/dev/null && log "Health check: OK" || log "Health check: FAILED" +else + log "No API pod found — skipping in-cluster health check" +fi + +sep "Resource Usage" +kubectl top pods -n "${NAMESPACE}" 2>/dev/null || log "Metrics server not available (install with: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml)" + +# ============================================================================= +# Security Verification +# ============================================================================= + +sep "Security: Secret Encryption" +# Check that secrets-encryption is configured on the K3s server +if kubectl get nodes -o jsonpath='{.items[0].metadata.name}' >/dev/null 2>&1; then + # Verify secrets are stored encrypted by checking the encryption config exists + if kubectl -n kube-system get cm k3s-config -o yaml 2>/dev/null | grep -q "secrets-encryption"; then + ok "secrets-encryption found in K3s config" + else + # Alternative: check if etcd stores encrypted data + ENCRYPTED_CHECK="$(kubectl get secret honeydue-secrets -n "${NAMESPACE}" -o jsonpath='{.metadata.name}' 2>/dev/null || true)" + if [[ -n "${ENCRYPTED_CHECK}" ]]; then + ok "honeydue-secrets exists (verify encryption with: k3s secrets-encrypt status)" + else + fail "Cannot verify secret encryption — run 'k3s secrets-encrypt status' on the server" + fi + fi +else + fail "Cannot reach cluster to verify secret encryption" +fi + +sep "Security: Network Policies" +NP_COUNT="$(kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')" +if (( NP_COUNT >= 5 )); then + ok "Found ${NP_COUNT} network policies" + kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do + echo " ${line}" + done +else + fail "Expected 5+ network policies, found ${NP_COUNT}" +fi + +sep "Security: Service Accounts" +SA_COUNT="$(kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | grep -cv default | tr -d ' ')" +if (( SA_COUNT >= 4 )); then + ok "Found ${SA_COUNT} custom service accounts (api, worker, admin, redis)" +else + fail "Expected 4 custom service accounts, found ${SA_COUNT}" +fi +kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do + echo " ${line}" +done + +sep "Security: Pod Security Contexts" +PODS_WITHOUT_SECURITY="$(kubectl get pods -n "${NAMESPACE}" -o json 2>/dev/null | python3 -c " +import json, sys +try: + data = json.load(sys.stdin) + issues = [] + for pod in data.get('items', []): + name = pod['metadata']['name'] + spec = pod['spec'] + sc = spec.get('securityContext', {}) + if not sc.get('runAsNonRoot'): + issues.append(f'{name}: missing runAsNonRoot') + for c in spec.get('containers', []): + csc = c.get('securityContext', {}) + if csc.get('allowPrivilegeEscalation', True): + issues.append(f'{name}/{c[\"name\"]}: allowPrivilegeEscalation not false') + if not csc.get('readOnlyRootFilesystem'): + issues.append(f'{name}/{c[\"name\"]}: readOnlyRootFilesystem not true') + if issues: + for i in issues: + print(i) + else: + print('OK') +except Exception as e: + print(f'Error: {e}') +" 2>/dev/null || echo "Error parsing pod specs")" + +if [[ "${PODS_WITHOUT_SECURITY}" == "OK" ]]; then + ok "All pods have proper security contexts" +else + fail "Pod security context issues:" + echo "${PODS_WITHOUT_SECURITY}" | while read -r line; do + echo " ${line}" + done +fi + +sep "Security: Pod Disruption Budgets" +PDB_COUNT="$(kubectl get pdb -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')" +if (( PDB_COUNT >= 2 )); then + ok "Found ${PDB_COUNT} pod disruption budgets" +else + fail "Expected 2+ PDBs, found ${PDB_COUNT}" +fi +kubectl get pdb -n "${NAMESPACE}" 2>/dev/null || true + +sep "Security: Cloudflare-Only Middleware" +CF_MIDDLEWARE="$(kubectl get middleware cloudflare-only -n "${NAMESPACE}" -o name 2>/dev/null || true)" +if [[ -n "${CF_MIDDLEWARE}" ]]; then + ok "cloudflare-only middleware exists" + # Check ingress annotations reference it + INGRESS_ANNOTATIONS="$(kubectl get ingress -n "${NAMESPACE}" -o jsonpath='{.items[*].metadata.annotations.traefik\.ingress\.kubernetes\.io/router\.middlewares}' 2>/dev/null || true)" + if echo "${INGRESS_ANNOTATIONS}" | grep -q "cloudflare-only"; then + ok "Ingress references cloudflare-only middleware" + else + fail "Ingress does NOT reference cloudflare-only middleware" + fi +else + fail "cloudflare-only middleware not found" +fi + +sep "Security: Admin Basic Auth" +ADMIN_AUTH="$(kubectl get secret admin-basic-auth -n "${NAMESPACE}" -o name 2>/dev/null || true)" +if [[ -n "${ADMIN_AUTH}" ]]; then + ok "admin-basic-auth secret exists" +else + fail "admin-basic-auth secret not found — admin panel has no additional auth layer" +fi + +echo "" +log "Verification complete." diff --git a/deploy-k3s/scripts/_config.sh b/deploy-k3s/scripts/_config.sh new file mode 100755 index 0000000..cb74f99 --- /dev/null +++ b/deploy-k3s/scripts/_config.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# Shared config helper — sourced by all deploy scripts. +# Provides cfg() to read values from config.yaml. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_FILE="${DEPLOY_DIR}/config.yaml" + +if [[ ! -f "${CONFIG_FILE}" ]]; then + if [[ -f "${CONFIG_FILE}.example" ]]; then + echo "[error] config.yaml not found. Run: cp config.yaml.example config.yaml" >&2 + else + echo "[error] config.yaml not found." >&2 + fi + exit 1 +fi + +# cfg "dotted.key.path" — reads a value from config.yaml +# Examples: cfg database.host, cfg nodes.0.ip, cfg features.push_enabled +cfg() { + python3 -c " +import yaml, json, sys +with open(sys.argv[1]) as f: + c = yaml.safe_load(f) +keys = sys.argv[2].split('.') +v = c +for k in keys: + if isinstance(v, list): + v = v[int(k)] + else: + v = v[k] +if isinstance(v, bool): + print(str(v).lower()) +elif isinstance(v, (dict, list)): + print(json.dumps(v)) +else: + print('' if v is None else v) +" "${CONFIG_FILE}" "$1" 2>/dev/null +} + +# cfg_require "key" "label" — reads value and dies if empty +cfg_require() { + local val + val="$(cfg "$1")" + if [[ -z "${val}" ]]; then + echo "[error] Missing required config: $1 ($2)" >&2 + exit 1 + fi + printf '%s' "${val}" +} + +# node_count — returns number of nodes +node_count() { + python3 -c " +import yaml +with open('${CONFIG_FILE}') as f: + c = yaml.safe_load(f) +print(len(c.get('nodes', []))) +" +} + +# nodes_with_role "role" — returns node names with a given role +nodes_with_role() { + python3 -c " +import yaml +with open('${CONFIG_FILE}') as f: + c = yaml.safe_load(f) +for n in c.get('nodes', []): + if '$1' in n.get('roles', []): + print(n['name']) +" +} + +# generate_env — writes the flat env file the app expects to stdout +generate_env() { + python3 -c " +import yaml + +with open('${CONFIG_FILE}') as f: + c = yaml.safe_load(f) + +d = c['domains'] +db = c['database'] +em = c['email'] +ps = c['push'] +st = c['storage'] +wk = c['worker'] +ft = c['features'] +aa = c.get('apple_auth', {}) +ga = c.get('google_auth', {}) +rd = c.get('redis', {}) + +def b(v): + return str(v).lower() if isinstance(v, bool) else str(v) + +def val(v): + return '' if v is None else str(v) + +lines = [ + # API + 'DEBUG=false', + f\"ALLOWED_HOSTS={d['api']},{d['base']}\", + f\"CORS_ALLOWED_ORIGINS=https://{d['base']},https://{d['admin']}\", + 'TIMEZONE=UTC', + f\"BASE_URL=https://{d['base']}\", + 'PORT=8000', + # Admin + f\"NEXT_PUBLIC_API_URL=https://{d['api']}\", + f\"ADMIN_PANEL_URL=https://{d['admin']}\", + # Database + f\"DB_HOST={val(db['host'])}\", + f\"DB_PORT={db['port']}\", + f\"POSTGRES_USER={val(db['user'])}\", + f\"POSTGRES_DB={db['name']}\", + f\"DB_SSLMODE={db['sslmode']}\", + f\"DB_MAX_OPEN_CONNS={db['max_open_conns']}\", + f\"DB_MAX_IDLE_CONNS={db['max_idle_conns']}\", + f\"DB_MAX_LIFETIME={db['max_lifetime']}\", + # Redis (K8s internal DNS — password injected if configured) + f\"REDIS_URL=redis://{':%s@' % val(rd.get('password')) if rd.get('password') else ''}redis.honeydue.svc.cluster.local:6379/0\", + 'REDIS_DB=0', + # Email + f\"EMAIL_HOST={em['host']}\", + f\"EMAIL_PORT={em['port']}\", + f\"EMAIL_USE_TLS={b(em['use_tls'])}\", + f\"EMAIL_HOST_USER={val(em['user'])}\", + f\"DEFAULT_FROM_EMAIL={val(em['from'])}\", + # Push + 'APNS_AUTH_KEY_PATH=/secrets/apns/apns_auth_key.p8', + f\"APNS_AUTH_KEY_ID={val(ps['apns_key_id'])}\", + f\"APNS_TEAM_ID={val(ps['apns_team_id'])}\", + f\"APNS_TOPIC={ps['apns_topic']}\", + f\"APNS_USE_SANDBOX={b(ps['apns_use_sandbox'])}\", + f\"APNS_PRODUCTION={b(ps['apns_production'])}\", + # Worker + f\"TASK_REMINDER_HOUR={wk['task_reminder_hour']}\", + f\"OVERDUE_REMINDER_HOUR={wk['overdue_reminder_hour']}\", + f\"DAILY_DIGEST_HOUR={wk['daily_digest_hour']}\", + # B2 Storage + f\"B2_KEY_ID={val(st['b2_key_id'])}\", + f\"B2_APP_KEY={val(st['b2_app_key'])}\", + f\"B2_BUCKET_NAME={val(st['b2_bucket'])}\", + f\"B2_ENDPOINT={val(st['b2_endpoint'])}\", + f\"STORAGE_MAX_FILE_SIZE={st['max_file_size']}\", + f\"STORAGE_ALLOWED_TYPES={st['allowed_types']}\", + # Features + f\"FEATURE_PUSH_ENABLED={b(ft['push_enabled'])}\", + f\"FEATURE_EMAIL_ENABLED={b(ft['email_enabled'])}\", + f\"FEATURE_WEBHOOKS_ENABLED={b(ft['webhooks_enabled'])}\", + f\"FEATURE_ONBOARDING_EMAILS_ENABLED={b(ft['onboarding_emails_enabled'])}\", + f\"FEATURE_PDF_REPORTS_ENABLED={b(ft['pdf_reports_enabled'])}\", + f\"FEATURE_WORKER_ENABLED={b(ft['worker_enabled'])}\", + # Apple auth/IAP + f\"APPLE_CLIENT_ID={val(aa.get('client_id'))}\", + f\"APPLE_TEAM_ID={val(aa.get('team_id'))}\", + f\"APPLE_IAP_KEY_ID={val(aa.get('iap_key_id'))}\", + f\"APPLE_IAP_ISSUER_ID={val(aa.get('iap_issuer_id'))}\", + f\"APPLE_IAP_BUNDLE_ID={val(aa.get('iap_bundle_id'))}\", + f\"APPLE_IAP_KEY_PATH={val(aa.get('iap_key_path'))}\", + f\"APPLE_IAP_SANDBOX={b(aa.get('iap_sandbox', False))}\", + # Google auth/IAP + f\"GOOGLE_CLIENT_ID={val(ga.get('client_id'))}\", + f\"GOOGLE_ANDROID_CLIENT_ID={val(ga.get('android_client_id'))}\", + f\"GOOGLE_IOS_CLIENT_ID={val(ga.get('ios_client_id'))}\", + f\"GOOGLE_IAP_PACKAGE_NAME={val(ga.get('iap_package_name'))}\", + f\"GOOGLE_IAP_SERVICE_ACCOUNT_PATH={val(ga.get('iap_service_account_path'))}\", +] + +print('\n'.join(lines)) +" +} + +# generate_cluster_config — writes hetzner-k3s YAML to stdout +generate_cluster_config() { + python3 -c " +import yaml + +with open('${CONFIG_FILE}') as f: + c = yaml.safe_load(f) + +cl = c['cluster'] + +config = { + 'cluster_name': 'honeydue', + 'kubeconfig_path': './kubeconfig', + 'k3s_version': cl['k3s_version'], + 'networking': { + 'ssh': { + 'port': 22, + 'use_agent': False, + 'public_key_path': cl['ssh_public_key'], + 'private_key_path': cl['ssh_private_key'], + }, + 'allowed_networks': { + 'ssh': ['0.0.0.0/0'], + 'api': ['0.0.0.0/0'], + }, + }, + 'api_server_hostname': '', + 'schedule_workloads_on_masters': True, + 'masters_pool': { + 'instance_type': cl['instance_type'], + 'instance_count': len(c.get('nodes', [])), + 'location': cl['location'], + 'image': 'ubuntu-24.04', + }, + 'additional_packages': ['open-iscsi'], + 'post_create_commands': ['sudo systemctl enable --now iscsid'], + 'k3s_config_file': 'secrets-encryption: true\n', +} + +print(yaml.dump(config, default_flow_style=False, sort_keys=False)) +" +} diff --git a/deploy-k3s/scripts/rollback.sh b/deploy-k3s/scripts/rollback.sh new file mode 100755 index 0000000..4d3f7ab --- /dev/null +++ b/deploy-k3s/scripts/rollback.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="honeydue" + +log() { printf '[rollback] %s\n' "$*"; } +die() { printf '[rollback][error] %s\n' "$*" >&2; exit 1; } + +command -v kubectl >/dev/null 2>&1 || die "Missing: kubectl" + +DEPLOYMENTS=("api" "worker" "admin") + +# --- Show current state --- + +echo "=== Current Rollout History ===" +for deploy in "${DEPLOYMENTS[@]}"; do + echo "" + echo "--- ${deploy} ---" + kubectl rollout history deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || echo " (not found)" +done + +echo "" +echo "=== Current Images ===" +for deploy in "${DEPLOYMENTS[@]}"; do + IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")" + echo " ${deploy}: ${IMAGE}" +done + +# --- Confirm --- + +echo "" +read -rp "Roll back all deployments to previous revision? [y/N] " confirm +if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then + log "Aborted." + exit 0 +fi + +# --- Rollback --- + +for deploy in "${DEPLOYMENTS[@]}"; do + log "Rolling back ${deploy}..." + kubectl rollout undo deployment/"${deploy}" -n "${NAMESPACE}" 2>/dev/null || log "Skipping ${deploy} (not found or no previous revision)" +done + +# --- Wait --- + +log "Waiting for rollouts..." +for deploy in "${DEPLOYMENTS[@]}"; do + kubectl rollout status deployment/"${deploy}" -n "${NAMESPACE}" --timeout=300s 2>/dev/null || true +done + +# --- Verify --- + +echo "" +echo "=== Post-Rollback Images ===" +for deploy in "${DEPLOYMENTS[@]}"; do + IMAGE="$(kubectl get deployment "${deploy}" -n "${NAMESPACE}" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || echo "n/a")" + echo " ${deploy}: ${IMAGE}" +done + +log "Rollback complete. Run ./scripts/04-verify.sh to check health." diff --git a/deploy-k3s/secrets/README.md b/deploy-k3s/secrets/README.md new file mode 100644 index 0000000..f2c6f5d --- /dev/null +++ b/deploy-k3s/secrets/README.md @@ -0,0 +1,19 @@ +# Secrets Directory + +Create these files before running `scripts/02-setup-secrets.sh`: + +| File | Purpose | +|------|---------| +| `postgres_password.txt` | Neon PostgreSQL password | +| `secret_key.txt` | App signing secret (minimum 32 characters) | +| `email_host_password.txt` | SMTP password (Fastmail app password) | +| `fcm_server_key.txt` | Firebase Cloud Messaging server key | +| `apns_auth_key.p8` | Apple Push Notification private key | +| `cloudflare-origin.crt` | Cloudflare origin certificate (PEM) | +| `cloudflare-origin.key` | Cloudflare origin certificate key (PEM) | + +The first five files are the same format as the Docker Swarm `deploy/secrets/` directory. +The Cloudflare files are new for K3s (TLS termination at the ingress). + +All string config (database host, registry token, etc.) goes in `config.yaml` instead. +These files are gitignored and should never be committed.