From d3708e6c72c256b81bf5c40be195f8833043d0e0 Mon Sep 17 00:00:00 2001
From: Trey t <treytartt@fastmail.com>
Date: Sat, 25 Apr 2026 14:42:15 -0500
Subject: [PATCH] Fix /metrics double-gzip + deploy script for amd64 build

The Echo gzip middleware was wrapping promhttp's pre-gzipped output, so
vmagent received double-compressed bytes that failed the Prometheus
parser with binary garbage. Skipping /metrics in the gzip Skipper.

Three deploy-script fixes uncovered while shipping this:
- _config.sh had backticks around \"kubectl get cm\" inside the python
  heredoc, which bash treated as command substitution when KUBECONFIG
  was set. Quoted the literal instead.
- 03-deploy.sh now passes --platform linux/amd64 to all docker builds
  so arm64 Macs don't push images that fail with \"exec format error\"
  on the Hetzner CX nodes.
- OBS_INGEST_TOKEN lookup was reading deploy-k3s/prod.env instead of
  the actual deploy/prod.env at the repo root.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 deploy-k3s/scripts/03-deploy.sh | 24 +++++++++++++++---------
 deploy-k3s/scripts/_config.sh   |  2 +-
 internal/router/router.go       |  7 +++++--
 3 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/deploy-k3s/scripts/03-deploy.sh b/deploy-k3s/scripts/03-deploy.sh
index 0f03e2e..47313c5 100755
--- a/deploy-k3s/scripts/03-deploy.sh
+++ b/deploy-k3s/scripts/03-deploy.sh
@@ -81,20 +81,24 @@ if [[ "${SKIP_BUILD}" == "false" ]]; then
   log "Logging in to ${REGISTRY_SERVER}..."
   printf '%s' "${REGISTRY_TOKEN}" | docker login "${REGISTRY_SERVER}" -u "${REGISTRY_USER}" --password-stdin >/dev/null
 
-  log "Building API image: ${API_IMAGE}"
-  docker build --target api -t "${API_IMAGE}" "${REPO_DIR}"
+  # k3s nodes are linux/amd64 (Hetzner CX). Force the build platform so
+  # local arm64 Macs don't push images that crash with "exec format error".
+  BUILD_PLATFORM="linux/amd64"
 
-  log "Building Worker image: ${WORKER_IMAGE}"
-  docker build --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}"
+  log "Building API image: ${API_IMAGE} (${BUILD_PLATFORM})"
+  docker build --platform "${BUILD_PLATFORM}" --target api -t "${API_IMAGE}" "${REPO_DIR}"
 
-  log "Building Admin image: ${ADMIN_IMAGE} (NEXT_PUBLIC_API_URL=${ADMIN_API_URL})"
-  docker build --target admin \
+  log "Building Worker image: ${WORKER_IMAGE} (${BUILD_PLATFORM})"
+  docker build --platform "${BUILD_PLATFORM}" --target worker -t "${WORKER_IMAGE}" "${REPO_DIR}"
+
+  log "Building Admin image: ${ADMIN_IMAGE} (${BUILD_PLATFORM}, NEXT_PUBLIC_API_URL=${ADMIN_API_URL})"
+  docker build --platform "${BUILD_PLATFORM}" --target admin \
     --build-arg "NEXT_PUBLIC_API_URL=${ADMIN_API_URL}" \
     -t "${ADMIN_IMAGE}" "${REPO_DIR}"
 
   if [[ -n "${WEB_REPO_DIR}" && -f "${WEB_REPO_DIR}/Dockerfile" ]]; then
-    log "Building Web image: ${WEB_IMAGE} (NEXT_PUBLIC_API_URL=${WEB_API_URL})"
-    docker build \
+    log "Building Web image: ${WEB_IMAGE} (${BUILD_PLATFORM}, NEXT_PUBLIC_API_URL=${WEB_API_URL})"
+    docker build --platform "${BUILD_PLATFORM}" \
       --build-arg "NEXT_PUBLIC_API_URL=${WEB_API_URL}" \
       --build-arg "NEXT_PUBLIC_POSTHOG_KEY=${NEXT_PUBLIC_POSTHOG_KEY}" \
       --build-arg "NEXT_PUBLIC_POSTHOG_HOST=${NEXT_PUBLIC_POSTHOG_HOST}" \
@@ -164,7 +168,9 @@ fi
 # to obs.88oakapps.com. The bearer token comes from deploy/prod.env so it
 # stays out of the repo; the manifest holds TOKEN_PLACEHOLDER.
 if [[ -d "${MANIFESTS}/observability" ]]; then
-  OBS_TOKEN="$(grep -E '^OBS_INGEST_TOKEN=' "${DEPLOY_DIR}/prod.env" 2>/dev/null | cut -d= -f2- || true)"
+  # prod.env lives at the repo's deploy/ dir (sibling of deploy-k3s/), not
+  # under deploy-k3s/. It's gitignored — operator copies values there once.
+  OBS_TOKEN="$(grep -E '^OBS_INGEST_TOKEN=' "${REPO_DIR}/deploy/prod.env" 2>/dev/null | cut -d= -f2- || true)"
   if [[ -z "${OBS_TOKEN}" ]]; then
     warn "OBS_INGEST_TOKEN not found in deploy/prod.env — skipping vmagent apply"
   else
diff --git a/deploy-k3s/scripts/_config.sh b/deploy-k3s/scripts/_config.sh
index 4bcf4df..4d125d4 100755
--- a/deploy-k3s/scripts/_config.sh
+++ b/deploy-k3s/scripts/_config.sh
@@ -145,7 +145,7 @@ lines = [
     # (set by 02-setup-secrets.sh). Wire them into the api/worker
     # deployments via envFrom: secretRef when B2 uploads need to be
     # active. Leaving them in cleartext here would leak via
-    # `kubectl get cm`.
+    # \"kubectl get cm\".
     f\"B2_BUCKET_NAME={val(st['b2_bucket'])}\",
     f\"B2_ENDPOINT={val(st['b2_endpoint'])}\",
     f\"B2_REGION={val(st.get('b2_region'))}\",
diff --git a/internal/router/router.go b/internal/router/router.go
index dbaf8aa..b6d9be5 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -107,11 +107,14 @@ func SetupRouter(deps *Dependencies) *echo.Echo {
 	e.Use(corsMiddleware(cfg))
 	e.Use(i18n.Middleware())
 
-	// Gzip compression (skip media endpoints since they serve binary files)
+	// Gzip compression (skip media endpoints since they serve binary files;
+	// also skip /metrics because promhttp does its own content negotiation,
+	// so wrapping it here produces double-gzipped output that breaks scrapers).
 	e.Use(middleware.GzipWithConfig(middleware.GzipConfig{
 		Level: 5,
 		Skipper: func(c echo.Context) bool {
-			return strings.HasPrefix(c.Request().URL.Path, "/api/media/")
+			path := c.Request().URL.Path
+			return strings.HasPrefix(path, "/api/media/") || path == "/metrics"
 		},
 	}))