Add Prometheus metrics + vmagent push to obs.88oakapps.com

Adds internal/prom package with histograms for HTTP, GORM, B2, APNs, and
FCM, wired into the Echo router (HTTPMiddleware + /metrics) and GORM via
statement-level callbacks (no ctx plumbing needed). Storage and push
clients call ObserveB2Upload / ObserveAPNsSend / ObserveFCMSend at the
network round-trip points.

Existing internal/monitoring metrics move to /metrics/legacy so the
canonical /metrics emits proper histogram buckets for p50/p95/p99 rollups.

deploy-k3s/manifests/observability/vmagent.yaml deploys a single-replica
vmagent in the honeydue namespace that scrapes api Pods on :8000/metrics
every 15s and remote-writes to https://obs.88oakapps.com/api/v1/write
with a bearer token (substituted at deploy time from OBS_INGEST_TOKEN
in deploy/prod.env). NetworkPolicies allow vmagent egress to api Pods
and to the public obs endpoint over :443; the obs side runs
VictoriaMetrics + Jaeger + Grafana on 88oakappsUpdate.

docs/observability-plan.md captures the full plan including resource
budget, instrumentation table, 4-step rollout, and migration triggers.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

internal/router/router.go

@@ -23,6 +23,7 @@ import (
 	"github.com/treytartt/honeydue-api/internal/i18n"
 	custommiddleware "github.com/treytartt/honeydue-api/internal/middleware"
 	"github.com/treytartt/honeydue-api/internal/monitoring"
+	"github.com/treytartt/honeydue-api/internal/prom"
 	"github.com/treytartt/honeydue-api/internal/push"
 	"github.com/treytartt/honeydue-api/internal/repositories"
 	"github.com/treytartt/honeydue-api/internal/services"
@@ -121,6 +122,15 @@ func SetupRouter(deps *Dependencies) *echo.Echo {
 		}
 	}
 
+	// Prometheus metrics middleware — feeds VictoriaMetrics on
+	// obs.88oakapps.com via vmagent. Records http_request_duration_seconds
+	// labeled by route pattern, method, and status code.
+	e.Use(prom.HTTPMiddleware())
+
+	// /metrics endpoint exposed for vmagent scrape. No auth — bound to
+	// the cluster network only; not exposed via Cloudflare.
+	e.GET("/metrics", prom.Handler())
+
 	// Serve landing page static files (if static directory is configured)
 	staticDir := cfg.Server.StaticDir
 	if staticDir != "" {
@@ -229,9 +239,11 @@ func SetupRouter(deps *Dependencies) *echo.Echo {
 		mediaHandler = handlers.NewMediaHandler(documentRepo, taskRepo, residenceRepo, deps.StorageService)
 	}
 
-	// Prometheus metrics endpoint (no auth required, for scraping)
+	// Legacy Prometheus-shaped metrics from internal/monitoring (consumed by
+	// GoAdmin dashboard). Now lives at /metrics/legacy so the canonical /metrics
+	// route (registered above) emits proper Prometheus histograms with labels.
 	if deps.MonitoringService != nil {
-		e.GET("/metrics", prometheusMetrics(deps.MonitoringService))
+		e.GET("/metrics/legacy", prometheusMetrics(deps.MonitoringService))
 	}
 
 	// Set up admin routes with monitoring handler (if available)