Add Prometheus metrics + vmagent push to obs.88oakapps.com

Adds internal/prom package with histograms for HTTP, GORM, B2, APNs, and
FCM, wired into the Echo router (HTTPMiddleware + /metrics) and GORM via
statement-level callbacks (no ctx plumbing needed). Storage and push
clients call ObserveB2Upload / ObserveAPNsSend / ObserveFCMSend at the
network round-trip points.

Existing internal/monitoring metrics move to /metrics/legacy so the
canonical /metrics emits proper histogram buckets for p50/p95/p99 rollups.

deploy-k3s/manifests/observability/vmagent.yaml deploys a single-replica
vmagent in the honeydue namespace that scrapes api Pods on :8000/metrics
every 15s and remote-writes to https://obs.88oakapps.com/api/v1/write
with a bearer token (substituted at deploy time from OBS_INGEST_TOKEN
in deploy/prod.env). NetworkPolicies allow vmagent egress to api Pods
and to the public obs endpoint over :443; the obs side runs
VictoriaMetrics + Jaeger + Grafana on 88oakappsUpdate.

docs/observability-plan.md captures the full plan including resource
budget, instrumentation table, 4-step rollout, and migration triggers.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

internal/push/apns.go

@@ -3,6 +3,7 @@ package push
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/rs/zerolog/log"
 	"github.com/sideshow/apns2"
@@ -10,6 +11,7 @@ import (
 	"github.com/sideshow/apns2/token"
 
 	"github.com/treytartt/honeydue-api/internal/config"
+	"github.com/treytartt/honeydue-api/internal/prom"
 )
 
 // APNsClient handles direct communication with Apple Push Notification service
@@ -84,8 +86,10 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 			Priority:    apns2.PriorityHigh,
 		}
 
+		sendStart := time.Now()
 		res, err := c.client.PushWithContext(ctx, notification)
 		if err != nil {
+			prom.ObserveAPNsSend("error", time.Since(sendStart))
 			log.Error().
 				Err(err).
 				Str("token", truncateToken(deviceToken)).
@@ -95,6 +99,7 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 		}
 
 		if !res.Sent() {
+			prom.ObserveAPNsSend("bad_token", time.Since(sendStart))
 			log.Error().
 				Str("token", truncateToken(deviceToken)).
 				Str("reason", res.Reason).
@@ -104,6 +109,7 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 			continue
 		}
 
+		prom.ObserveAPNsSend("ok", time.Since(sendStart))
 		successCount++
 		log.Debug().
 			Str("token", truncateToken(deviceToken)).
@@ -154,8 +160,10 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 			Priority:    apns2.PriorityHigh,
 		}
 
+		sendStart := time.Now()
 		res, err := c.client.PushWithContext(ctx, notification)
 		if err != nil {
+			prom.ObserveAPNsSend("error", time.Since(sendStart))
 			log.Error().
 				Err(err).
 				Str("token", truncateToken(deviceToken)).
@@ -166,6 +174,7 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 		}
 
 		if !res.Sent() {
+			prom.ObserveAPNsSend("bad_token", time.Since(sendStart))
 			log.Error().
 				Str("token", truncateToken(deviceToken)).
 				Str("reason", res.Reason).
@@ -176,6 +185,7 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 			continue
 		}
 
+		prom.ObserveAPNsSend("ok", time.Since(sendStart))
 		successCount++
 		log.Debug().
 			Str("token", truncateToken(deviceToken)).

internal/push/fcm.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -14,6 +15,7 @@ import (
 	"golang.org/x/oauth2/google"
 
 	"github.com/treytartt/honeydue-api/internal/config"
+	"github.com/treytartt/honeydue-api/internal/prom"
 )
 
 const (
@@ -213,8 +215,15 @@ func (c *FCMClient) Send(ctx context.Context, tokens []string, title, message st
 	successCount := 0
 
 	for _, token := range tokens {
+		sendStart := time.Now()
 		err := c.sendOne(ctx, token, title, message, data)
 		if err != nil {
+			result := "error"
+			var fcmErr *FCMSendError
+			if errors.As(err, &fcmErr) && fcmErr.IsUnregistered() {
+				result = "bad_token"
+			}
+			prom.ObserveFCMSend(result, time.Since(sendStart))
 			log.Error().
 				Err(err).
 				Str("token", truncateToken(token)).
@@ -223,6 +232,7 @@ func (c *FCMClient) Send(ctx context.Context, tokens []string, title, message st
 			continue
 		}
 
+		prom.ObserveFCMSend("ok", time.Since(sendStart))
 		successCount++
 		log.Debug().
 			Str("token", truncateToken(token)).