Wire OpenTelemetry tracing — HTTP, B2, APNs, FCM, asynq, GORM (partial)

Step 1 — OTel SDK: cmd/api and cmd/worker initialize a tracer provider
that exports OTLP/HTTP to obs.88oakapps.com (Jaeger all-in-one). Sampling
is AlwaysSample in dev (DEBUG=true) and TraceIDRatioBased(0.1) in prod,
overridable via OTEL_TRACES_SAMPLER_ARG. Service names are honeydue-api
and honeydue-worker. otelecho.Middleware opens a span per HTTP request.

Step 2 — Manual spans: storage_service.Upload now takes ctx and emits
storage.upload + b2.PutObject spans (size_bytes, key, mime_type, bucket,
result attrs). APNs Send/SendWithCategory and FCM sendOne emit per-token
spans with topic, status_code, reason. Asynq middleware emits
asynq.handle:<task_type> per job with retry/payload attrs and records
asynq_job_duration_seconds.

Step 3 — Database: otelgorm plugin registered in database.Connect, so
any SQL emitted via db.WithContext(ctx) attaches to the request span.
Every repository now exposes WithContext(ctx) *XRepository as the
migration helper. TaskService.ListTasks and GetTasksByResidence are
migrated end-to-end (ctx threaded through handler → service → repo);
remaining services adopt the same pattern incrementally — pre-migration
methods still emit untraced SQL via the unchanged db field.

OBS_TRACES_URL and OBS_INGEST_TOKEN flow from deploy/prod.env →
honeydue-secrets → api+worker Deployments via secretKeyRef (optional).
02-setup-secrets.sh sources them from prod.env on next run; manifests
mark both env vars optional so the deployment rolls without traces if
the secret is absent.

ch15 observability doc now lists what produces spans today vs the
remaining migration work, with the explicit per-method pattern.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

cmd/api/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/treytartt/honeydue-api/internal/push"
 	"github.com/treytartt/honeydue-api/internal/router"
 	"github.com/treytartt/honeydue-api/internal/services"
+	"github.com/treytartt/honeydue-api/internal/tracing"
 	"github.com/treytartt/honeydue-api/pkg/utils"
 )
 
@@ -50,6 +51,27 @@ func main() {
 		Str("redis_url", config.MaskURLCredentials(cfg.Redis.URL)).
 		Msg("Starting HoneyDue API server")
 
+	// Initialize OpenTelemetry tracing — exports to obs.88oakapps.com
+	// (Jaeger via OTLP/HTTP) when OBS_TRACES_URL is set; otherwise installs
+	// a no-op tracer so call sites can use otel.Tracer() unconditionally.
+	tracingShutdown, err := tracing.Init(context.Background(), tracing.Config{
+		ServiceName: "honeydue-api",
+		Environment: deploymentEnvironment(cfg.Server.Debug),
+		EndpointURL: os.Getenv("OBS_TRACES_URL"),
+		BearerToken: os.Getenv("OBS_INGEST_TOKEN"),
+		SampleRatio: tracing.SampleRatioFromEnv(),
+	})
+	if err != nil {
+		log.Error().Err(err).Msg("tracing init failed — continuing without traces")
+	}
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := tracingShutdown(shutdownCtx); err != nil {
+			log.Warn().Err(err).Msg("tracing shutdown error")
+		}
+	}()
+
 	// Connect to database (retry with backoff)
 	var db *gorm.DB
 	var dbErr error
@@ -217,3 +239,15 @@ func main() {
 
 	log.Info().Msg("Server exited")
 }
+
+// deploymentEnvironment turns the boolean Debug flag into the conventional
+// environment label spans get tagged with.
+func deploymentEnvironment(debug bool) string {
+	if env := os.Getenv("DEPLOYMENT_ENVIRONMENT"); env != "" {
+		return env
+	}
+	if debug {
+		return "dev"
+	}
+	return "prod"
+}

cmd/worker/main.go

@@ -11,13 +11,18 @@ import (
 	"github.com/hibiken/asynq"
 	"github.com/redis/go-redis/v9"
 	"github.com/rs/zerolog/log"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/treytartt/honeydue-api/internal/config"
 	"github.com/treytartt/honeydue-api/internal/database"
 	"github.com/treytartt/honeydue-api/internal/monitoring"
+	"github.com/treytartt/honeydue-api/internal/prom"
 	"github.com/treytartt/honeydue-api/internal/push"
 	"github.com/treytartt/honeydue-api/internal/repositories"
 	"github.com/treytartt/honeydue-api/internal/services"
+	"github.com/treytartt/honeydue-api/internal/tracing"
 	"github.com/treytartt/honeydue-api/internal/worker/jobs"
 	"github.com/treytartt/honeydue-api/pkg/utils"
 )
@@ -40,6 +45,27 @@ func main() {
 		os.Exit(0)
 	}
 
+	// Initialize OpenTelemetry tracing for the worker process. Same OTLP
+	// destination as the api; service.name distinguishes them in Jaeger.
+	tracingShutdown, err := tracing.Init(context.Background(), tracing.Config{
+		ServiceName: "honeydue-worker",
+		Environment: workerDeploymentEnv(cfg.Server.Debug),
+		EndpointURL: os.Getenv("OBS_TRACES_URL"),
+		BearerToken: os.Getenv("OBS_INGEST_TOKEN"),
+		SampleRatio: tracing.SampleRatioFromEnv(),
+	})
+	if err != nil {
+		log.Error().Err(err).Msg("worker tracing init failed — continuing without traces")
+	}
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := tracingShutdown(shutdownCtx); err != nil {
+			log.Warn().Err(err).Msg("worker tracing shutdown error")
+		}
+	}()
+	asynqTracer := tracing.Tracer("honeydue/worker/asynq")
+
 	// Initialize database
 	db, err := database.Connect(&cfg.Database, cfg.Server.Debug)
 	if err != nil {
@@ -143,6 +169,11 @@ func main() {
 
 	// Create Asynq mux and register handlers
 	mux := asynq.NewServeMux()
+
+	// Tracing + metrics middleware: every job runs inside a span and emits
+	// asynq_job_duration_seconds{task_type,result}.
+	mux.Use(asynqTracingMiddleware(asynqTracer))
+
 	mux.HandleFunc(jobs.TypeSmartReminder, jobHandler.HandleSmartReminder)
 	mux.HandleFunc(jobs.TypeDailyDigest, jobHandler.HandleDailyDigest)
 	mux.HandleFunc(jobs.TypeSendEmail, jobHandler.HandleSendEmail)
@@ -238,3 +269,44 @@ func main() {
 
 	log.Info().Msg("Worker stopped")
 }
+
+// asynqTracingMiddleware returns an asynq.MiddlewareFunc that opens a span
+// per task execution and records asynq_job_duration_seconds. Span attrs
+// include task type, queue, retry count, and the result outcome.
+func asynqTracingMiddleware(tracer trace.Tracer) asynq.MiddlewareFunc {
+	return func(next asynq.Handler) asynq.Handler {
+		return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
+			ctx, span := tracer.Start(ctx, "asynq.handle:"+t.Type(),
+				trace.WithAttributes(
+					attribute.String("asynq.task_type", t.Type()),
+					attribute.Int("asynq.payload_bytes", len(t.Payload())),
+				),
+			)
+			defer span.End()
+
+			start := time.Now()
+			err := next.ProcessTask(ctx, t)
+			dur := time.Since(start)
+			result := "ok"
+			if err != nil {
+				result = "error"
+				span.SetStatus(codes.Error, err.Error())
+				span.RecordError(err)
+			}
+			span.SetAttributes(attribute.String("asynq.result", result))
+			prom.ObserveAsynqJob(t.Type(), result, dur)
+			return err
+		})
+	}
+}
+
+// workerDeploymentEnv mirrors deploymentEnvironment in cmd/api/main.go.
+func workerDeploymentEnv(debug bool) string {
+	if env := os.Getenv("DEPLOYMENT_ENVIRONMENT"); env != "" {
+		return env
+	}
+	if debug {
+		return "dev"
+	}
+	return "prod"
+}

deploy-k3s/manifests/api/deployment.yaml

@@ -88,6 +88,22 @@ spec:
                 secretKeyRef:
                   name: honeydue-secrets
                   key: B2_APP_KEY
+            # Observability — push traces (and any future OTLP metrics) to
+            # obs.88oakapps.com. Token gates ingest at nginx; URL is the
+            # same one vmagent uses for metric remote-write. Both come from
+            # honeydue-secrets so they aren't world-readable in ConfigMap.
+            - name: OBS_TRACES_URL
+              valueFrom:
+                secretKeyRef:
+                  name: honeydue-secrets
+                  key: OBS_TRACES_URL
+                  optional: true
+            - name: OBS_INGEST_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: honeydue-secrets
+                  key: OBS_INGEST_TOKEN
+                  optional: true
           volumeMounts:
             - name: apns-key
               mountPath: /secrets/apns

deploy-k3s/manifests/worker/deployment.yaml

@@ -74,6 +74,21 @@ spec:
                   name: honeydue-secrets
                   key: REDIS_PASSWORD
                   optional: true
+            # Observability — workers emit traces (e.g., asynq job spans) to
+            # obs.88oakapps.com over OTLP/HTTP. service.name=honeydue-worker
+            # so api and worker show up as separate services in Jaeger.
+            - name: OBS_TRACES_URL
+              valueFrom:
+                secretKeyRef:
+                  name: honeydue-secrets
+                  key: OBS_TRACES_URL
+                  optional: true
+            - name: OBS_INGEST_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: honeydue-secrets
+                  key: OBS_INGEST_TOKEN
+                  optional: true
           volumeMounts:
             - name: apns-key
               mountPath: /secrets/apns

deploy-k3s/scripts/02-setup-secrets.sh

@@ -70,6 +70,24 @@ if [[ -n "${REDIS_PASSWORD}" ]]; then
   SECRET_ARGS+=(--from-literal="REDIS_PASSWORD=${REDIS_PASSWORD}")
 fi
 
+# Observability ingest credentials live in deploy/prod.env (gitignored) so
+# the values aren't checked into config.yaml. Skipped silently when the
+# file or keys are absent — the api/worker manifests mark these env vars
+# optional, so the deployment still rolls without traces.
+PROD_ENV_FILE="${DEPLOY_DIR}/../deploy/prod.env"
+if [[ -f "${PROD_ENV_FILE}" ]]; then
+  OBS_TOKEN_VAL="$(grep -E '^OBS_INGEST_TOKEN=' "${PROD_ENV_FILE}" 2>/dev/null | cut -d= -f2- || true)"
+  OBS_URL_VAL="$(grep -E '^OBS_TRACES_URL=' "${PROD_ENV_FILE}" 2>/dev/null | cut -d= -f2- || true)"
+  if [[ -n "${OBS_TOKEN_VAL}" ]]; then
+    log "  Including OBS_INGEST_TOKEN in secrets"
+    SECRET_ARGS+=(--from-literal="OBS_INGEST_TOKEN=${OBS_TOKEN_VAL}")
+  fi
+  if [[ -n "${OBS_URL_VAL}" ]]; then
+    log "  Including OBS_TRACES_URL in secrets"
+    SECRET_ARGS+=(--from-literal="OBS_TRACES_URL=${OBS_URL_VAL}")
+  fi
+fi
+
 kubectl create secret generic honeydue-secrets \
   "${SECRET_ARGS[@]}" \
   --dry-run=client -o yaml | kubectl apply -f -

docs/deployment/15-observability.md

@@ -220,17 +220,41 @@ Cheapest fix path:
 We'd want both eventually. Grafana alerting first because the data is
 already there.
 
-### Partial distributed tracing
+### Distributed tracing — adoption is in flight
 
-The OTel SDK is **not yet wired** in `cmd/api/main.go`. When it ships:
-- `otelecho.Middleware` produces a span per HTTP request
-- `otelgorm` plugin produces a span per SQL query (requires threading
-  `ctx` through repositories — the largest diff in the rollout)
-- Manual spans wrap B2 uploads, APNs/FCM sends, asynq jobs
+The OTel SDK is **wired** in `cmd/api/main.go` and `cmd/worker/main.go`
+and ships traces to Jaeger via `obs.88oakapps.com/v1/traces`. What's
+already producing spans:
 
-Until then, we have aggregate latency by route from the histograms but
-no per-request flame graph. For "why is *this one* request slow" we
-still rely on logs + the GORM duration histogram.
+| Span source | Status |
+|---|---|
+| `otelecho.Middleware` — span per HTTP request | ✅ live |
+| Manual span around `storage_service.Upload` (B2 PutObject) | ✅ live |
+| Manual span around APNs `Send` / `SendWithCategory` | ✅ live |
+| Manual span around FCM `sendOne` | ✅ live |
+| Asynq middleware — span per task type with retry/payload attrs | ✅ live |
+| `otelgorm` plugin — span per SQL statement | ✅ plugin registered |
+
+What's still in flight: SQL spans appear in a request's trace **only when
+the service method took the request's `ctx` and called
+`repo.WithContext(ctx)`** before issuing queries. Every repository now
+exposes `WithContext(ctx) *XRepository`, but services need to be
+migrated one method at a time.
+
+**Migration pattern:** for each service method on the request hot path,
+add `ctx context.Context` as the first arg, change the handler call site
+to pass `c.Request().Context()`, and replace `s.repo.X(...)` with
+`s.repo.WithContext(ctx).X(...)`. Tests pass `context.Background()`.
+
+Already migrated:
+- `TaskService.ListTasks` → `GET /api/tasks/`
+- `TaskService.GetTasksByResidence` → `GET /api/tasks/by-residence/:id/`
+
+Remaining: every other public method on `TaskService`, `ResidenceService`,
+`ContractorService`, `DocumentService`, `AuthService`,
+`NotificationService`, `SubscriptionService`. Mechanical work; can be
+done a method at a time without breaking anything (untouched methods
+just emit untraced SQL like before).
 
 ### No APM (Application Performance Monitoring)
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/treytartt/honeydue-api
 
-go 1.25
+go 1.25.0
 
 require (
 	github.com/go-pdf/fpdf v0.9.0
@@ -9,9 +9,10 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/hibiken/asynq v0.25.1
-	github.com/labstack/echo/v4 v4.11.4
+	github.com/labstack/echo/v4 v4.15.1
 	github.com/minio/minio-go/v7 v7.0.99
 	github.com/nicksnyder/go-i18n/v2 v2.6.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/redis/go-redis/v9 v9.17.1
 	github.com/rs/zerolog v1.34.0
 	github.com/shirou/gopsutil/v3 v3.24.5
@@ -20,11 +21,16 @@ require (
 	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.11.1
 	github.com/stripe/stripe-go/v81 v81.4.0
+	github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2
 	github.com/wneessen/go-mail v0.7.2
-	golang.org/x/crypto v0.46.0
-	golang.org/x/oauth2 v0.34.0
-	golang.org/x/text v0.32.0
-	golang.org/x/time v0.14.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.68.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0
+	go.opentelemetry.io/otel/sdk v1.43.0
+	golang.org/x/crypto v0.49.0
+	golang.org/x/oauth2 v0.35.0
+	golang.org/x/text v0.35.0
+	golang.org/x/time v0.15.0
 	google.golang.org/api v0.257.0
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/postgres v1.6.0
@@ -34,8 +40,10 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/klauspost/compress v1.18.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
 	github.com/klauspost/crc32 v1.3.0 // indirect
@@ -43,14 +51,16 @@ require (
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/tinylib/msgp v1.6.1 // indirect
+	github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
 )
 
 require (
@@ -69,7 +79,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect; TODO(S-19): Pulled by echo/v4 middleware — upgrade Echo to v4.12+ which removes built-in JWT middleware (uses echo-jwt/v4 with jwt/v5 instead), eliminating this vulnerable transitive dep
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
@@ -83,7 +92,7 @@ require (
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
@@ -104,13 +113,13 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
-	go.opentelemetry.io/otel v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
-	golang.org/x/net v0.48.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.39.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect
-	google.golang.org/grpc v1.77.0 // indirect
-	google.golang.org/protobuf v1.36.10 // indirect
+	go.opentelemetry.io/otel v1.43.0
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/grpc v1.80.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 )

go.sum

@@ -14,6 +14,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -54,8 +56,6 @@ github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaC
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
 github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
@@ -76,6 +76,8 @@ github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81
 github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
 github.com/hibiken/asynq v0.25.1 h1:phj028N0nm15n8O2ims+IvJ2gz4k2auvermngh9JhTw=
 github.com/hibiken/asynq v0.25.1/go.mod h1:pazWNOLBu0FEynQRBvHA26qdIKRSmfdIfUm4HdsLmXg=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@@ -101,16 +103,19 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8=
-github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/labstack/echo/v4 v4.15.1 h1:S9keusg26gZpjMmPqB5hOEvNKnmd1lNmcHrbbH2lnFs=
+github.com/labstack/echo/v4 v4.15.1/go.mod h1:xmw1clThob0BSVRX1CRQkGQ/vjwcpOMjQZSZa9fKA/c=
 github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
 github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -192,6 +197,10 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 h1:Jjn3zoRz13f8b1bR6LrXWglx93Sbh4kYfwgmPju3E2k=
+github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2/go.mod h1:wocb5pNrj/sjhWB9J5jctnC0K2eisSdz/nJJBNFHo+A=
+github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 h1:ZjUj9BLYf9PEqBn8W/OapxhPjVRdC6CsXTdULHsyk5c=
+github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2/go.mod h1:O8bHQfyinKwTXKkiKNGmLQS7vRsqRxIQTFZpYpHK3IQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
@@ -202,18 +211,28 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.68.0 h1:7N94HrYgVc2tng6xEjmbycupxteYLll7lPlEi/UK5ok=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.68.0/go.mod h1:1i+7wBOfx0kn7PSGRKZ8e7zIhs+AmvLCiCloySDUeck=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
+go.opentelemetry.io/contrib/propagators/b3 v1.43.0 h1:CETqV3QLLPTy5yNrqyMr41VnAOOD4lsRved7n4QG00A=
+go.opentelemetry.io/contrib/propagators/b3 v1.43.0/go.mod h1:Q4mCiCdziYzpNR0g+6UqVotAlCDZdzz6L8jwY4knOrw=
+go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
+go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
+go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
+go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
+go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
+go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
+go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
+go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
+go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
+go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
@@ -221,16 +240,16 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
-golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
-golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
-golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
-golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
+golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -242,32 +261,32 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
-golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.257.0 h1:8Y0lzvHlZps53PEaw+G29SsQIkuKrumGWs9puiexNAA=
 google.golang.org/api v0.257.0/go.mod h1:4eJrr+vbVaZSqs7vovFd1Jb/A6ml6iw2e6FBYf3GAO4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=
 google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=
-google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
-google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
-google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
-google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
+google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
+google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/database/database.go

@@ -15,6 +15,8 @@ import (
 	"github.com/treytartt/honeydue-api/internal/config"
 	"github.com/treytartt/honeydue-api/internal/models"
 	"github.com/treytartt/honeydue-api/internal/prom"
+
+	"github.com/uptrace/opentelemetry-go-extra/otelgorm"
 )
 
 // migrationAdvisoryLockKey is the pg_advisory_lock key that serializes
@@ -92,6 +94,15 @@ func Connect(cfg *config.DatabaseConfig, debug bool) (*gorm.DB, error) {
 		log.Warn().Err(err).Msg("failed to register prometheus GORM callbacks; metrics will be partial")
 	}
 
+	// Register otelgorm plugin — emits a span per SQL statement, attached to
+	// whatever trace context is set via db.WithContext(ctx). Repositories that
+	// have been migrated to use WithContext (see internal/repositories/*.go)
+	// will produce nested SQL spans inside the request trace; pre-migration
+	// repositories silently emit untraced queries.
+	if err := db.Use(otelgorm.NewPlugin(otelgorm.WithDBName(cfg.Database))); err != nil {
+		log.Warn().Err(err).Msg("failed to register otelgorm plugin; SQL spans disabled")
+	}
+
 	return db, nil
 }
 

internal/handlers/document_handler.go

@@ -201,7 +201,7 @@ func (h *DocumentHandler) CreateDocument(c echo.Context) error {
 			if h.storageService == nil {
 				return apperrors.Internal(nil)
 			}
-			result, err := h.storageService.Upload(uploadedFile, "documents")
+			result, err := h.storageService.Upload(c.Request().Context(), uploadedFile, "documents")
 			if err != nil {
 				return apperrors.BadRequest("error.failed_to_upload_file")
 			}
@@ -342,7 +342,7 @@ func (h *DocumentHandler) UploadDocumentImage(c echo.Context) error {
 		return apperrors.Internal(nil)
 	}
 
-	result, err := h.storageService.Upload(uploadedFile, "images")
+	result, err := h.storageService.Upload(c.Request().Context(), uploadedFile, "images")
 	if err != nil {
 		return apperrors.BadRequest("error.failed_to_upload_file")
 	}

internal/handlers/task_handler.go

@@ -65,7 +65,7 @@ func (h *TaskHandler) ListTasks(c echo.Context) error {
 		}
 	}
 
-	response, err := h.taskService.ListTasks(user.ID, daysThreshold, userNow)
+	response, err := h.taskService.ListTasks(c.Request().Context(), user.ID, daysThreshold, userNow)
 	if err != nil {
 		return err
 	}
@@ -121,7 +121,7 @@ func (h *TaskHandler) GetTasksByResidence(c echo.Context) error {
 		}
 	}
 
-	response, err := h.taskService.GetTasksByResidence(uint(residenceID), user.ID, daysThreshold, userNow)
+	response, err := h.taskService.GetTasksByResidence(c.Request().Context(), uint(residenceID), user.ID, daysThreshold, userNow)
 	if err != nil {
 		return err
 	}
@@ -446,7 +446,7 @@ func (h *TaskHandler) CreateCompletion(c echo.Context) error {
 			for _, fieldName := range []string{"images", "image", "photo", "files"} {
 				files := c.Request().MultipartForm.File[fieldName]
 				for _, file := range files {
-					result, err := h.storageService.Upload(file, "completions")
+					result, err := h.storageService.Upload(c.Request().Context(), file, "completions")
 					if err != nil {
 						return apperrors.BadRequest("error.failed_to_upload_image")
 					}

internal/handlers/upload_handler.go

@@ -48,7 +48,7 @@ func (h *UploadHandler) UploadImage(c echo.Context) error {
 		category = "images"
 	}
 
-	result, err := h.storageService.Upload(file, category)
+	result, err := h.storageService.Upload(c.Request().Context(), file, category)
 	if err != nil {
 		return err
 	}
@@ -64,7 +64,7 @@ func (h *UploadHandler) UploadDocument(c echo.Context) error {
 		return apperrors.BadRequest("error.no_file_provided")
 	}
 
-	result, err := h.storageService.Upload(file, "documents")
+	result, err := h.storageService.Upload(c.Request().Context(), file, "documents")
 	if err != nil {
 		return err
 	}
@@ -80,7 +80,7 @@ func (h *UploadHandler) UploadCompletion(c echo.Context) error {
 		return apperrors.BadRequest("error.no_file_provided")
 	}
 
-	result, err := h.storageService.Upload(file, "completions")
+	result, err := h.storageService.Upload(c.Request().Context(), file, "completions")
 	if err != nil {
 		return err
 	}

internal/push/apns.go

@@ -9,11 +9,17 @@ import (
 	"github.com/sideshow/apns2"
 	"github.com/sideshow/apns2/payload"
 	"github.com/sideshow/apns2/token"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/treytartt/honeydue-api/internal/config"
 	"github.com/treytartt/honeydue-api/internal/prom"
+	"github.com/treytartt/honeydue-api/internal/tracing"
 )
 
+var apnsTracer = tracing.Tracer("honeydue/push/apns")
+
 // APNsClient handles direct communication with Apple Push Notification service
 type APNsClient struct {
 	client *apns2.Client
@@ -86,10 +92,20 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 			Priority:    apns2.PriorityHigh,
 		}
 
+		sendCtx, span := apnsTracer.Start(ctx, "apns.send",
+			trace.WithAttributes(
+				attribute.String("apns.topic", c.topic),
+				attribute.String("apns.token", truncateToken(deviceToken)),
+				attribute.String("apns.priority", "high"),
+			),
+		)
 		sendStart := time.Now()
-		res, err := c.client.PushWithContext(ctx, notification)
+		res, err := c.client.PushWithContext(sendCtx, notification)
 		if err != nil {
 			prom.ObserveAPNsSend("error", time.Since(sendStart))
+			span.SetStatus(codes.Error, "push failed")
+			span.RecordError(err)
+			span.End()
 			log.Error().
 				Err(err).
 				Str("token", truncateToken(deviceToken)).
@@ -100,6 +116,12 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 
 		if !res.Sent() {
 			prom.ObserveAPNsSend("bad_token", time.Since(sendStart))
+			span.SetAttributes(
+				attribute.Int("apns.status_code", res.StatusCode),
+				attribute.String("apns.reason", res.Reason),
+			)
+			span.SetStatus(codes.Error, "bad token")
+			span.End()
 			log.Error().
 				Str("token", truncateToken(deviceToken)).
 				Str("reason", res.Reason).
@@ -110,6 +132,8 @@ func (c *APNsClient) Send(ctx context.Context, tokens []string, title, message s
 		}
 
 		prom.ObserveAPNsSend("ok", time.Since(sendStart))
+		span.SetAttributes(attribute.String("apns.id", res.ApnsID))
+		span.End()
 		successCount++
 		log.Debug().
 			Str("token", truncateToken(deviceToken)).
@@ -160,10 +184,20 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 			Priority:    apns2.PriorityHigh,
 		}
 
+		sendCtx, span := apnsTracer.Start(ctx, "apns.send.category",
+			trace.WithAttributes(
+				attribute.String("apns.topic", c.topic),
+				attribute.String("apns.token", truncateToken(deviceToken)),
+				attribute.String("apns.category_id", categoryID),
+			),
+		)
 		sendStart := time.Now()
-		res, err := c.client.PushWithContext(ctx, notification)
+		res, err := c.client.PushWithContext(sendCtx, notification)
 		if err != nil {
 			prom.ObserveAPNsSend("error", time.Since(sendStart))
+			span.SetStatus(codes.Error, "push failed")
+			span.RecordError(err)
+			span.End()
 			log.Error().
 				Err(err).
 				Str("token", truncateToken(deviceToken)).
@@ -175,6 +209,12 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 
 		if !res.Sent() {
 			prom.ObserveAPNsSend("bad_token", time.Since(sendStart))
+			span.SetAttributes(
+				attribute.Int("apns.status_code", res.StatusCode),
+				attribute.String("apns.reason", res.Reason),
+			)
+			span.SetStatus(codes.Error, "bad token")
+			span.End()
 			log.Error().
 				Str("token", truncateToken(deviceToken)).
 				Str("reason", res.Reason).
@@ -186,6 +226,8 @@ func (c *APNsClient) SendWithCategory(ctx context.Context, tokens []string, titl
 		}
 
 		prom.ObserveAPNsSend("ok", time.Since(sendStart))
+		span.SetAttributes(attribute.String("apns.id", res.ApnsID))
+		span.End()
 		successCount++
 		log.Debug().
 			Str("token", truncateToken(deviceToken)).

internal/push/fcm.go

@@ -12,12 +12,18 @@ import (
 	"time"
 
 	"github.com/rs/zerolog/log"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2/google"
 
 	"github.com/treytartt/honeydue-api/internal/config"
 	"github.com/treytartt/honeydue-api/internal/prom"
+	"github.com/treytartt/honeydue-api/internal/tracing"
 )
 
+var fcmTracer = tracing.Tracer("honeydue/push/fcm")
+
 const (
 	// fcmV1EndpointFmt is the FCM HTTP v1 API endpoint template.
 	fcmV1EndpointFmt = "https://fcm.googleapis.com/v1/projects/%s/messages:send"
@@ -214,9 +220,15 @@ func (c *FCMClient) Send(ctx context.Context, tokens []string, title, message st
 	var sendErrors []error
 	successCount := 0
 
-	for _, token := range tokens {
+	for _, tokenStr := range tokens {
+		sendCtx, span := fcmTracer.Start(ctx, "fcm.send",
+			trace.WithAttributes(
+				attribute.String("fcm.token", truncateToken(tokenStr)),
+				attribute.String("fcm.priority", "HIGH"),
+			),
+		)
 		sendStart := time.Now()
-		err := c.sendOne(ctx, token, title, message, data)
+		err := c.sendOne(sendCtx, tokenStr, title, message, data)
 		if err != nil {
 			result := "error"
 			var fcmErr *FCMSendError
@@ -224,18 +236,24 @@ func (c *FCMClient) Send(ctx context.Context, tokens []string, title, message st
 				result = "bad_token"
 			}
 			prom.ObserveFCMSend(result, time.Since(sendStart))
+			span.SetAttributes(attribute.String("fcm.result", result))
+			span.SetStatus(codes.Error, result)
+			span.RecordError(err)
+			span.End()
 			log.Error().
 				Err(err).
-				Str("token", truncateToken(token)).
+				Str("token", truncateToken(tokenStr)).
 				Msg("FCM v1 notification failed")
 			sendErrors = append(sendErrors, err)
 			continue
 		}
 
 		prom.ObserveFCMSend("ok", time.Since(sendStart))
+		span.SetAttributes(attribute.String("fcm.result", "ok"))
+		span.End()
 		successCount++
 		log.Debug().
-			Str("token", truncateToken(token)).
+			Str("token", truncateToken(tokenStr)).
 			Msg("FCM v1 notification sent successfully")
 	}
 

internal/repositories/admin_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"errors"
 	"time"
 
@@ -105,3 +106,10 @@ func (r *AdminRepository) ExistsByEmail(email string) (bool, error) {
 	}
 	return count > 0, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *AdminRepository) WithContext(ctx context.Context) *AdminRepository {
+	return &AdminRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/contractor_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"gorm.io/gorm"
 
 	"github.com/treytartt/honeydue-api/internal/models"
@@ -193,3 +194,10 @@ func (r *ContractorRepository) FindSpecialtyByID(id uint) (*models.ContractorSpe
 	}
 	return &specialty, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *ContractorRepository) WithContext(ctx context.Context) *ContractorRepository {
+	return &ContractorRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/document_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"time"
 
 	"gorm.io/gorm"
@@ -214,3 +215,10 @@ func (r *DocumentRepository) FindImageByID(id uint) (*models.DocumentImage, erro
 	}
 	return &image, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *DocumentRepository) WithContext(ctx context.Context) *DocumentRepository {
+	return &DocumentRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/notification_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"errors"
 	"time"
 
@@ -291,3 +292,10 @@ func (r *NotificationRepository) GetActiveTokensForUser(userID uint) (iosTokens
 
 	return iosTokens, androidTokens, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *NotificationRepository) WithContext(ctx context.Context) *NotificationRepository {
+	return &NotificationRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/reminder_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"time"
 
 	"gorm.io/gorm"
@@ -224,3 +225,10 @@ func (r *ReminderRepository) GetRecentReminderStats(sinceHours int) (map[string]
 
 	return stats, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *ReminderRepository) WithContext(ctx context.Context) *ReminderRepository {
+	return &ReminderRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/residence_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"crypto/rand"
 	"errors"
 	"math/big"
@@ -360,3 +361,10 @@ func (r *ResidenceRepository) GetTasksForReport(residenceID uint) ([]models.Task
 		Find(&tasks).Error
 	return tasks, err
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *ResidenceRepository) WithContext(ctx context.Context) *ResidenceRepository {
+	return &ResidenceRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/subscription_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"errors"
 	"time"
 
@@ -327,3 +328,10 @@ func (r *SubscriptionRepository) UpdateExpiresAt(userID uint, expiresAt time.Tim
 	return r.db.Model(&models.UserSubscription{}).Where("user_id = ?", userID).
 		Update("expires_at", expiresAt).Error
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *SubscriptionRepository) WithContext(ctx context.Context) *SubscriptionRepository {
+	return &SubscriptionRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/task_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"time"
@@ -1057,3 +1058,10 @@ func (r *TaskRepository) GetBatchCompletionSummaries(residenceIDs []uint, now ti
 
 	return result, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *TaskRepository) WithContext(ctx context.Context) *TaskRepository {
+	return &TaskRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/task_template_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"strings"
 
 	"gorm.io/gorm"
@@ -122,3 +123,10 @@ func (r *TaskTemplateRepository) GetGroupedByCategory() (map[string][]models.Tas
 
 	return result, nil
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *TaskTemplateRepository) WithContext(ctx context.Context) *TaskTemplateRepository {
+	return &TaskTemplateRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/user_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"errors"
 	"strings"
 	"time"
@@ -772,3 +773,10 @@ func (r *UserRepository) CreateGoogleSocialAuth(auth *models.GoogleSocialAuth) e
 func (r *UserRepository) UpdateGoogleSocialAuth(auth *models.GoogleSocialAuth) error {
 	return r.db.Save(auth).Error
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *UserRepository) WithContext(ctx context.Context) *UserRepository {
+	return &UserRepository{db: r.db.WithContext(ctx)}
+}

internal/repositories/webhook_event_repo.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	"context"
 	"time"
 
 	"gorm.io/gorm"
@@ -52,3 +53,10 @@ func (r *WebhookEventRepository) RecordEvent(provider, eventID, eventType, paylo
 	}
 	return r.db.Create(event).Error
 }
+
+// WithContext returns a copy of the repository whose underlying *gorm.DB carries
+// the supplied context. SQL emitted via this copy gets attached to ctx's trace span
+// (when otelgorm is registered) and respects ctx cancellation/deadlines.
+func (r *WebhookEventRepository) WithContext(ctx context.Context) *WebhookEventRepository {
+	return &WebhookEventRepository{db: r.db.WithContext(ctx)}
+}

internal/router/router.go

@@ -13,6 +13,7 @@ import (
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/rs/zerolog/log"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho"
 	"gorm.io/gorm"
 
 	"github.com/treytartt/honeydue-api/internal/admin"
@@ -62,6 +63,11 @@ func SetupRouter(deps *Dependencies) *echo.Echo {
 	e.Use(utils.EchoRecovery())
 	e.Use(custommiddleware.StructuredLogger())
 
+	// OpenTelemetry HTTP middleware — opens a span per request, attaches the
+	// route pattern, method, status, and request_id. Sits early so subsequent
+	// middleware + handlers run inside the request span.
+	e.Use(otelecho.Middleware("honeydue-api"))
+
 	// Security headers (X-Frame-Options, X-Content-Type-Options, X-XSS-Protection, etc.)
 	//
 	// CSP is permissive enough to serve the marketing landing page at / (which

internal/services/storage_service.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"mime/multipart"
@@ -14,8 +15,14 @@ import (
 
 	"github.com/treytartt/honeydue-api/internal/config"
 	"github.com/treytartt/honeydue-api/internal/prom"
+	"github.com/treytartt/honeydue-api/internal/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
+var storageTracer = tracing.Tracer("honeydue/services/storage")
+
 // StorageService handles file uploads, validation, encryption, and URL generation.
 // It delegates raw I/O to a StorageBackend (local filesystem or S3-compatible).
 type StorageService struct {
@@ -66,8 +73,17 @@ func NewStorageService(cfg *config.StorageConfig) (*StorageService, error) {
 	return &StorageService{cfg: cfg, backend: backend, allowedTypes: allowedTypes}, nil
 }
 
-// Upload saves a file to storage (local or S3)
-func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*UploadResult, error) {
+// Upload saves a file to storage (local or S3). The ctx is used to attach
+// the underlying B2/S3 PutObject span to the request trace.
+func (s *StorageService) Upload(ctx context.Context, file *multipart.FileHeader, category string) (*UploadResult, error) {
+	ctx, span := storageTracer.Start(ctx, "storage.upload",
+		trace.WithAttributes(
+			attribute.String("file.name", file.Filename),
+			attribute.Int64("file.size_bytes", file.Size),
+			attribute.String("upload.category", category),
+		),
+	)
+	defer span.End()
 	// Validate file size
 	if file.Size > s.cfg.MaxFileSize {
 		return nil, fmt.Errorf("file size %d exceeds maximum allowed %d bytes", file.Size, s.cfg.MaxFileSize)
@@ -150,18 +166,31 @@ func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*U
 		}
 	}
 
-	// Write to backend (B2/S3 round trip — instrumented for Prometheus)
+	// Write to backend (B2/S3 round trip — instrumented for Prometheus + traces)
 	bucket := s.cfg.S3Bucket
 	if bucket == "" {
 		bucket = "local"
 	}
+	_, putSpan := storageTracer.Start(ctx, "b2.PutObject",
+		trace.WithAttributes(
+			attribute.String("b2.bucket", bucket),
+			attribute.String("b2.key", key),
+			attribute.Int64("b2.size_bytes", int64(len(fileData))),
+			attribute.String("b2.mime_type", mimeType),
+		),
+	)
 	uploadStart := time.Now()
 	if err := s.backend.Write(key, fileData); err != nil {
 		prom.ObserveB2Upload(bucket, "error", time.Since(uploadStart), 0)
+		putSpan.SetStatus(codes.Error, "write failed")
+		putSpan.RecordError(err)
+		putSpan.End()
 		return nil, fmt.Errorf("failed to save file: %w", err)
 	}
 	written := int64(len(fileData))
 	prom.ObserveB2Upload(bucket, "ok", time.Since(uploadStart), written)
+	putSpan.SetAttributes(attribute.Int64("b2.bytes_written", written))
+	putSpan.End()
 
 	// Generate URL (always uses the original filename without .enc suffix)
 	url := fmt.Sprintf("%s/%s/%s", s.cfg.BaseURL, subdir, newFilename)

internal/services/task_service.go

@@ -103,13 +103,13 @@ func (s *TaskService) GetTask(taskID, userID uint) (*responses.TaskResponse, err
 
 // ListTasks lists all tasks accessible to a user as a kanban board.
 // The `now` parameter should be the start of day in the user's timezone for accurate overdue detection.
-func (s *TaskService) ListTasks(userID uint, daysThreshold int, now time.Time) (*responses.KanbanBoardResponse, error) {
+func (s *TaskService) ListTasks(ctx context.Context, userID uint, daysThreshold int, now time.Time) (*responses.KanbanBoardResponse, error) {
 	if daysThreshold <= 0 {
 		daysThreshold = 30 // Default
 	}
 
 	// Get all residence IDs accessible to user (lightweight - no preloads)
-	residenceIDs, err := s.residenceRepo.FindResidenceIDsByUser(userID)
+	residenceIDs, err := s.residenceRepo.WithContext(ctx).FindResidenceIDsByUser(userID)
 	if err != nil {
 		return nil, apperrors.Internal(err)
 	}
@@ -124,7 +124,7 @@ func (s *TaskService) ListTasks(userID uint, daysThreshold int, now time.Time) (
 	}
 
 	// Get kanban data aggregated across all residences using user's timezone-aware time
-	board, err := s.taskRepo.GetKanbanDataForMultipleResidences(residenceIDs, daysThreshold, now)
+	board, err := s.taskRepo.WithContext(ctx).GetKanbanDataForMultipleResidences(residenceIDs, daysThreshold, now)
 	if err != nil {
 		return nil, apperrors.Internal(err)
 	}
@@ -136,9 +136,10 @@ func (s *TaskService) ListTasks(userID uint, daysThreshold int, now time.Time) (
 
 // GetTasksByResidence gets tasks for a specific residence (kanban board).
 // The `now` parameter should be the start of day in the user's timezone for accurate overdue detection.
-func (s *TaskService) GetTasksByResidence(residenceID, userID uint, daysThreshold int, now time.Time) (*responses.KanbanBoardResponse, error) {
-	// Check access
-	hasAccess, err := s.residenceRepo.HasAccess(residenceID, userID)
+func (s *TaskService) GetTasksByResidence(ctx context.Context, residenceID, userID uint, daysThreshold int, now time.Time) (*responses.KanbanBoardResponse, error) {
+	// Check access — uses repo.WithContext(ctx) so the SQL span is attached
+	// to the inbound HTTP request's trace via otelgorm.
+	hasAccess, err := s.residenceRepo.WithContext(ctx).HasAccess(residenceID, userID)
 	if err != nil {
 		return nil, apperrors.Internal(err)
 	}
@@ -151,7 +152,7 @@ func (s *TaskService) GetTasksByResidence(residenceID, userID uint, daysThreshol
 	}
 
 	// Get kanban data using user's timezone-aware time
-	board, err := s.taskRepo.GetKanbanData(residenceID, daysThreshold, now)
+	board, err := s.taskRepo.WithContext(ctx).GetKanbanData(residenceID, daysThreshold, now)
 	if err != nil {
 		return nil, apperrors.Internal(err)
 	}

internal/services/task_service_test.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"context"
 	"net/http"
 	"testing"
 	"time"
@@ -301,7 +302,7 @@ func TestTaskService_ListTasks(t *testing.T) {
 	testutil.CreateTestTask(t, db, residence.ID, user.ID, "Task 2")
 	testutil.CreateTestTask(t, db, residence.ID, user.ID, "Task 3")
 
-	resp, err := service.ListTasks(user.ID, 30, time.Now().UTC())
+	resp, err := service.ListTasks(context.Background(), user.ID, 30, time.Now().UTC())
 	require.NoError(t, err)
 	// ListTasks returns a KanbanBoardResponse with columns
 	// Count total tasks across all columns

internal/tracing/tracing.go

@@ -0,0 +1,161 @@
+// Package tracing wires the OpenTelemetry SDK with an OTLP/HTTP exporter
+// targeting obs.88oakapps.com (Jaeger all-in-one behind nginx + bearer auth).
+//
+// The package owns the global TracerProvider for the api process; everything
+// else acquires a tracer via tracing.Tracer(name).
+//
+// Sampling defaults to AlwaysSample in DEBUG mode and TraceIDRatioBased(0.1)
+// otherwise, controllable via OTEL_TRACES_SAMPLER_ARG.
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
+)
+
+// Config controls the tracer provider that Init installs globally.
+type Config struct {
+	// ServiceName labels every span with service.name=<this>. Required.
+	ServiceName string
+
+	// Environment labels every span with deployment.environment.
+	// Conventionally "prod", "dev", "local".
+	Environment string
+
+	// EndpointURL is the full OTLP/HTTP traces URL — e.g.
+	// https://obs.88oakapps.com/v1/traces. Empty means tracing is disabled
+	// (returns a no-op provider).
+	EndpointURL string
+
+	// BearerToken, if non-empty, is sent as Authorization: Bearer <token>.
+	BearerToken string
+
+	// SampleRatio is the fraction of root traces sampled. 1.0 = all, 0.1 = 10%.
+	// 0 disables sampling entirely; -1 means "AlwaysSample" (debug).
+	SampleRatio float64
+
+	// Insecure forces plain HTTP. Only useful for local testing.
+	Insecure bool
+}
+
+// Init configures the global TracerProvider and returns a shutdown function.
+// Call shutdown on graceful exit so spans in flight get flushed.
+//
+// Init is safe to call when EndpointURL is empty: it installs a no-op
+// provider and returns a no-op shutdown.
+func Init(ctx context.Context, cfg Config) (shutdown func(context.Context) error, err error) {
+	if cfg.EndpointURL == "" {
+		log.Info().Msg("tracing: no OBS_TRACES_URL configured, installing no-op tracer")
+		otel.SetTracerProvider(noop.NewTracerProvider())
+		return func(context.Context) error { return nil }, nil
+	}
+
+	parsed, err := url.Parse(cfg.EndpointURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid OBS_TRACES_URL %q: %w", cfg.EndpointURL, err)
+	}
+
+	opts := []otlptracehttp.Option{
+		otlptracehttp.WithEndpoint(parsed.Host),
+		otlptracehttp.WithURLPath(parsed.Path),
+		otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
+		otlptracehttp.WithTimeout(10 * time.Second),
+	}
+	if cfg.Insecure || parsed.Scheme == "http" {
+		opts = append(opts, otlptracehttp.WithInsecure())
+	}
+	if cfg.BearerToken != "" {
+		opts = append(opts, otlptracehttp.WithHeaders(map[string]string{
+			"Authorization": "Bearer " + cfg.BearerToken,
+		}))
+	}
+
+	exporter, err := otlptrace.New(ctx, otlptracehttp.NewClient(opts...))
+	if err != nil {
+		return nil, fmt.Errorf("create OTLP exporter: %w", err)
+	}
+
+	res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(
+		semconv.SchemaURL,
+		semconv.ServiceName(cfg.ServiceName),
+		semconv.DeploymentEnvironmentName(cfg.Environment),
+	))
+	if err != nil {
+		return nil, fmt.Errorf("build resource: %w", err)
+	}
+
+	var sampler sdktrace.Sampler
+	switch {
+	case cfg.SampleRatio < 0:
+		sampler = sdktrace.AlwaysSample()
+	case cfg.SampleRatio == 0:
+		sampler = sdktrace.NeverSample()
+	case cfg.SampleRatio >= 1:
+		sampler = sdktrace.AlwaysSample()
+	default:
+		// ParentBased so the inbound parent's sampling decision wins;
+		// otherwise root-span ratio applies.
+		sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))
+	}
+
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exporter,
+			sdktrace.WithBatchTimeout(5*time.Second),
+			sdktrace.WithMaxExportBatchSize(512),
+		),
+		sdktrace.WithResource(res),
+		sdktrace.WithSampler(sampler),
+	)
+
+	otel.SetTracerProvider(tp)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	))
+
+	log.Info().
+		Str("endpoint", cfg.EndpointURL).
+		Str("service", cfg.ServiceName).
+		Str("env", cfg.Environment).
+		Float64("sample_ratio", cfg.SampleRatio).
+		Bool("auth", cfg.BearerToken != "").
+		Msg("tracing: OTLP exporter initialized")
+
+	return tp.Shutdown, nil
+}
+
+// Tracer returns a named tracer from the global provider. Safe to call before
+// Init (returns a no-op tracer in that case).
+func Tracer(name string) trace.Tracer {
+	return otel.Tracer(name)
+}
+
+// SampleRatioFromEnv reads OTEL_TRACES_SAMPLER_ARG with sensible defaults.
+// Returns -1 ("always") when DEBUG=true, 0.1 ("10%") otherwise.
+func SampleRatioFromEnv() float64 {
+	if v := strings.TrimSpace(os.Getenv("OTEL_TRACES_SAMPLER_ARG")); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			return f
+		}
+	}
+	if strings.EqualFold(os.Getenv("DEBUG"), "true") {
+		return -1
+	}
+	return 0.1
+}