df78d9ccd8
Adds internal/prom package with histograms for HTTP, GORM, B2, APNs, and FCM, wired into the Echo router (HTTPMiddleware + /metrics) and GORM via statement-level callbacks (no ctx plumbing needed). Storage and push clients call ObserveB2Upload / ObserveAPNsSend / ObserveFCMSend at the network round-trip points. Existing internal/monitoring metrics move to /metrics/legacy so the canonical /metrics emits proper histogram buckets for p50/p95/p99 rollups. deploy-k3s/manifests/observability/vmagent.yaml deploys a single-replica vmagent in the honeydue namespace that scrapes api Pods on :8000/metrics every 15s and remote-writes to https://obs.88oakapps.com/api/v1/write with a bearer token (substituted at deploy time from OBS_INGEST_TOKEN in deploy/prod.env). NetworkPolicies allow vmagent egress to api Pods and to the public obs endpoint over :443; the obs side runs VictoriaMetrics + Jaeger + Grafana on 88oakappsUpdate. docs/observability-plan.md captures the full plan including resource budget, instrumentation table, 4-step rollout, and migration triggers. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
188 lines
5.0 KiB
YAML
188 lines
5.0 KiB
YAML
# vmagent — scrapes Prometheus /metrics from in-cluster services and
|
|
# remote-writes them to https://obs.88oakapps.com/api/v1/write
|
|
# (VictoriaMetrics on 88oakappsUpdate, fronted by Cloudflare + nginx
|
|
# bearer-token auth). Single replica is fine — vmagent buffers locally
|
|
# during transient remote outages.
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: vmagent-config
|
|
namespace: honeydue
|
|
labels:
|
|
app.kubernetes.io/name: vmagent
|
|
app.kubernetes.io/part-of: honeydue
|
|
data:
|
|
scrape.yaml: |
|
|
global:
|
|
scrape_interval: 15s
|
|
external_labels:
|
|
cluster: honeydue-k3s
|
|
environment: prod
|
|
|
|
scrape_configs:
|
|
# honeyDue Go API — exposes /metrics on :8000
|
|
- job_name: api
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names: [honeydue]
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
|
action: keep
|
|
regex: api
|
|
- source_labels: [__meta_kubernetes_pod_container_port_number]
|
|
action: keep
|
|
regex: "8000"
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
|
target_label: node
|
|
- target_label: service
|
|
replacement: api
|
|
|
|
# honeyDue worker — also exposes /metrics if/when we add it.
|
|
# Keep this stanza commented until the worker has a /metrics endpoint;
|
|
# uncommented form drops scrapes silently.
|
|
# - job_name: worker
|
|
# kubernetes_sd_configs:
|
|
# - role: pod
|
|
# namespaces:
|
|
# names: [honeydue]
|
|
# relabel_configs:
|
|
# - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
|
# action: keep
|
|
# regex: worker
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: vmagent-remote-write
|
|
namespace: honeydue
|
|
labels:
|
|
app.kubernetes.io/name: vmagent
|
|
app.kubernetes.io/part-of: honeydue
|
|
type: Opaque
|
|
stringData:
|
|
# Bearer token for obs.88oakapps.com. Provisioned at deploy time from
|
|
# deploy/prod.env (OBS_INGEST_TOKEN). The cluster-side token must match
|
|
# the token in /etc/honeydue-obs/ingest_token on 88oakappsUpdate.
|
|
bearer_token: TOKEN_PLACEHOLDER
|
|
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: vmagent
|
|
namespace: honeydue
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: [pods, services, endpoints]
|
|
verbs: [get, list, watch]
|
|
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: vmagent
|
|
namespace: honeydue
|
|
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: vmagent
|
|
namespace: honeydue
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: vmagent
|
|
namespace: honeydue
|
|
roleRef:
|
|
kind: Role
|
|
name: vmagent
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vmagent
|
|
namespace: honeydue
|
|
labels:
|
|
app.kubernetes.io/name: vmagent
|
|
app.kubernetes.io/part-of: honeydue
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: vmagent
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: vmagent
|
|
app.kubernetes.io/part-of: honeydue
|
|
spec:
|
|
serviceAccountName: vmagent
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
fsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: vmagent
|
|
image: victoriametrics/vmagent:v1.106.1
|
|
args:
|
|
- "-promscrape.config=/etc/vmagent/scrape.yaml"
|
|
- "-remoteWrite.url=https://obs.88oakapps.com/api/v1/write"
|
|
- "-remoteWrite.bearerTokenFile=/etc/vmagent-secrets/bearer_token"
|
|
- "-remoteWrite.tmpDataPath=/tmp/vmagent"
|
|
- "-remoteWrite.maxDiskUsagePerURL=512MB"
|
|
- "-loggerLevel=INFO"
|
|
ports:
|
|
- containerPort: 8429
|
|
name: http
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/vmagent
|
|
readOnly: true
|
|
- name: secrets
|
|
mountPath: /etc/vmagent-secrets
|
|
readOnly: true
|
|
- name: buffer
|
|
mountPath: /tmp/vmagent
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /-/healthy
|
|
port: http
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 30
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /-/healthy
|
|
port: http
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: vmagent-config
|
|
- name: secrets
|
|
secret:
|
|
secretName: vmagent-remote-write
|
|
defaultMode: 0400
|
|
- name: buffer
|
|
emptyDir:
|
|
sizeLimit: 512Mi
|