Add Prometheus metrics + vmagent push to obs.88oakapps.com
Adds internal/prom package with histograms for HTTP, GORM, B2, APNs, and FCM, wired into the Echo router (HTTPMiddleware + /metrics) and GORM via statement-level callbacks (no ctx plumbing needed). Storage and push clients call ObserveB2Upload / ObserveAPNsSend / ObserveFCMSend at the network round-trip points. Existing internal/monitoring metrics move to /metrics/legacy so the canonical /metrics emits proper histogram buckets for p50/p95/p99 rollups. deploy-k3s/manifests/observability/vmagent.yaml deploys a single-replica vmagent in the honeydue namespace that scrapes api Pods on :8000/metrics every 15s and remote-writes to https://obs.88oakapps.com/api/v1/write with a bearer token (substituted at deploy time from OBS_INGEST_TOKEN in deploy/prod.env). NetworkPolicies allow vmagent egress to api Pods and to the public obs endpoint over :443; the obs side runs VictoriaMetrics + Jaeger + Grafana on 88oakappsUpdate. docs/observability-plan.md captures the full plan including resource budget, instrumentation table, 4-step rollout, and migration triggers. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey t
@@ -0,0 +1,187 @@
|
||||
# vmagent — scrapes Prometheus /metrics from in-cluster services and
|
||||
# remote-writes them to https://obs.88oakapps.com/api/v1/write
|
||||
# (VictoriaMetrics on 88oakappsUpdate, fronted by Cloudflare + nginx
|
||||
# bearer-token auth). Single replica is fine — vmagent buffers locally
|
||||
# during transient remote outages.
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vmagent-config
|
||||
namespace: honeydue
|
||||
labels:
|
||||
app.kubernetes.io/name: vmagent
|
||||
app.kubernetes.io/part-of: honeydue
|
||||
data:
|
||||
scrape.yaml: |
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
external_labels:
|
||||
cluster: honeydue-k3s
|
||||
environment: prod
|
||||
|
||||
scrape_configs:
|
||||
# honeyDue Go API — exposes /metrics on :8000
|
||||
- job_name: api
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [honeydue]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||||
action: keep
|
||||
regex: api
|
||||
- source_labels: [__meta_kubernetes_pod_container_port_number]
|
||||
action: keep
|
||||
regex: "8000"
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
- target_label: service
|
||||
replacement: api
|
||||
|
||||
# honeyDue worker — also exposes /metrics if/when we add it.
|
||||
# Keep this stanza commented until the worker has a /metrics endpoint;
|
||||
# uncommented form drops scrapes silently.
|
||||
# - job_name: worker
|
||||
# kubernetes_sd_configs:
|
||||
# - role: pod
|
||||
# namespaces:
|
||||
# names: [honeydue]
|
||||
# relabel_configs:
|
||||
# - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||||
# action: keep
|
||||
# regex: worker
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: vmagent-remote-write
|
||||
namespace: honeydue
|
||||
labels:
|
||||
app.kubernetes.io/name: vmagent
|
||||
app.kubernetes.io/part-of: honeydue
|
||||
type: Opaque
|
||||
stringData:
|
||||
# Bearer token for obs.88oakapps.com. Provisioned at deploy time from
|
||||
# deploy/prod.env (OBS_INGEST_TOKEN). The cluster-side token must match
|
||||
# the token in /etc/honeydue-obs/ingest_token on 88oakappsUpdate.
|
||||
bearer_token: TOKEN_PLACEHOLDER
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: vmagent
|
||||
namespace: honeydue
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: [pods, services, endpoints]
|
||||
verbs: [get, list, watch]
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: vmagent
|
||||
namespace: honeydue
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: vmagent
|
||||
namespace: honeydue
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: vmagent
|
||||
namespace: honeydue
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: vmagent
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vmagent
|
||||
namespace: honeydue
|
||||
labels:
|
||||
app.kubernetes.io/name: vmagent
|
||||
app.kubernetes.io/part-of: honeydue
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: vmagent
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: vmagent
|
||||
app.kubernetes.io/part-of: honeydue
|
||||
spec:
|
||||
serviceAccountName: vmagent
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: vmagent
|
||||
image: victoriametrics/vmagent:v1.106.1
|
||||
args:
|
||||
- "-promscrape.config=/etc/vmagent/scrape.yaml"
|
||||
- "-remoteWrite.url=https://obs.88oakapps.com/api/v1/write"
|
||||
- "-remoteWrite.bearerTokenFile=/etc/vmagent-secrets/bearer_token"
|
||||
- "-remoteWrite.tmpDataPath=/tmp/vmagent"
|
||||
- "-remoteWrite.maxDiskUsagePerURL=512MB"
|
||||
- "-loggerLevel=INFO"
|
||||
ports:
|
||||
- containerPort: 8429
|
||||
name: http
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/vmagent
|
||||
readOnly: true
|
||||
- name: secrets
|
||||
mountPath: /etc/vmagent-secrets
|
||||
readOnly: true
|
||||
- name: buffer
|
||||
mountPath: /tmp/vmagent
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /-/healthy
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/healthy
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: vmagent-config
|
||||
- name: secrets
|
||||
secret:
|
||||
secretName: vmagent-remote-write
|
||||
defaultMode: 0400
|
||||
- name: buffer
|
||||
emptyDir:
|
||||
sizeLimit: 512Mi
|
||||
Reference in New Issue
Block a user