Trey t
93fddc3769
Adds a Grafana Alloy DaemonSet that tails honeydue-namespace pod logs from /var/log/pods and pushes them to Loki at obs.88oakapps.com, reusing the existing OBS_INGEST_TOKEN (14-day retention). - deploy-k3s/manifests/observability/alloy-logs.yaml — DaemonSet + RBAC + token Secret + Alloy config. Runs as root (/var/log/pods is 0750 root:root) but otherwise locked down: all caps dropped, read-only root filesystem, seccomp RuntimeDefault, read-only hostPath mount. - network-policies.yaml — allow-egress-from-alloy-logs (DNS + k8s API + obs HTTPS), mirroring the vmagent egress policy. - 03-deploy.sh — applies alloy-logs with the OBS_INGEST_TOKEN substitution and waits for the DaemonSet rollout. The Loki container, nginx /loki/api/v1/push route, and Grafana Loki datasource live on the obs server and are not repo-managed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
429 lines
11 KiB
YAML
429 lines
11 KiB
YAML
# Network Policies — default-deny with explicit allows
|
|
# Apply AFTER namespace and deployments are created.
|
|
# Verify: kubectl get networkpolicy -n honeydue
|
|
|
|
# --- Default deny all ingress and egress ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: default-deny-all
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes:
|
|
- Ingress
|
|
- Egress
|
|
|
|
---
|
|
# --- Allow DNS for all pods (required for service discovery) ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-dns
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
- to: []
|
|
ports:
|
|
- protocol: UDP
|
|
port: 53
|
|
- protocol: TCP
|
|
port: 53
|
|
|
|
---
|
|
# --- API: allow ingress from Traefik (kube-system namespace) ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-ingress-to-api
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: api
|
|
policyTypes:
|
|
- Ingress
|
|
ingress:
|
|
# Traefik runs as DaemonSet with hostNetwork=true, so traffic from it
|
|
# arrives with the NODE IP as source (not a pod IP). The node pod CIDR
|
|
# 10.42.0.0/16 covers any intra-cluster caller; the three node IPs
|
|
# cover Traefik on hostNetwork.
|
|
- from:
|
|
- ipBlock:
|
|
cidr: 178.105.32.198/32 # ubuntu-8gb-nbg1-1
|
|
- ipBlock:
|
|
cidr: 178.104.247.152/32 # ubuntu-8gb-nbg1-2
|
|
- ipBlock:
|
|
cidr: 178.104.249.189/32 # ubuntu-8gb-nbg1-3
|
|
- ipBlock:
|
|
cidr: 10.42.0.0/16 # cluster pod CIDR
|
|
ports:
|
|
- protocol: TCP
|
|
port: 8000
|
|
|
|
---
|
|
# --- Admin: allow ingress from Traefik (kube-system namespace) ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-ingress-to-admin
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: admin
|
|
policyTypes:
|
|
- Ingress
|
|
ingress:
|
|
# Traefik runs as DaemonSet with hostNetwork=true — see allow-ingress-to-api
|
|
# for the rationale. Same ipBlock list.
|
|
- from:
|
|
- ipBlock:
|
|
cidr: 178.105.32.198/32
|
|
- ipBlock:
|
|
cidr: 178.104.247.152/32
|
|
- ipBlock:
|
|
cidr: 178.104.249.189/32
|
|
- ipBlock:
|
|
cidr: 10.42.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 3000
|
|
|
|
---
|
|
# --- Redis: allow ingress ONLY from api + worker pods ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-ingress-to-redis
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
policyTypes:
|
|
- Ingress
|
|
ingress:
|
|
- from:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: api
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: worker
|
|
ports:
|
|
- protocol: TCP
|
|
port: 6379
|
|
|
|
---
|
|
# --- API: allow egress to Redis, external services (Neon DB, APNs, FCM, B2, SMTP) ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-api
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: api
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# Redis (in-cluster)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
ports:
|
|
- protocol: TCP
|
|
port: 6379
|
|
# External services: Neon DB (5432), SMTP (587), HTTPS (443 — APNs, FCM, B2, PostHog)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.0.0.0/8
|
|
- 172.16.0.0/12
|
|
- 192.168.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 5432
|
|
- protocol: TCP
|
|
port: 587
|
|
- protocol: TCP
|
|
port: 443
|
|
|
|
---
|
|
# --- Worker: allow egress to Redis, external services ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-worker
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: worker
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# Redis (in-cluster)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
ports:
|
|
- protocol: TCP
|
|
port: 6379
|
|
# External services: Neon DB (5432), SMTP (587), HTTPS (443 — APNs, FCM, B2)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.0.0.0/8
|
|
- 172.16.0.0/12
|
|
- 192.168.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 5432
|
|
- protocol: TCP
|
|
port: 587
|
|
- protocol: TCP
|
|
port: 443
|
|
|
|
---
|
|
# --- Admin: allow egress to API (internal) for SSR ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-admin
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: admin
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# API service (in-cluster, for server-side API calls)
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: api
|
|
ports:
|
|
- protocol: TCP
|
|
port: 8000
|
|
|
|
---
|
|
# --- Web: allow ingress from Traefik (kube-system namespace) ---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-ingress-to-web
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: web
|
|
policyTypes:
|
|
- Ingress
|
|
ingress:
|
|
# Traefik runs as DaemonSet with hostNetwork=true — see allow-ingress-to-api
|
|
# for the rationale. Same ipBlock list.
|
|
- from:
|
|
- ipBlock:
|
|
cidr: 178.105.32.198/32
|
|
- ipBlock:
|
|
cidr: 178.104.247.152/32
|
|
- ipBlock:
|
|
cidr: 178.104.249.189/32
|
|
- ipBlock:
|
|
cidr: 10.42.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 3000
|
|
|
|
---
|
|
# --- Web: allow egress for the Next.js server-side proxy routes ---
|
|
# Browser → app.myhoneydue.com → web pod (Node.js) → api.myhoneydue.com
|
|
# The web pod resolves api.myhoneydue.com via public DNS and hits
|
|
# Cloudflare (143.). We don't know which CF IP yet at policy time, so
|
|
# allow HTTPS to public ipBlock (except private CIDRs).
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-web
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: web
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# HTTPS to public (api.myhoneydue.com via CF, PostHog, any other remote)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.0.0.0/8
|
|
- 172.16.0.0/12
|
|
- 192.168.0.0/16
|
|
ports:
|
|
- protocol: TCP
|
|
port: 443
|
|
|
|
---
|
|
# vmagent egress.
|
|
#
|
|
# IMPORTANT (gotcha): k3s's built-in NetworkPolicy controller appears to
|
|
# evaluate egress rules AFTER kube-proxy's DNAT, not before (contrary to
|
|
# the k8s spec). So traffic from a pod to the kubernetes Service
|
|
# (ClusterIP 10.43.0.1:443) is policy-checked as dst=<node_public_ip>:6443.
|
|
# That's why we need an explicit rule for :6443 to public IPs, even though
|
|
# we already allow :443 to the cluster service CIDR.
|
|
#
|
|
# Without the :6443 rule, vmagent's k8s service discovery silently fails
|
|
# and zero pods get scraped. See deploy-k3s/RUNBOOK.md ("vmagent SD broken").
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-vmagent
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: vmagent
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# DNS (cluster-internal)
|
|
- to:
|
|
- namespaceSelector: {}
|
|
ports:
|
|
- port: 53
|
|
protocol: UDP
|
|
- port: 53
|
|
protocol: TCP
|
|
# k8s API server via ClusterIP (pre-DNAT view)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 10.43.0.0/16
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
# k8s API server post-DNAT (real path k3s NetPol enforcer sees) — REQUIRED
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.42.0.0/16
|
|
ports:
|
|
- port: 6443
|
|
protocol: TCP
|
|
# Scrape api Pods on :8000
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 10.42.0.0/16
|
|
ports:
|
|
- port: 8000
|
|
protocol: TCP
|
|
# Scrape kube-state-metrics Pod on :8080 (pod CIDR)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 10.42.0.0/16
|
|
ports:
|
|
- port: 8080
|
|
protocol: TCP
|
|
# HTTPS to public (remote-write to obs.88oakapps.com via Cloudflare)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.42.0.0/16
|
|
- 10.43.0.0/16
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
|
|
---
|
|
# Allow vmagent → api ingress on :8000 so api pods accept scrapes.
|
|
# api Pods are otherwise locked down by default-deny-all + allow-ingress-to-api
|
|
# (which only allows Traefik). This adds vmagent specifically.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-vmagent-to-api
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: api
|
|
policyTypes:
|
|
- Ingress
|
|
ingress:
|
|
- from:
|
|
- podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: vmagent
|
|
ports:
|
|
- port: 8000
|
|
protocol: TCP
|
|
|
|
---
|
|
# alloy-logs egress — Grafana Alloy discovers honeydue pods via the k8s API
|
|
# and pushes their logs to Loki at obs.88oakapps.com. Same k3s NetworkPolicy
|
|
# DNAT gotcha as vmagent: API-server traffic is policy-checked as
|
|
# dst=<node_public_ip>:6443, so an explicit :6443 rule is required.
|
|
# Alloy reads log FILES from a hostPath, so it needs no ingress and no
|
|
# egress to pod :8000/:8080 — only DNS, the API server, and obs HTTPS.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: allow-egress-from-alloy-logs
|
|
namespace: honeydue
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: alloy-logs
|
|
policyTypes:
|
|
- Egress
|
|
egress:
|
|
# DNS (cluster-internal)
|
|
- to:
|
|
- namespaceSelector: {}
|
|
ports:
|
|
- port: 53
|
|
protocol: UDP
|
|
- port: 53
|
|
protocol: TCP
|
|
# k8s API server via ClusterIP (pre-DNAT view)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 10.43.0.0/16
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
# k8s API server post-DNAT (real path k3s NetPol enforcer sees) — REQUIRED
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.42.0.0/16
|
|
ports:
|
|
- port: 6443
|
|
protocol: TCP
|
|
# HTTPS to public (log push to obs.88oakapps.com via Cloudflare)
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
except:
|
|
- 10.42.0.0/16
|
|
- 10.43.0.0/16
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|