#!/usr/bin/env bash set -euo pipefail NAMESPACE="honeydue" log() { printf '[verify] %s\n' "$*"; } sep() { printf '\n%s\n' "--- $1 ---"; } ok() { printf '[verify] ✓ %s\n' "$*"; } fail() { printf '[verify] ✗ %s\n' "$*"; } command -v kubectl >/dev/null 2>&1 || { echo "Missing: kubectl" >&2; exit 1; } sep "Node" kubectl get nodes -o wide sep "Pods" kubectl get pods -n "${NAMESPACE}" -o wide sep "Services" kubectl get svc -n "${NAMESPACE}" sep "Ingress" kubectl get ingress -n "${NAMESPACE}" sep "PVCs" kubectl get pvc -n "${NAMESPACE}" sep "Secrets (names only)" kubectl get secrets -n "${NAMESPACE}" sep "ConfigMap keys" kubectl get configmap honeydue-config -n "${NAMESPACE}" -o jsonpath='{.data}' 2>/dev/null | python3 -c " import json, sys try: d = json.load(sys.stdin) for k in sorted(d.keys()): v = d[k] if any(s in k.upper() for s in ['PASSWORD', 'SECRET', 'TOKEN', 'KEY']): v = '***REDACTED***' print(f' {k}={v}') except: print(' (could not parse)') " 2>/dev/null || log "ConfigMap not found or not parseable" sep "Warning Events (last 15 min)" kubectl get events -n "${NAMESPACE}" --field-selector type=Warning --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || log "No warning events" sep "Pod Restart Counts" kubectl get pods -n "${NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' 2>/dev/null || true # ============================================================================= # Infrastructure Health # ============================================================================= sep "PostgreSQL Health" PG_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "${PG_POD}" ]]; then kubectl exec -n "${NAMESPACE}" "${PG_POD}" -- pg_isready -U honeydue 2>/dev/null && ok "PostgreSQL is ready" || fail "PostgreSQL is NOT ready" else fail "No PostgreSQL pod found" fi sep "Redis Health" REDIS_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "${REDIS_POD}" ]]; then kubectl exec -n "${NAMESPACE}" "${REDIS_POD}" -- sh -c 'if [ -n "$REDIS_PASSWORD" ]; then redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null; else redis-cli ping; fi' 2>/dev/null | grep -q PONG && ok "Redis is ready" || fail "Redis is NOT ready" else fail "No Redis pod found" fi sep "MinIO Health" MINIO_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "${MINIO_POD}" ]]; then kubectl exec -n "${NAMESPACE}" "${MINIO_POD}" -- curl -sf http://localhost:9000/minio/health/ready 2>/dev/null && ok "MinIO is ready" || fail "MinIO is NOT ready" else fail "No MinIO pod found" fi sep "API Health Check" API_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "${API_POD}" ]]; then kubectl exec -n "${NAMESPACE}" "${API_POD}" -- curl -sf http://localhost:8000/api/health/ 2>/dev/null && ok "API health check passed" || fail "API health check FAILED" else fail "No API pod found" fi sep "Resource Usage" kubectl top pods -n "${NAMESPACE}" 2>/dev/null || log "Metrics server not available (install with: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml)" # ============================================================================= # Security Verification # ============================================================================= sep "Security: Network Policies" NP_COUNT="$(kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')" if (( NP_COUNT >= 5 )); then ok "Found ${NP_COUNT} network policies" kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do echo " ${line}" done else fail "Expected 5+ network policies, found ${NP_COUNT}" fi sep "Security: Service Accounts" SA_COUNT="$(kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | grep -cv default | tr -d ' ')" if (( SA_COUNT >= 6 )); then ok "Found ${SA_COUNT} custom service accounts (api, worker, admin, redis, postgres, minio)" else fail "Expected 6 custom service accounts, found ${SA_COUNT}" fi kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do echo " ${line}" done sep "Security: Pod Security Contexts" PODS_WITHOUT_SECURITY="$(kubectl get pods -n "${NAMESPACE}" -o json 2>/dev/null | python3 -c " import json, sys try: data = json.load(sys.stdin) issues = [] for pod in data.get('items', []): name = pod['metadata']['name'] spec = pod['spec'] sc = spec.get('securityContext', {}) # Postgres is exempt from runAsNonRoot (entrypoint requirement) is_postgres = any('postgres' in c.get('image', '') for c in spec.get('containers', [])) if not sc.get('runAsNonRoot') and not is_postgres: issues.append(f'{name}: missing runAsNonRoot') for c in spec.get('containers', []): csc = c.get('securityContext', {}) if csc.get('allowPrivilegeEscalation', True): issues.append(f'{name}/{c[\"name\"]}: allowPrivilegeEscalation not false') if issues: for i in issues: print(i) else: print('OK') except Exception as e: print(f'Error: {e}') " 2>/dev/null || echo "Error parsing pod specs")" if [[ "${PODS_WITHOUT_SECURITY}" == "OK" ]]; then ok "All pods have proper security contexts" else fail "Pod security context issues:" echo "${PODS_WITHOUT_SECURITY}" | while read -r line; do echo " ${line}" done fi sep "Security: Admin Basic Auth" ADMIN_AUTH="$(kubectl get secret admin-basic-auth -n "${NAMESPACE}" -o name 2>/dev/null || true)" if [[ -n "${ADMIN_AUTH}" ]]; then ok "admin-basic-auth secret exists" else fail "admin-basic-auth secret not found — admin panel has no additional auth layer" fi echo "" log "Verification complete."