honeyDueAPI/deploy-k3s-dev/scripts/04-verify.sh

#!/usr/bin/env bash
set -euo pipefail

NAMESPACE="honeydue"

log() { printf '[verify] %s\n' "$*"; }
sep() { printf '\n%s\n' "--- $1 ---"; }
ok() { printf '[verify] ✓ %s\n' "$*"; }
fail() { printf '[verify] ✗ %s\n' "$*"; }

command -v kubectl >/dev/null 2>&1 || { echo "Missing: kubectl" >&2; exit 1; }

sep "Node"
kubectl get nodes -o wide

sep "Pods"
kubectl get pods -n "${NAMESPACE}" -o wide

sep "Services"
kubectl get svc -n "${NAMESPACE}"

sep "Ingress"
kubectl get ingress -n "${NAMESPACE}"

sep "PVCs"
kubectl get pvc -n "${NAMESPACE}"

sep "Secrets (names only)"
kubectl get secrets -n "${NAMESPACE}"

sep "ConfigMap keys"
kubectl get configmap honeydue-config -n "${NAMESPACE}" -o jsonpath='{.data}' 2>/dev/null | python3 -c "
import json, sys
try:
    d = json.load(sys.stdin)
    for k in sorted(d.keys()):
        v = d[k]
        if any(s in k.upper() for s in ['PASSWORD', 'SECRET', 'TOKEN', 'KEY']):
            v = '***REDACTED***'
        print(f'  {k}={v}')
except:
    print('  (could not parse)')
" 2>/dev/null || log "ConfigMap not found or not parseable"

sep "Warning Events (last 15 min)"
kubectl get events -n "${NAMESPACE}" --field-selector type=Warning --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || log "No warning events"

sep "Pod Restart Counts"
kubectl get pods -n "${NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .status.containerStatuses[*]}{.restartCount}{end}{"\n"}{end}' 2>/dev/null || true

# =============================================================================
# Infrastructure Health
# =============================================================================

sep "PostgreSQL Health"
PG_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -n "${PG_POD}" ]]; then
  kubectl exec -n "${NAMESPACE}" "${PG_POD}" -- pg_isready -U honeydue 2>/dev/null && ok "PostgreSQL is ready" || fail "PostgreSQL is NOT ready"
else
  fail "No PostgreSQL pod found"
fi

sep "Redis Health"
REDIS_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -n "${REDIS_POD}" ]]; then
  kubectl exec -n "${NAMESPACE}" "${REDIS_POD}" -- sh -c 'if [ -n "$REDIS_PASSWORD" ]; then redis-cli -a "$REDIS_PASSWORD" ping 2>/dev/null; else redis-cli ping; fi' 2>/dev/null | grep -q PONG && ok "Redis is ready" || fail "Redis is NOT ready"
else
  fail "No Redis pod found"
fi

sep "MinIO Health"
MINIO_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -n "${MINIO_POD}" ]]; then
  kubectl exec -n "${NAMESPACE}" "${MINIO_POD}" -- curl -sf http://localhost:9000/minio/health/ready 2>/dev/null && ok "MinIO is ready" || fail "MinIO is NOT ready"
else
  fail "No MinIO pod found"
fi

sep "API Health Check"
API_POD="$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -n "${API_POD}" ]]; then
  kubectl exec -n "${NAMESPACE}" "${API_POD}" -- curl -sf http://localhost:8000/api/health/ 2>/dev/null && ok "API health check passed" || fail "API health check FAILED"
else
  fail "No API pod found"
fi

sep "Resource Usage"
kubectl top pods -n "${NAMESPACE}" 2>/dev/null || log "Metrics server not available (install with: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml)"

# =============================================================================
# Security Verification
# =============================================================================

sep "Security: Network Policies"
NP_COUNT="$(kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | wc -l | tr -d ' ')"
if (( NP_COUNT >= 5 )); then
  ok "Found ${NP_COUNT} network policies"
  kubectl get networkpolicy -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do
    echo "  ${line}"
  done
else
  fail "Expected 5+ network policies, found ${NP_COUNT}"
fi

sep "Security: Service Accounts"
SA_COUNT="$(kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | grep -cv default | tr -d ' ')"
if (( SA_COUNT >= 6 )); then
  ok "Found ${SA_COUNT} custom service accounts (api, worker, admin, redis, postgres, minio)"
else
  fail "Expected 6 custom service accounts, found ${SA_COUNT}"
fi
kubectl get sa -n "${NAMESPACE}" --no-headers 2>/dev/null | while read -r line; do
  echo "  ${line}"
done

sep "Security: Pod Security Contexts"
PODS_WITHOUT_SECURITY="$(kubectl get pods -n "${NAMESPACE}" -o json 2>/dev/null | python3 -c "
import json, sys
try:
    data = json.load(sys.stdin)
    issues = []
    for pod in data.get('items', []):
        name = pod['metadata']['name']
        spec = pod['spec']
        sc = spec.get('securityContext', {})
        # Postgres is exempt from runAsNonRoot (entrypoint requirement)
        is_postgres = any('postgres' in c.get('image', '') for c in spec.get('containers', []))
        if not sc.get('runAsNonRoot') and not is_postgres:
            issues.append(f'{name}: missing runAsNonRoot')
        for c in spec.get('containers', []):
            csc = c.get('securityContext', {})
            if csc.get('allowPrivilegeEscalation', True):
                issues.append(f'{name}/{c[\"name\"]}: allowPrivilegeEscalation not false')
    if issues:
        for i in issues:
            print(i)
    else:
        print('OK')
except Exception as e:
    print(f'Error: {e}')
" 2>/dev/null || echo "Error parsing pod specs")"

if [[ "${PODS_WITHOUT_SECURITY}" == "OK" ]]; then
  ok "All pods have proper security contexts"
else
  fail "Pod security context issues:"
  echo "${PODS_WITHOUT_SECURITY}" | while read -r line; do
    echo "    ${line}"
  done
fi

sep "Security: Admin Basic Auth"
ADMIN_AUTH="$(kubectl get secret admin-basic-auth -n "${NAMESPACE}" -o name 2>/dev/null || true)"
if [[ -n "${ADMIN_AUTH}" ]]; then
  ok "admin-basic-auth secret exists"
else
  fail "admin-basic-auth secret not found — admin panel has no additional auth layer"
fi

echo ""
log "Verification complete."