Production hardening: security, resilience, observability, and compliance

Password complexity: custom validator requiring uppercase, lowercase, digit (min 8 chars)
Token expiry: 90-day token lifetime with refresh endpoint (60-90 day renewal window)
Health check: /api/health/ now pings Postgres + Redis, returns 503 on failure
Audit logging: async audit_log table for auth events (login, register, delete, etc.)
Circuit breaker: APNs/FCM push sends wrapped with 5-failure threshold, 30s recovery
FK indexes: 27 missing foreign key indexes across all tables (migration 017)
CSP header: default-src 'none'; frame-ancestors 'none'
Gzip compression: level 5 with media endpoint skipper
Prometheus metrics: /metrics endpoint using existing monitoring service
External timeouts: 15s push, 30s SMTP, context timeouts on all external calls

Migrations: 016 (token created_at), 017 (FK indexes), 018 (audit_log)
Tests: circuit breaker (15), audit service (8), token refresh (7), health (4),
       middleware expiry (5), validator (new)

internal/push/circuit_breaker_test.go

@@ -0,0 +1,275 @@
+package push
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCircuitBreaker_StartsInClosedState(t *testing.T) {
+	cb := NewCircuitBreaker("test")
+	assert.Equal(t, "closed", cb.State())
+	assert.True(t, cb.Allow())
+}
+
+func TestCircuitBreaker_OpensAfterThresholdFailures(t *testing.T) {
+	cb := NewCircuitBreaker("test", WithFailureThreshold(3))
+
+	// First two failures should keep it closed
+	cb.RecordFailure()
+	assert.Equal(t, "closed", cb.State())
+	assert.True(t, cb.Allow())
+
+	cb.RecordFailure()
+	assert.Equal(t, "closed", cb.State())
+	assert.True(t, cb.Allow())
+
+	// Third failure should open it
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+	assert.False(t, cb.Allow())
+}
+
+func TestCircuitBreaker_DefaultThresholdIsFive(t *testing.T) {
+	cb := NewCircuitBreaker("test")
+
+	for i := 0; i < 4; i++ {
+		cb.RecordFailure()
+		assert.Equal(t, "closed", cb.State())
+	}
+
+	cb.RecordFailure() // 5th failure
+	assert.Equal(t, "open", cb.State())
+}
+
+func TestCircuitBreaker_RejectsRequestsWhenOpen(t *testing.T) {
+	cb := NewCircuitBreaker("test", WithFailureThreshold(1))
+
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+
+	// Multiple calls should all be rejected
+	for i := 0; i < 10; i++ {
+		assert.False(t, cb.Allow())
+	}
+}
+
+func TestCircuitBreaker_TransitionsToHalfOpenAfterRecoveryTimeout(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(1),
+		WithRecoveryTimeout(50*time.Millisecond),
+	)
+
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+	assert.False(t, cb.Allow())
+
+	// Wait for recovery timeout
+	time.Sleep(60 * time.Millisecond)
+
+	// Should now allow one request (half-open)
+	assert.True(t, cb.Allow())
+	assert.Equal(t, "half-open", cb.State())
+}
+
+func TestCircuitBreaker_HalfOpenRejectsSecondRequest(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(1),
+		WithRecoveryTimeout(50*time.Millisecond),
+	)
+
+	cb.RecordFailure()
+	time.Sleep(60 * time.Millisecond)
+
+	// First request allowed (probe)
+	assert.True(t, cb.Allow())
+	assert.Equal(t, "half-open", cb.State())
+
+	// Second request rejected while probe is in flight
+	assert.False(t, cb.Allow())
+}
+
+func TestCircuitBreaker_HalfOpenSuccess_ResetsToClosed(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(1),
+		WithRecoveryTimeout(50*time.Millisecond),
+	)
+
+	cb.RecordFailure()
+	time.Sleep(60 * time.Millisecond)
+
+	// Probe request
+	assert.True(t, cb.Allow())
+
+	// Probe succeeds
+	cb.RecordSuccess()
+	assert.Equal(t, "closed", cb.State())
+	assert.Equal(t, 0, cb.Counts())
+
+	// Normal operation resumes
+	assert.True(t, cb.Allow())
+	assert.True(t, cb.Allow())
+}
+
+func TestCircuitBreaker_HalfOpenFailure_ReturnsToOpen(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(2),
+		WithRecoveryTimeout(50*time.Millisecond),
+	)
+
+	// Open the circuit
+	cb.RecordFailure()
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+
+	time.Sleep(60 * time.Millisecond)
+
+	// Probe request
+	assert.True(t, cb.Allow())
+	assert.Equal(t, "half-open", cb.State())
+
+	// Probe fails - the failure count is now 3 which is >= threshold of 2
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+	assert.False(t, cb.Allow())
+}
+
+func TestCircuitBreaker_SuccessResetsFailureCount(t *testing.T) {
+	cb := NewCircuitBreaker("test", WithFailureThreshold(3))
+
+	cb.RecordFailure()
+	cb.RecordFailure()
+	assert.Equal(t, 2, cb.Counts())
+
+	// A success should reset the counter
+	cb.RecordSuccess()
+	assert.Equal(t, 0, cb.Counts())
+	assert.Equal(t, "closed", cb.State())
+
+	// Now it should take 3 more failures to open
+	cb.RecordFailure()
+	cb.RecordFailure()
+	assert.Equal(t, "closed", cb.State())
+
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+}
+
+func TestCircuitBreaker_Reset(t *testing.T) {
+	cb := NewCircuitBreaker("test", WithFailureThreshold(1))
+
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+
+	cb.Reset()
+	assert.Equal(t, "closed", cb.State())
+	assert.Equal(t, 0, cb.Counts())
+	assert.True(t, cb.Allow())
+}
+
+func TestCircuitBreaker_Name(t *testing.T) {
+	cb := NewCircuitBreaker("apns-breaker")
+	assert.Equal(t, "apns-breaker", cb.Name())
+}
+
+func TestCircuitBreaker_CustomOptions(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(10),
+		WithRecoveryTimeout(5*time.Minute),
+	)
+
+	// Should take 10 failures to open
+	for i := 0; i < 9; i++ {
+		cb.RecordFailure()
+		assert.Equal(t, "closed", cb.State())
+	}
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+}
+
+func TestCircuitBreaker_InvalidOptionsIgnored(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(0),  // Should be ignored (keeps default)
+		WithRecoveryTimeout(-1),  // Should be ignored (keeps default)
+	)
+
+	// Default threshold of 5 should still apply
+	for i := 0; i < 4; i++ {
+		cb.RecordFailure()
+		assert.Equal(t, "closed", cb.State())
+	}
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+}
+
+func TestCircuitBreaker_ThreadSafety(t *testing.T) {
+	cb := NewCircuitBreaker("test",
+		WithFailureThreshold(100),
+		WithRecoveryTimeout(10*time.Millisecond),
+	)
+
+	var wg sync.WaitGroup
+	const goroutines = 50
+	const iterations = 100
+
+	// Hammer it from many goroutines
+	for i := 0; i < goroutines; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			for j := 0; j < iterations; j++ {
+				cb.Allow()
+				if j%2 == 0 {
+					cb.RecordFailure()
+				} else {
+					cb.RecordSuccess()
+				}
+				_ = cb.State()
+				_ = cb.Counts()
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Should not panic or deadlock. State should be valid.
+	state := cb.State()
+	require.Contains(t, []string{"closed", "open", "half-open"}, state)
+}
+
+func TestCircuitBreaker_FullLifecycle(t *testing.T) {
+	cb := NewCircuitBreaker("lifecycle-test",
+		WithFailureThreshold(3),
+		WithRecoveryTimeout(50*time.Millisecond),
+	)
+
+	// 1. Closed: requests flow normally
+	assert.True(t, cb.Allow())
+	cb.RecordSuccess()
+	assert.Equal(t, "closed", cb.State())
+
+	// 2. Accumulate failures
+	cb.RecordFailure()
+	cb.RecordFailure()
+	assert.Equal(t, "closed", cb.State())
+
+	// 3. Third failure opens the circuit
+	cb.RecordFailure()
+	assert.Equal(t, "open", cb.State())
+	assert.False(t, cb.Allow())
+
+	// 4. Wait for recovery
+	time.Sleep(60 * time.Millisecond)
+
+	// 5. Half-open: probe request allowed
+	assert.True(t, cb.Allow())
+	assert.Equal(t, "half-open", cb.State())
+
+	// 6. Probe succeeds, back to closed
+	cb.RecordSuccess()
+	assert.Equal(t, "closed", cb.State())
+	assert.True(t, cb.Allow())
+}