Files
honeyDueAPI/internal/push/circuit_breaker_test.go
Trey T b679f28e55 Production hardening: security, resilience, observability, and compliance
Password complexity: custom validator requiring uppercase, lowercase, digit (min 8 chars)
Token expiry: 90-day token lifetime with refresh endpoint (60-90 day renewal window)
Health check: /api/health/ now pings Postgres + Redis, returns 503 on failure
Audit logging: async audit_log table for auth events (login, register, delete, etc.)
Circuit breaker: APNs/FCM push sends wrapped with 5-failure threshold, 30s recovery
FK indexes: 27 missing foreign key indexes across all tables (migration 017)
CSP header: default-src 'none'; frame-ancestors 'none'
Gzip compression: level 5 with media endpoint skipper
Prometheus metrics: /metrics endpoint using existing monitoring service
External timeouts: 15s push, 30s SMTP, context timeouts on all external calls

Migrations: 016 (token created_at), 017 (FK indexes), 018 (audit_log)
Tests: circuit breaker (15), audit service (8), token refresh (7), health (4),
       middleware expiry (5), validator (new)
2026-03-26 14:05:28 -05:00

276 lines
6.3 KiB
Go

package push
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCircuitBreaker_StartsInClosedState(t *testing.T) {
cb := NewCircuitBreaker("test")
assert.Equal(t, "closed", cb.State())
assert.True(t, cb.Allow())
}
func TestCircuitBreaker_OpensAfterThresholdFailures(t *testing.T) {
cb := NewCircuitBreaker("test", WithFailureThreshold(3))
// First two failures should keep it closed
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
assert.True(t, cb.Allow())
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
assert.True(t, cb.Allow())
// Third failure should open it
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
assert.False(t, cb.Allow())
}
func TestCircuitBreaker_DefaultThresholdIsFive(t *testing.T) {
cb := NewCircuitBreaker("test")
for i := 0; i < 4; i++ {
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
}
cb.RecordFailure() // 5th failure
assert.Equal(t, "open", cb.State())
}
func TestCircuitBreaker_RejectsRequestsWhenOpen(t *testing.T) {
cb := NewCircuitBreaker("test", WithFailureThreshold(1))
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
// Multiple calls should all be rejected
for i := 0; i < 10; i++ {
assert.False(t, cb.Allow())
}
}
func TestCircuitBreaker_TransitionsToHalfOpenAfterRecoveryTimeout(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(1),
WithRecoveryTimeout(50*time.Millisecond),
)
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
assert.False(t, cb.Allow())
// Wait for recovery timeout
time.Sleep(60 * time.Millisecond)
// Should now allow one request (half-open)
assert.True(t, cb.Allow())
assert.Equal(t, "half-open", cb.State())
}
func TestCircuitBreaker_HalfOpenRejectsSecondRequest(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(1),
WithRecoveryTimeout(50*time.Millisecond),
)
cb.RecordFailure()
time.Sleep(60 * time.Millisecond)
// First request allowed (probe)
assert.True(t, cb.Allow())
assert.Equal(t, "half-open", cb.State())
// Second request rejected while probe is in flight
assert.False(t, cb.Allow())
}
func TestCircuitBreaker_HalfOpenSuccess_ResetsToClosed(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(1),
WithRecoveryTimeout(50*time.Millisecond),
)
cb.RecordFailure()
time.Sleep(60 * time.Millisecond)
// Probe request
assert.True(t, cb.Allow())
// Probe succeeds
cb.RecordSuccess()
assert.Equal(t, "closed", cb.State())
assert.Equal(t, 0, cb.Counts())
// Normal operation resumes
assert.True(t, cb.Allow())
assert.True(t, cb.Allow())
}
func TestCircuitBreaker_HalfOpenFailure_ReturnsToOpen(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(2),
WithRecoveryTimeout(50*time.Millisecond),
)
// Open the circuit
cb.RecordFailure()
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
time.Sleep(60 * time.Millisecond)
// Probe request
assert.True(t, cb.Allow())
assert.Equal(t, "half-open", cb.State())
// Probe fails - the failure count is now 3 which is >= threshold of 2
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
assert.False(t, cb.Allow())
}
func TestCircuitBreaker_SuccessResetsFailureCount(t *testing.T) {
cb := NewCircuitBreaker("test", WithFailureThreshold(3))
cb.RecordFailure()
cb.RecordFailure()
assert.Equal(t, 2, cb.Counts())
// A success should reset the counter
cb.RecordSuccess()
assert.Equal(t, 0, cb.Counts())
assert.Equal(t, "closed", cb.State())
// Now it should take 3 more failures to open
cb.RecordFailure()
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
}
func TestCircuitBreaker_Reset(t *testing.T) {
cb := NewCircuitBreaker("test", WithFailureThreshold(1))
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
cb.Reset()
assert.Equal(t, "closed", cb.State())
assert.Equal(t, 0, cb.Counts())
assert.True(t, cb.Allow())
}
func TestCircuitBreaker_Name(t *testing.T) {
cb := NewCircuitBreaker("apns-breaker")
assert.Equal(t, "apns-breaker", cb.Name())
}
func TestCircuitBreaker_CustomOptions(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(10),
WithRecoveryTimeout(5*time.Minute),
)
// Should take 10 failures to open
for i := 0; i < 9; i++ {
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
}
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
}
func TestCircuitBreaker_InvalidOptionsIgnored(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(0), // Should be ignored (keeps default)
WithRecoveryTimeout(-1), // Should be ignored (keeps default)
)
// Default threshold of 5 should still apply
for i := 0; i < 4; i++ {
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
}
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
}
func TestCircuitBreaker_ThreadSafety(t *testing.T) {
cb := NewCircuitBreaker("test",
WithFailureThreshold(100),
WithRecoveryTimeout(10*time.Millisecond),
)
var wg sync.WaitGroup
const goroutines = 50
const iterations = 100
// Hammer it from many goroutines
for i := 0; i < goroutines; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for j := 0; j < iterations; j++ {
cb.Allow()
if j%2 == 0 {
cb.RecordFailure()
} else {
cb.RecordSuccess()
}
_ = cb.State()
_ = cb.Counts()
}
}(i)
}
wg.Wait()
// Should not panic or deadlock. State should be valid.
state := cb.State()
require.Contains(t, []string{"closed", "open", "half-open"}, state)
}
func TestCircuitBreaker_FullLifecycle(t *testing.T) {
cb := NewCircuitBreaker("lifecycle-test",
WithFailureThreshold(3),
WithRecoveryTimeout(50*time.Millisecond),
)
// 1. Closed: requests flow normally
assert.True(t, cb.Allow())
cb.RecordSuccess()
assert.Equal(t, "closed", cb.State())
// 2. Accumulate failures
cb.RecordFailure()
cb.RecordFailure()
assert.Equal(t, "closed", cb.State())
// 3. Third failure opens the circuit
cb.RecordFailure()
assert.Equal(t, "open", cb.State())
assert.False(t, cb.Allow())
// 4. Wait for recovery
time.Sleep(60 * time.Millisecond)
// 5. Half-open: probe request allowed
assert.True(t, cb.Allow())
assert.Equal(t, "half-open", cb.State())
// 6. Probe succeeds, back to closed
cb.RecordSuccess()
assert.Equal(t, "closed", cb.State())
assert.True(t, cb.Allow())
}