Cut /api/tasks/ p99 from ~2500ms toward ~150-300ms
Backend CI / Test (push) Has been cancelled
Backend CI / Contract Tests (push) Has been cancelled
Backend CI / Build (push) Has been cancelled
Backend CI / Lint (push) Has been cancelled
Backend CI / Secret Scanning (push) Has been cancelled

Stack of optimizations against the same Hetzner→Neon transatlantic link.
The trace revealed every visible ms was network/proxy overhead — DB
execution itself is sub-millisecond per query (verified via EXPLAIN
ANALYZE: index scans on every hot path).

Connection layer:
- DB_HOST → Neon pooler endpoint (-pooler suffix). PgBouncer
  transaction-mode keeps backend Postgres connections warm so we no
  longer pay the ~110ms Postgres-startup RTT on cold queries.
- GORM pool tuned: MaxIdleConns 10→20, MaxLifetime 600s→1800s,
  MaxIdleTime added (default 0 = never close idle).
- Eager pool warm-up at boot via parallel pings — first user request
  no longer pays the ~440ms TCP+TLS+startup handshake.
- Redis maxmemory-policy noeviction → allkeys-lru. Cache writes will
  evict cold keys instead of erroring at the 256MB limit.

Auth layer:
- TokenCacheTTL 5min → 1 hour (Redis token cache).
- UserCacheTTL 30s → 5min (in-memory User cache, per pod).
- UserCache gains a 5,000-entry LRU cap so a flood of unique users
  can't blow up pod RSS. ~5MB worst-case per pod.
- Token + user lookup collapsed from 2 GORM Preload queries into a
  single INNER JOIN. Saves 1 RTT per cold-cache request.
- Auth middleware's m.db.* now use db.WithContext(ctx) so the SQL
  spans nest under the parent HTTP request in Jaeger.

Service layer:
- TaskService.ListTasks: replaced two-step
  FindResidenceIDsByUser → GetKanbanDataForMultipleResidences
  with a single GetKanbanDataForUser that uses a Postgres subquery
  for residence-access. One round-trip instead of two.
- New CacheService residence-IDs cache: \"residence_ids_user:<id>\"
  with 5-min TTL. Wired into Task/Residence/Contractor/Document
  services for the four hot read paths that need this list.
- Cache invalidation on every relevant mutation: CreateResidence,
  DeleteResidence, JoinWithCode, RemoveUser. DeleteResidence
  invalidates every member of the residence, not just the owner.

What this stacks up to (Hetzner→Neon, before US migration):
  Path                                 Before        After (target)
  Cache-warm authed read               ~800ms        ~100-200ms
  Cache-cold authed read (1st in 1hr)  ~2500ms       ~500-700ms
  First request after deploy           ~2500ms       ~700-900ms

The endgame US-region migration on top of this gets us to ~30-50ms
warm-cache, but we're shippable at ~150ms warm right now.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey t
2026-04-25 17:13:50 -05:00
parent 9410da7497
commit 88fb1751c7
15 changed files with 443 additions and 59 deletions
+85 -13
View File
@@ -22,13 +22,22 @@ const (
AuthUserKey = "auth_user"
// AuthTokenKey is the key used to store the token in the context
AuthTokenKey = "auth_token"
// TokenCacheTTL is the duration to cache tokens in Redis
TokenCacheTTL = 5 * time.Minute
// TokenCacheTTL is the duration to cache tokens in Redis. Tokens are
// valid for DefaultTokenExpiryDays (90), and explicit logout invalidates
// the cache, so a long TTL here just means most authed requests skip the
// auth-token SQL query entirely.
TokenCacheTTL = 1 * time.Hour
// TokenCachePrefix is the prefix for token cache keys
TokenCachePrefix = "auth_token_"
// UserCacheTTL is how long full user records are cached in memory to
// avoid hitting the database on every authenticated request.
UserCacheTTL = 30 * time.Second
// avoid hitting the database on every authenticated request. Bumped from
// 30s — at 30s the trace showed a SELECT auth_user query on most warm
// requests because users aren't in cache long enough to hit twice.
UserCacheTTL = 5 * time.Minute
// UserCacheMaxSize bounds the per-pod in-memory user cache. With ~1KB
// per User struct, 5000 entries = ~5MB per pod. Older entries are
// evicted LRU before the limit is exceeded.
UserCacheMaxSize = 5000
// DefaultTokenExpiryDays is the default number of days before a token expires.
DefaultTokenExpiryDays = 90
@@ -47,7 +56,7 @@ func NewAuthMiddleware(db *gorm.DB, cache *services.CacheService) *AuthMiddlewar
return &AuthMiddleware{
db: db,
cache: cache,
userCache: NewUserCache(UserCacheTTL),
userCache: NewUserCache(UserCacheTTL, UserCacheMaxSize),
tokenExpiryDays: DefaultTokenExpiryDays,
}
}
@@ -61,7 +70,7 @@ func NewAuthMiddlewareWithConfig(db *gorm.DB, cache *services.CacheService, cfg
return &AuthMiddleware{
db: db,
cache: cache,
userCache: NewUserCache(UserCacheTTL),
userCache: NewUserCache(UserCacheTTL, UserCacheMaxSize),
tokenExpiryDays: expiryDays,
}
}
@@ -244,20 +253,83 @@ func (m *AuthMiddleware) getUserFromCache(ctx context.Context, token string) (*m
// getUserFromDatabaseWithToken looks up the token in the database and returns
// both the user and the auth token record (for expiry checking). The ctx is
// threaded into the GORM session so the SQL span attaches to the request trace.
//
// Uses a single JOIN query instead of GORM's Preload (which issues 2 SELECTs).
// Over a transatlantic link this saves ~110ms RTT per cache miss.
func (m *AuthMiddleware) getUserFromDatabaseWithToken(ctx context.Context, token string) (*models.User, *models.AuthToken, error) {
var authToken models.AuthToken
if err := m.db.WithContext(ctx).Preload("User").Where("key = ?", token).First(&authToken).Error; err != nil {
// Flat result row: every column from auth_user prefixed `u_`, every
// column from user_authtoken left in its native shape. Mapping to two
// structs is mechanical so we don't need a struct tag soup.
type joinedRow struct {
// AuthToken columns
Key string `gorm:"column:key"`
Created time.Time `gorm:"column:created"`
UserID uint `gorm:"column:user_id"`
// User columns (prefixed to avoid collision with UserID)
UID uint `gorm:"column:u_id"`
UUsername string `gorm:"column:u_username"`
UEmail string `gorm:"column:u_email"`
UFirstName string `gorm:"column:u_first_name"`
ULastName string `gorm:"column:u_last_name"`
UPassword string `gorm:"column:u_password"`
UIsActive bool `gorm:"column:u_is_active"`
UIsStaff bool `gorm:"column:u_is_staff"`
UIsSuper bool `gorm:"column:u_is_superuser"`
UDateJoined time.Time `gorm:"column:u_date_joined"`
ULastLogin *time.Time `gorm:"column:u_last_login"`
}
var row joinedRow
err := m.db.WithContext(ctx).
Table("user_authtoken AS t").
Select(`
t.key, t.created, t.user_id,
u.id AS u_id,
u.username AS u_username,
u.email AS u_email,
u.first_name AS u_first_name,
u.last_name AS u_last_name,
u.password AS u_password,
u.is_active AS u_is_active,
u.is_staff AS u_is_staff,
u.is_superuser AS u_is_superuser,
u.date_joined AS u_date_joined,
u.last_login AS u_last_login
`).
Joins("INNER JOIN auth_user u ON u.id = t.user_id").
Where("t.key = ?", token).
Limit(1).
Scan(&row).Error
if err != nil || row.Key == "" {
return nil, nil, fmt.Errorf("token not found")
}
// Check if user is active
if !authToken.User.IsActive {
user := models.User{
ID: row.UID,
Username: row.UUsername,
Email: row.UEmail,
FirstName: row.UFirstName,
LastName: row.ULastName,
Password: row.UPassword,
IsActive: row.UIsActive,
IsStaff: row.UIsStaff,
IsSuperuser: row.UIsSuper,
DateJoined: row.UDateJoined,
LastLogin: row.ULastLogin,
}
authToken := models.AuthToken{
Key: row.Key,
Created: row.Created,
UserID: row.UserID,
User: user,
}
if !user.IsActive {
return nil, nil, fmt.Errorf("user is inactive")
}
// Store in in-memory cache for subsequent requests
m.userCache.Set(&authToken.User)
return &authToken.User, &authToken, nil
m.userCache.Set(&user)
return &user, &authToken, nil
}
// getUserFromDatabase looks up the token in the database and caches the