Add clear stuck jobs admin feature and simplify worker scheduling

- Add POST /api/admin/settings/clear-stuck-jobs endpoint to clear
  stuck/failed asynq worker jobs from Redis (retry queue, archived,
  orphaned task metadata)
- Add "Clear Stuck Jobs" button to admin settings UI
- Remove TASK_REMINDER_MINUTE config - all jobs now run at minute 0
- Simplify formatCron to only take hour parameter
- Update default notification times to CST-friendly hours

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Trey t
2025-12-06 23:05:48 -06:00
parent 3b9e37d12b
commit af87bd943e
6 changed files with 150 additions and 17 deletions

View File

@@ -467,6 +467,75 @@ type ClearAllDataResponse struct {
PreservedUsers int64 `json:"preserved_users"`
}
// ClearStuckJobsResponse represents the response after clearing stuck Redis jobs
type ClearStuckJobsResponse struct {
Message string `json:"message"`
KeysDeleted int `json:"keys_deleted"`
DeletedKeys []string `json:"deleted_keys"`
}
// ClearStuckJobs handles POST /api/admin/settings/clear-stuck-jobs
// This clears stuck/failed asynq worker jobs from Redis
func (h *AdminSettingsHandler) ClearStuckJobs(c *gin.Context) {
cache := services.GetCache()
if cache == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Redis cache not available"})
return
}
ctx := c.Request.Context()
client := cache.Client()
var deletedKeys []string
// Patterns for asynq job keys that can get stuck
patterns := []string{
"asynq:{default}:retry", // Retry queue
"asynq:{default}:archived", // Archived/dead jobs
"asynq:{default}:t:*", // Individual task metadata
}
for _, pattern := range patterns {
if strings.Contains(pattern, "*") {
// Use SCAN for patterns with wildcards
iter := client.Scan(ctx, 0, pattern, 0).Iterator()
for iter.Next(ctx) {
key := iter.Val()
if err := client.Del(ctx, key).Err(); err != nil {
log.Warn().Err(err).Str("key", key).Msg("Failed to delete key")
continue
}
deletedKeys = append(deletedKeys, key)
}
if err := iter.Err(); err != nil {
log.Warn().Err(err).Str("pattern", pattern).Msg("Error scanning keys")
}
} else {
// Direct key deletion
exists, err := client.Exists(ctx, pattern).Result()
if err != nil {
log.Warn().Err(err).Str("key", pattern).Msg("Failed to check key existence")
continue
}
if exists > 0 {
if err := client.Del(ctx, pattern).Err(); err != nil {
log.Warn().Err(err).Str("key", pattern).Msg("Failed to delete key")
continue
}
deletedKeys = append(deletedKeys, pattern)
}
}
}
log.Info().Int("count", len(deletedKeys)).Strs("keys", deletedKeys).Msg("Cleared stuck Redis jobs")
c.JSON(http.StatusOK, ClearStuckJobsResponse{
Message: "Stuck jobs cleared successfully",
KeysDeleted: len(deletedKeys),
DeletedKeys: deletedKeys,
})
}
// ClearAllData handles POST /api/admin/settings/clear-all-data
// This clears all data except super admin accounts and lookup tables
func (h *AdminSettingsHandler) ClearAllData(c *gin.Context) {

View File

@@ -342,6 +342,7 @@ func SetupRoutes(router *gin.Engine, db *gorm.DB, cfg *config.Config, deps *Depe
settings.POST("/seed-test-data", settingsHandler.SeedTestData)
settings.POST("/seed-task-templates", settingsHandler.SeedTaskTemplates)
settings.POST("/clear-all-data", settingsHandler.ClearAllData)
settings.POST("/clear-stuck-jobs", settingsHandler.ClearStuckJobs)
}
// Limitations management (tier limits, upgrade triggers)