backend: GDPR export + retention cleanups + worker metrics (BE-1/2/3)
Backend CI / Test (push) Has been cancelled
Backend CI / Contract Tests (push) Has been cancelled
Backend CI / Lint (push) Has been cancelled
Backend CI / Secret Scanning (push) Has been cancelled
Backend CI / Build (push) Has been cancelled

BE-3 observability: expose the worker's Prometheus metrics on :6060/metrics
(apns/fcm/asynq histograms + a new cache_ops_total counter were recorded all
along but never scraped — which is why those dashboard panels read empty); add
the worker containerPort, the vmagent worker scrape job, and two additive
NetworkPolicies. Instrument cache Get/Set hit/miss.

BE-2 retention: three periodic Asynq cleanup crons mirroring the reminder-log
cleanup — notifications (90d), webhook dedup log (180d), audit_log (365d).

BE-1 GDPR data export: POST /api/auth/export/ enqueues a low-priority Asynq job
that gathers all of the user's data (owned residences + their tasks/contractors/
documents/share-codes, plus profile/notifications/prefs/push-tokens/subscription/
audit log), zips one JSON file per category, and emails it as an attachment.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
2026-06-08 22:15:26 -05:00
parent 3b2ea9959a
commit b54493f785
14 changed files with 421 additions and 13 deletions
+27
View File
@@ -232,7 +232,11 @@ func main() {
mux.HandleFunc(jobs.TypeOnboardingEmails, jobHandler.HandleOnboardingEmails) mux.HandleFunc(jobs.TypeOnboardingEmails, jobHandler.HandleOnboardingEmails)
mux.HandleFunc(jobs.TypeReminderLogCleanup, jobHandler.HandleReminderLogCleanup) mux.HandleFunc(jobs.TypeReminderLogCleanup, jobHandler.HandleReminderLogCleanup)
mux.HandleFunc(jobs.TypeUploadCleanup, jobHandler.HandleUploadCleanup) mux.HandleFunc(jobs.TypeUploadCleanup, jobHandler.HandleUploadCleanup)
mux.HandleFunc(jobs.TypeNotificationCleanup, jobHandler.HandleNotificationCleanup)
mux.HandleFunc(jobs.TypeWebhookLogCleanup, jobHandler.HandleWebhookLogCleanup)
mux.HandleFunc(jobs.TypeAuditLogCleanup, jobHandler.HandleAuditLogCleanup)
mux.HandleFunc(worker.TypeTaskCompletedNotification, jobHandler.HandleTaskCompletedNotification) mux.HandleFunc(worker.TypeTaskCompletedNotification, jobHandler.HandleTaskCompletedNotification)
mux.HandleFunc(worker.TypeDataExport, jobHandler.HandleDataExport)
// Register email job handlers (welcome, verification, password reset, password changed) // Register email job handlers (welcome, verification, password reset, password changed)
if emailService != nil { if emailService != nil {
@@ -281,6 +285,23 @@ func main() {
} }
log.Info().Str("cron", "30 * * * *").Msg("Registered pending_uploads cleanup job (runs hourly)") log.Info().Str("cron", "30 * * * *").Msg("Registered pending_uploads cleanup job (runs hourly)")
// Data-retention cleanups (BE-2). Staggered off the 3:00 reminder cleanup to
// avoid piling DELETEs onto the same Neon connection window.
if _, err := scheduler.Register("0 2 * * *", asynq.NewTask(jobs.TypeNotificationCleanup, nil)); err != nil {
log.Fatal().Err(err).Msg("Failed to register notification cleanup job")
}
log.Info().Str("cron", "0 2 * * *").Msg("Registered notification cleanup job (daily 02:00 UTC, 90d retention)")
if _, err := scheduler.Register("30 2 * * 0", asynq.NewTask(jobs.TypeWebhookLogCleanup, nil)); err != nil {
log.Fatal().Err(err).Msg("Failed to register webhook log cleanup job")
}
log.Info().Str("cron", "30 2 * * 0").Msg("Registered webhook log cleanup job (weekly Sun 02:30 UTC, 180d retention)")
if _, err := scheduler.Register("30 3 * * 0", asynq.NewTask(jobs.TypeAuditLogCleanup, nil)); err != nil {
log.Fatal().Err(err).Msg("Failed to register audit log cleanup job")
}
log.Info().Str("cron", "30 3 * * 0").Msg("Registered audit log cleanup job (weekly Sun 03:30 UTC, 365d retention)")
// Handle graceful shutdown // Handle graceful shutdown
quit := make(chan os.Signal, 1) quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
@@ -292,6 +313,12 @@ func main() {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"status":"ok"}`)) _, _ = w.Write([]byte(`{"status":"ok"}`))
}) })
// Expose Prometheus metrics so vmagent can scrape the worker. The
// apns_send_*, fcm_send_*, asynq_job_* and cache_ops_* series have been
// recorded on this process all along — they were just never exposed, which
// is why those dashboard panels read empty. Same :6060 as health; in-cluster
// only (not externally published).
healthMux.Handle("/metrics", prom.HTTPHandler())
healthSrv := &http.Server{ healthSrv := &http.Server{
Addr: workerHealthAddr, Addr: workerHealthAddr,
Handler: healthMux, Handler: healthMux,
+19 -12
View File
@@ -78,18 +78,25 @@ data:
- target_label: service - target_label: service
replacement: node-exporter replacement: node-exporter
# honeyDue worker — also exposes /metrics if/when we add it. # honeyDue worker — exposes /metrics on :6060 (apns/fcm/asynq/cache series).
# Keep this stanza commented until the worker has a /metrics endpoint; - job_name: worker
# uncommented form drops scrapes silently. kubernetes_sd_configs:
# - job_name: worker - role: pod
# kubernetes_sd_configs: namespaces:
# - role: pod names: [honeydue]
# namespaces: relabel_configs:
# names: [honeydue] - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
# relabel_configs: action: keep
# - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] regex: worker
# action: keep - source_labels: [__meta_kubernetes_pod_container_port_number]
# regex: worker action: keep
regex: "6060"
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
- target_label: service
replacement: worker
--- ---
apiVersion: v1 apiVersion: v1
@@ -43,6 +43,11 @@ spec:
- name: worker - name: worker
image: IMAGE_PLACEHOLDER # Replaced by 03-deploy.sh image: IMAGE_PLACEHOLDER # Replaced by 03-deploy.sh
imagePullPolicy: IfNotPresent # audit CODE-L4 — explicit; images are SHA/digest-pinned imagePullPolicy: IfNotPresent # audit CODE-L4 — explicit; images are SHA/digest-pinned
ports:
# health + Prometheus /metrics (in-cluster only; scraped by vmagent)
- name: metrics
containerPort: 6060
protocol: TCP
securityContext: securityContext:
allowPrivilegeEscalation: false allowPrivilegeEscalation: false
readOnlyRootFilesystem: true readOnlyRootFilesystem: true
@@ -95,3 +100,46 @@ spec:
- name: tmp - name: tmp
emptyDir: emptyDir:
sizeLimit: 64Mi sizeLimit: 64Mi
---
# Allow vmagent to scrape the worker's /metrics on :6060 (default-deny-all is in
# force; the worker otherwise receives no ingress). Additive — see node-exporter.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-ingress-to-worker-metrics
namespace: honeydue
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: worker
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: vmagent
ports:
- port: 6060
protocol: TCP
---
# vmagent's base egress policy only opens :8000/:8080 to the pod CIDR; this
# additive policy opens :6060 for the worker scrape (leaves the base untouched).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-egress-from-vmagent-to-worker
namespace: honeydue
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: vmagent
policyTypes:
- Egress
egress:
- to:
- ipBlock:
cidr: 10.42.0.0/16
ports:
- port: 6060
protocol: TCP
+34
View File
@@ -12,6 +12,7 @@ import (
"github.com/treytartt/honeydue-api/internal/middleware" "github.com/treytartt/honeydue-api/internal/middleware"
"github.com/treytartt/honeydue-api/internal/services" "github.com/treytartt/honeydue-api/internal/services"
"github.com/treytartt/honeydue-api/internal/validator" "github.com/treytartt/honeydue-api/internal/validator"
"github.com/treytartt/honeydue-api/internal/worker"
) )
// AuthHandler handles user profile and account management endpoints. // AuthHandler handles user profile and account management endpoints.
@@ -23,6 +24,7 @@ type AuthHandler struct {
cache *services.CacheService cache *services.CacheService
storageService *services.StorageService storageService *services.StorageService
auditService *services.AuditService auditService *services.AuditService
enqueuer worker.Enqueuer
} }
// NewAuthHandler creates a new auth handler. // NewAuthHandler creates a new auth handler.
@@ -44,6 +46,38 @@ func (h *AuthHandler) SetAuditService(auditService *services.AuditService) {
h.auditService = auditService h.auditService = auditService
} }
// SetEnqueuer sets the async task enqueuer (used by the GDPR data-export endpoint).
func (h *AuthHandler) SetEnqueuer(enqueuer worker.Enqueuer) {
h.enqueuer = enqueuer
}
// ExportData handles POST /api/auth/export/ — queues a GDPR data-export job that
// emails the user a zip of all their data. Async (202) because gathering,
// zipping, and emailing can take seconds; doing it inline would block the request.
func (h *AuthHandler) ExportData(c echo.Context) error {
noStore(c)
user, err := middleware.MustGetAuthUser(c)
if err != nil {
return err
}
if h.enqueuer == nil {
return echo.NewHTTPError(http.StatusServiceUnavailable, "data export is temporarily unavailable")
}
if err := h.enqueuer.EnqueueDataExport(user.ID); err != nil {
log.Error().Err(err).Uint("user_id", user.ID).Msg("Failed to enqueue data export")
return echo.NewHTTPError(http.StatusInternalServerError, "failed to queue data export")
}
if h.auditService != nil {
h.auditService.LogEvent(c, &user.ID, services.AuditEventDataExport, map[string]interface{}{
"user_id": user.ID,
"email": user.Email,
})
}
return c.JSON(http.StatusAccepted, map[string]string{
"message": "Your data export has been queued. You'll receive an email with your data shortly.",
})
}
// noStore marks a response as non-cacheable. // noStore marks a response as non-cacheable.
func noStore(c echo.Context) { func noStore(c echo.Context) {
c.Response().Header().Set("Cache-Control", "no-store") c.Response().Header().Set("Cache-Control", "no-store")
+21
View File
@@ -1,6 +1,7 @@
package prom package prom
import ( import (
"net/http"
"strconv" "strconv"
"time" "time"
@@ -54,6 +55,11 @@ var (
Help: "Duration of asynq background job execution in seconds.", Help: "Duration of asynq background job execution in seconds.",
Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300}, Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300},
}, []string{"task_type", "result"}) }, []string{"task_type", "result"})
cacheOps = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "cache_ops_total",
Help: "Redis cache operations by type and result.",
}, []string{"operation", "result"}) // operation: get|set; result: hit|miss|ok|error
) )
func init() { func init() {
@@ -67,6 +73,7 @@ func init() {
apnsSendDuration, apnsSendDuration,
fcmSendDuration, fcmSendDuration,
asynqJobDuration, asynqJobDuration,
cacheOps,
) )
} }
@@ -77,6 +84,20 @@ func Handler() echo.HandlerFunc {
return echo.WrapHandler(h) return echo.WrapHandler(h)
} }
// HTTPHandler returns a net/http handler bound to the package Registry, for the
// worker's plain http.ServeMux (the api uses Handler() for Echo). This is what
// lets the worker's apns/fcm/asynq histograms actually get scraped — they were
// recorded all along but the worker exposed no /metrics endpoint.
func HTTPHandler() http.Handler {
return promhttp.HandlerFor(Registry, promhttp.HandlerOpts{Registry: Registry})
}
// ObserveCacheOp records a Redis cache operation. operation is "get" or "set";
// result is "hit"/"miss"/"error" for gets and "ok"/"error" for sets.
func ObserveCacheOp(operation, result string) {
cacheOps.WithLabelValues(operation, result).Inc()
}
// HTTPMiddleware records http_request_duration_seconds for every request, // HTTPMiddleware records http_request_duration_seconds for every request,
// labeled by Echo route pattern, method, and status code. // labeled by Echo route pattern, method, and status code.
func HTTPMiddleware() echo.MiddlewareFunc { func HTTPMiddleware() echo.MiddlewareFunc {
+4
View File
@@ -270,6 +270,9 @@ func SetupRouter(deps *Dependencies) *echo.Echo {
authHandler := handlers.NewAuthHandler(authService, deps.EmailService, deps.Cache) authHandler := handlers.NewAuthHandler(authService, deps.EmailService, deps.Cache)
authHandler.SetStorageService(deps.StorageService) authHandler.SetStorageService(deps.StorageService)
authHandler.SetAuditService(auditService) authHandler.SetAuditService(auditService)
if deps.TaskEnqueuer != nil {
authHandler.SetEnqueuer(deps.TaskEnqueuer)
}
userHandler := handlers.NewUserHandler(userService) userHandler := handlers.NewUserHandler(userService)
residenceHandler := handlers.NewResidenceHandler(residenceService, deps.PDFService, deps.EmailService, cfg.Features.PDFReportsEnabled) residenceHandler := handlers.NewResidenceHandler(residenceService, deps.PDFService, deps.EmailService, cfg.Features.PDFReportsEnabled)
taskHandler := handlers.NewTaskHandler(taskService, deps.StorageService) taskHandler := handlers.NewTaskHandler(taskService, deps.StorageService)
@@ -559,6 +562,7 @@ func setupProtectedAuthRoutes(api *echo.Group, authHandler *handlers.AuthHandler
auth.PUT("/profile/", authHandler.UpdateProfile) auth.PUT("/profile/", authHandler.UpdateProfile)
auth.PATCH("/profile/", authHandler.UpdateProfile) auth.PATCH("/profile/", authHandler.UpdateProfile)
auth.DELETE("/account/", authHandler.DeleteAccount) auth.DELETE("/account/", authHandler.DeleteAccount)
auth.POST("/export/", authHandler.ExportData)
} }
} }
+1
View File
@@ -19,6 +19,7 @@ const (
AuditEventPasswordReset = "auth.password_reset" AuditEventPasswordReset = "auth.password_reset"
AuditEventPasswordChanged = "auth.password_changed" AuditEventPasswordChanged = "auth.password_changed"
AuditEventAccountDeleted = "auth.account_deleted" AuditEventAccountDeleted = "auth.account_deleted"
AuditEventDataExport = "auth.data_export_requested"
) )
// AuditService handles audit logging for security-relevant events. // AuditService handles audit logging for security-relevant events.
+15 -1
View File
@@ -3,6 +3,7 @@ package services
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"hash/fnv" "hash/fnv"
"sync" "sync"
@@ -13,6 +14,7 @@ import (
"github.com/treytartt/honeydue-api/internal/config" "github.com/treytartt/honeydue-api/internal/config"
"github.com/treytartt/honeydue-api/internal/i18n" "github.com/treytartt/honeydue-api/internal/i18n"
"github.com/treytartt/honeydue-api/internal/prom"
) )
// CacheService provides Redis caching functionality // CacheService provides Redis caching functionality
@@ -93,16 +95,28 @@ func (c *CacheService) Set(ctx context.Context, key string, value interface{}, e
return fmt.Errorf("failed to marshal value: %w", err) return fmt.Errorf("failed to marshal value: %w", err)
} }
return c.client.Set(ctx, key, data, expiration).Err() err = c.client.Set(ctx, key, data, expiration).Err()
if err != nil {
prom.ObserveCacheOp("set", "error")
} else {
prom.ObserveCacheOp("set", "ok")
}
return err
} }
// Get retrieves a value by key // Get retrieves a value by key
func (c *CacheService) Get(ctx context.Context, key string, dest interface{}) error { func (c *CacheService) Get(ctx context.Context, key string, dest interface{}) error {
data, err := c.client.Get(ctx, key).Bytes() data, err := c.client.Get(ctx, key).Bytes()
if err != nil { if err != nil {
if errors.Is(err, redis.Nil) {
prom.ObserveCacheOp("get", "miss")
} else {
prom.ObserveCacheOp("get", "error")
}
return err return err
} }
prom.ObserveCacheOp("get", "hit")
return json.Unmarshal(data, dest) return json.Unmarshal(data, dest)
} }
+6
View File
@@ -12,11 +12,17 @@ type Enqueuer interface {
EnqueuePasswordResetEmail(to, firstName, code, resetToken string) error EnqueuePasswordResetEmail(to, firstName, code, resetToken string) error
EnqueuePasswordChangedEmail(to, firstName string) error EnqueuePasswordChangedEmail(to, firstName string) error
EnqueueTaskCompletedNotification(taskID, completionID uint) error EnqueueTaskCompletedNotification(taskID, completionID uint) error
EnqueueDataExport(userID uint) error
} }
// Verify TaskClient satisfies the interface at compile time. // Verify TaskClient satisfies the interface at compile time.
var _ Enqueuer = (*TaskClient)(nil) var _ Enqueuer = (*TaskClient)(nil)
// BuildDataExportPayload marshals a DataExportPayload to JSON bytes.
func BuildDataExportPayload(userID uint) ([]byte, error) {
return json.Marshal(DataExportPayload{UserID: userID})
}
// BuildWelcomeEmailPayload marshals a WelcomeEmailPayload to JSON bytes. // BuildWelcomeEmailPayload marshals a WelcomeEmailPayload to JSON bytes.
func BuildWelcomeEmailPayload(to, firstName, code string) ([]byte, error) { func BuildWelcomeEmailPayload(to, firstName, code string) ([]byte, error) {
return json.Marshal(WelcomeEmailPayload{ return json.Marshal(WelcomeEmailPayload{
+69
View File
@@ -0,0 +1,69 @@
package jobs
import (
"context"
"time"
"github.com/hibiken/asynq"
"github.com/rs/zerolog/log"
"github.com/treytartt/honeydue-api/internal/models"
"github.com/treytartt/honeydue-api/internal/repositories"
)
// Data-retention cleanup job types. Registered as periodic crons in
// cmd/worker/main.go. These keep transient/log tables from growing unbounded;
// none touch user-facing data that the app reads back.
const (
TypeNotificationCleanup = "maintenance:notification_cleanup"
TypeWebhookLogCleanup = "maintenance:webhook_log_cleanup"
TypeAuditLogCleanup = "maintenance:audit_log_cleanup"
)
// Retention windows (days).
const (
notificationRetentionDays = 90
webhookLogRetentionDays = 180
auditLogRetentionDays = 365 // keep 1 year of security events
)
// HandleNotificationCleanup deletes notification rows older than the retention
// window. Notifications are delivery records (push/digest history); 90 days is
// ample for any in-app history a client might show.
func (h *Handler) HandleNotificationCleanup(ctx context.Context, _ *asynq.Task) error {
cutoff := time.Now().UTC().AddDate(0, 0, -notificationRetentionDays)
res := h.db.WithContext(ctx).Where("created_at < ?", cutoff).Delete(&models.Notification{})
if res.Error != nil {
log.Error().Err(res.Error).Msg("notification cleanup failed")
return res.Error
}
log.Info().Int64("deleted", res.RowsAffected).Int("retention_days", notificationRetentionDays).Msg("notification cleanup completed")
return nil
}
// HandleWebhookLogCleanup prunes the webhook dedup log. Rows only matter for the
// window in which a provider (Apple/Google) might redeliver an event; 180 days
// is a generous safety margin past any real redelivery.
func (h *Handler) HandleWebhookLogCleanup(ctx context.Context, _ *asynq.Task) error {
cutoff := time.Now().UTC().AddDate(0, 0, -webhookLogRetentionDays)
res := h.db.WithContext(ctx).Where("processed_at < ?", cutoff).Delete(&repositories.WebhookEvent{})
if res.Error != nil {
log.Error().Err(res.Error).Msg("webhook log cleanup failed")
return res.Error
}
log.Info().Int64("deleted", res.RowsAffected).Int("retention_days", webhookLogRetentionDays).Msg("webhook log cleanup completed")
return nil
}
// HandleAuditLogCleanup prunes audit events older than the retention window.
// One year of security events is retained for compliance/forensics.
func (h *Handler) HandleAuditLogCleanup(ctx context.Context, _ *asynq.Task) error {
cutoff := time.Now().UTC().AddDate(0, 0, -auditLogRetentionDays)
res := h.db.WithContext(ctx).Where("created_at < ?", cutoff).Delete(&models.AuditLog{})
if res.Error != nil {
log.Error().Err(res.Error).Msg("audit log cleanup failed")
return res.Error
}
log.Info().Int64("deleted", res.RowsAffected).Int("retention_days", auditLogRetentionDays).Msg("audit log cleanup completed")
return nil
}
+138
View File
@@ -0,0 +1,138 @@
package jobs
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
"time"
"github.com/hibiken/asynq"
"github.com/rs/zerolog/log"
"github.com/treytartt/honeydue-api/internal/models"
"github.com/treytartt/honeydue-api/internal/services"
"github.com/treytartt/honeydue-api/internal/worker"
)
// HandleDataExport gathers all of a user's data (GDPR data portability), zips it
// as one JSON file per category, and emails the archive as an attachment.
// Triggered by POST /api/auth/export/ -> Enqueuer.EnqueueDataExport.
//
// Residence-scoped data (tasks, contractors, documents, share codes) covers only
// residences the user OWNS — shared residences belong to their owner and are
// intentionally excluded. Document/photo *files* are referenced by URL (in
// documents.json); the bytes live in B2 and aren't inlined.
func (h *Handler) HandleDataExport(ctx context.Context, task *asynq.Task) error {
var payload worker.DataExportPayload
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
log.Error().Err(err).Msg("data export: malformed payload")
return asynq.SkipRetry
}
db := h.db.WithContext(ctx)
var user models.User
if err := db.First(&user, payload.UserID).Error; err != nil {
log.Error().Err(err).Uint("user_id", payload.UserID).Msg("data export: user not found")
return err
}
if h.emailService == nil {
log.Warn().Uint("user_id", payload.UserID).Msg("data export: email service unavailable; cannot deliver")
return nil // retrying won't help a structurally-disabled mailer
}
var ownedIDs []uint
db.Model(&models.Residence{}).Where("owner_id = ?", payload.UserID).Pluck("id", &ownedIDs)
var (
profile []models.UserProfile
residences []models.Residence
tasks []models.Task
contractors []models.Contractor
documents []models.Document
shareCodes []models.ResidenceShareCode
notifs []models.Notification
notifPrefs []models.NotificationPreference
apnsDevices []models.APNSDevice
gcmDevices []models.GCMDevice
subscription []models.UserSubscription
auditLog []models.AuditLog
)
db.Where("user_id = ?", payload.UserID).Find(&profile)
db.Where("owner_id = ?", payload.UserID).Find(&residences)
if len(ownedIDs) > 0 {
db.Where("residence_id IN ?", ownedIDs).Find(&tasks)
db.Where("residence_id IN ?", ownedIDs).Find(&contractors)
db.Where("residence_id IN ?", ownedIDs).Find(&documents)
db.Where("residence_id IN ?", ownedIDs).Find(&shareCodes)
}
db.Where("user_id = ?", payload.UserID).Find(&notifs)
db.Where("user_id = ?", payload.UserID).Find(&notifPrefs)
db.Where("user_id = ?", payload.UserID).Find(&apnsDevices)
db.Where("user_id = ?", payload.UserID).Find(&gcmDevices)
db.Where("user_id = ?", payload.UserID).Find(&subscription)
db.Where("user_id = ?", payload.UserID).Find(&auditLog)
sections := []struct {
name string
data interface{}
}{
{"account", user},
{"profile", profile},
{"residences", residences},
{"tasks", tasks},
{"contractors", contractors},
{"documents", documents},
{"share_codes", shareCodes},
{"notifications", notifs},
{"notification_preferences", notifPrefs},
{"push_tokens_ios", apnsDevices},
{"push_tokens_android", gcmDevices},
{"subscription", subscription},
{"audit_log", auditLog},
}
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
readme := fmt.Sprintf("honeyDue data export\nGenerated: %s UTC\nAccount: %s\n\n"+
"One JSON file per data category. Residence-scoped data covers residences you own.\n"+
"Document and photo files are referenced by URL in documents.json.\n",
time.Now().UTC().Format(time.RFC3339), user.Email)
if w, err := zw.Create("README.txt"); err == nil {
_, _ = w.Write([]byte(readme))
}
for _, s := range sections {
w, err := zw.Create(s.name + ".json")
if err != nil {
_ = zw.Close()
return fmt.Errorf("data export: zip create %s: %w", s.name, err)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
if err := enc.Encode(s.data); err != nil {
_ = zw.Close()
return fmt.Errorf("data export: encode %s: %w", s.name, err)
}
}
if err := zw.Close(); err != nil {
return fmt.Errorf("data export: finalize zip: %w", err)
}
subject := "Your honeyDue data export"
text := "Attached is a copy of your honeyDue data, as a zip of JSON files.\n" +
"If you didn't request this, you can ignore this email.\n"
html := "<p>Attached is a copy of your honeyDue data, as a zip of JSON files.</p>" +
"<p>If you didn't request this, you can ignore this email.</p>"
attach := &services.EmailAttachment{
Filename: "honeydue-data-export.zip",
ContentType: "application/zip",
Data: buf.Bytes(),
}
if err := h.emailService.SendEmailWithAttachment(user.Email, subject, html, text, attach); err != nil {
log.Error().Err(err).Uint("user_id", payload.UserID).Msg("data export: email send failed")
return err
}
log.Info().Uint("user_id", payload.UserID).Int("zip_bytes", buf.Len()).Msg("data export emailed")
return nil
}
+8
View File
@@ -12,6 +12,7 @@ import (
"github.com/treytartt/honeydue-api/internal/config" "github.com/treytartt/honeydue-api/internal/config"
"github.com/treytartt/honeydue-api/internal/models" "github.com/treytartt/honeydue-api/internal/models"
"github.com/treytartt/honeydue-api/internal/repositories" "github.com/treytartt/honeydue-api/internal/repositories"
"github.com/treytartt/honeydue-api/internal/services"
) )
// --- Mock implementations --- // --- Mock implementations ---
@@ -27,6 +28,13 @@ func (m *mockEmailSender) SendEmail(to, subject, htmlBody, textBody string) erro
return nil return nil
} }
func (m *mockEmailSender) SendEmailWithAttachment(to, subject, htmlBody, textBody string, _ *services.EmailAttachment) error {
if m.sendFn != nil {
return m.sendFn(to, subject, htmlBody, textBody)
}
return nil
}
type mockPushSender struct { type mockPushSender struct {
sendFn func(ctx context.Context, iosTokens, androidTokens []string, title, message string, data map[string]string) error sendFn func(ctx context.Context, iosTokens, androidTokens []string, title, message string, data map[string]string) error
} }
+2
View File
@@ -6,6 +6,7 @@ import (
"github.com/treytartt/honeydue-api/internal/models" "github.com/treytartt/honeydue-api/internal/models"
"github.com/treytartt/honeydue-api/internal/repositories" "github.com/treytartt/honeydue-api/internal/repositories"
"github.com/treytartt/honeydue-api/internal/services"
) )
// TaskRepo defines task query operations needed by job handlers. // TaskRepo defines task query operations needed by job handlers.
@@ -46,6 +47,7 @@ type PushSender interface {
// EmailSender sends emails. // EmailSender sends emails.
type EmailSender interface { type EmailSender interface {
SendEmail(to, subject, htmlBody, textBody string) error SendEmail(to, subject, htmlBody, textBody string) error
SendEmailWithAttachment(to, subject, htmlBody, textBody string, attachment *services.EmailAttachment) error
} }
// OnboardingEmailSender sends onboarding campaign emails. // OnboardingEmailSender sends onboarding campaign emails.
+29
View File
@@ -21,8 +21,17 @@ const (
// Moves the ~1-1.5s of synchronous APNs+SMTP+B2-fetch work out of the // Moves the ~1-1.5s of synchronous APNs+SMTP+B2-fetch work out of the
// POST /api/task-completions/ request path. // POST /api/task-completions/ request path.
TypeTaskCompletedNotification = "notification:task_completed" TypeTaskCompletedNotification = "notification:task_completed"
// TypeDataExport is emitted by POST /api/auth/export/. The worker gathers
// all of the user's data into a zip and emails it (GDPR data portability).
TypeDataExport = "user:data_export"
) )
// DataExportPayload carries just the user id; the worker re-fetches all rows.
type DataExportPayload struct {
UserID uint `json:"user_id"`
}
// TaskCompletedNotificationPayload carries only the IDs needed for the // TaskCompletedNotificationPayload carries only the IDs needed for the
// worker to re-fetch the canonical Task + TaskCompletion rows. Keeping the // worker to re-fetch the canonical Task + TaskCompletion rows. Keeping the
// payload to IDs (vs. full model graphs) keeps the Redis queue cheap and // payload to IDs (vs. full model graphs) keeps the Redis queue cheap and
@@ -93,6 +102,26 @@ func (c *TaskClient) EnqueueWelcomeEmail(to, firstName, code string) error {
return nil return nil
} }
// EnqueueDataExport enqueues a GDPR data-export task for a user. The worker
// gathers the user's data, zips it, and emails it. Low priority — there's no
// rush, and it shouldn't compete with notifications for the critical queue.
func (c *TaskClient) EnqueueDataExport(userID uint) error {
payload, err := BuildDataExportPayload(userID)
if err != nil {
return err
}
task := asynq.NewTask(TypeDataExport, payload)
_, err = c.client.Enqueue(task, asynq.Queue("low"), asynq.MaxRetry(3))
if err != nil {
log.Error().Err(err).Uint("user_id", userID).Msg("Failed to enqueue data export")
return err
}
log.Info().Uint("user_id", userID).Msg("Data export task enqueued")
return nil
}
// EnqueueVerificationEmail enqueues a verification email task // EnqueueVerificationEmail enqueues a verification email task
func (c *TaskClient) EnqueueVerificationEmail(to, firstName, code string) error { func (c *TaskClient) EnqueueVerificationEmail(to, firstName, code string) error {
payload, err := BuildVerificationEmailPayload(to, firstName, code) payload, err := BuildVerificationEmailPayload(to, firstName, code)