honeyDueAPI/internal/database/database.go

package database

import (
	"context"
	"fmt"
	"time"

	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
	"golang.org/x/crypto/bcrypt"
	"gorm.io/driver/postgres"
	"gorm.io/gorm"
	"gorm.io/gorm/logger"

	"github.com/treytartt/honeydue-api/internal/config"
	"github.com/treytartt/honeydue-api/internal/models"
	"github.com/treytartt/honeydue-api/internal/prom"

	"github.com/uptrace/opentelemetry-go-extra/otelgorm"
)

// zerologGormWriter adapts zerolog for GORM's logger interface
type zerologGormWriter struct{}

func (w zerologGormWriter) Printf(format string, args ...interface{}) {
	log.Warn().Msgf(format, args...)
}

var db *gorm.DB

// Connect establishes a connection to the PostgreSQL database
func Connect(cfg *config.DatabaseConfig, debug bool) (*gorm.DB, error) {
	// Configure GORM logger with slow query detection
	logLevel := logger.Silent
	if debug {
		logLevel = logger.Info
	}

	gormLogger := logger.New(
		zerologGormWriter{},
		logger.Config{
			SlowThreshold:             200 * time.Millisecond,
			LogLevel:                  logLevel,
			IgnoreRecordNotFoundError: true,
		},
	)

	gormConfig := &gorm.Config{
		Logger: gormLogger,
		NowFunc: func() time.Time {
			return time.Now().UTC()
		},
		PrepareStmt: true,
	}

	// Connect to database
	var err error
	db, err = gorm.Open(postgres.Open(cfg.DSN()), gormConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to database: %w", err)
	}

	// Get underlying sql.DB for connection pool settings
	sqlDB, err := db.DB()
	if err != nil {
		return nil, fmt.Errorf("failed to get underlying sql.DB: %w", err)
	}

	// Configure connection pool. The Neon pooler endpoint keeps backend
	// connections warm, so we keep our client-side pool warm too — that
	// eliminates the ~440ms TCP+TLS+startup handshake on the first query
	// after a cold pod / idle period.
	sqlDB.SetMaxOpenConns(cfg.MaxOpenConns)
	sqlDB.SetMaxIdleConns(cfg.MaxIdleConns)
	sqlDB.SetConnMaxLifetime(cfg.MaxLifetime)
	if cfg.MaxIdleTime > 0 {
		sqlDB.SetConnMaxIdleTime(cfg.MaxIdleTime)
	}
	// MaxIdleTime=0 means "never close idle" — the pool fills up to
	// MaxIdleConns and they stay alive until MaxLifetime expires.

	// Test connection
	if err := sqlDB.Ping(); err != nil {
		return nil, fmt.Errorf("failed to ping database: %w", err)
	}

	// Eagerly warm the connection pool to MaxIdleConns. Without this, the
	// first N user requests each pay the full handshake (~440ms over a
	// transatlantic link). Pings are issued in parallel so warm-up is
	// bounded by handshake time, not handshake-time × N.
	warmUpPool(sqlDB, cfg.MaxIdleConns)

	log.Info().
		Str("host", cfg.Host).
		Int("port", cfg.Port).
		Str("database", cfg.Database).
		Msg("Connected to PostgreSQL database")

	// Register Prometheus GORM callbacks — emits gorm_query_duration_seconds
	// for every SQL operation. Operates at the statement level, so does not
	// require ctx to be threaded through repositories.
	if err := prom.RegisterGORMCallbacks(db); err != nil {
		log.Warn().Err(err).Msg("failed to register prometheus GORM callbacks; metrics will be partial")
	}

	// Register otelgorm plugin — emits a span per SQL statement, attached to
	// whatever trace context is set via db.WithContext(ctx). Repositories that
	// have been migrated to use WithContext (see internal/repositories/*.go)
	// will produce nested SQL spans inside the request trace; pre-migration
	// repositories silently emit untraced queries.
	if err := db.Use(otelgorm.NewPlugin(otelgorm.WithDBName(cfg.Database))); err != nil {
		log.Warn().Err(err).Msg("failed to register otelgorm plugin; SQL spans disabled")
	}

	return db, nil
}

// warmUpPool issues N parallel pings so the pool fills with established
// connections before the first user request lands. Failures are logged but
// not fatal — the pool will fill on demand under traffic if pre-warm fails.
//
// On a transatlantic link to Neon (~110ms RTT, ~440ms cold handshake), this
// turns "first request pays the cold handshake" into "first request finds a
// warm pool" — at the cost of ~440ms during pod startup.
func warmUpPool(sqlDB interface {
	PingContext(context.Context) error
}, n int) {
	if n <= 0 {
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	done := make(chan error, n)
	for i := 0; i < n; i++ {
		go func() { done <- sqlDB.PingContext(ctx) }()
	}
	successes := 0
	for i := 0; i < n; i++ {
		if err := <-done; err == nil {
			successes++
		}
	}
	log.Info().Int("requested", n).Int("warmed", successes).Msg("DB pool warm-up complete")
}

// Get returns the database instance
func Get() *gorm.DB {
	return db
}

// Close closes the database connection
func Close() error {
	if db != nil {
		sqlDB, err := db.DB()
		if err != nil {
			return err
		}
		return sqlDB.Close()
	}
	return nil
}

// WithTransaction executes a function within a database transaction
func WithTransaction(fn func(tx *gorm.DB) error) error {
	return db.Transaction(fn)
}

// Paginate returns a GORM scope for pagination
func Paginate(page, pageSize int) func(db *gorm.DB) *gorm.DB {
	return func(db *gorm.DB) *gorm.DB {
		if page <= 0 {
			page = 1
		}
		if pageSize <= 0 {
			pageSize = 100
		}
		if pageSize > 1000 {
			pageSize = 1000
		}

		offset := (page - 1) * pageSize
		return db.Offset(offset).Limit(pageSize)
	}
}

// RequireSchemaApplied verifies that goose's version table exists and has
// at least one applied entry. This is the fail-fast that runs at api/worker
// boot: if the operator forgot to run the migrate Job, the pod refuses to
// start with a clear error instead of throwing mysterious "relation does
// not exist" errors deep in a request handler.
//
// On non-Postgres dialects (sqlite in tests) this is a no-op — tests use
// AutoMigrate via testutil.SetupTestDB to create a fresh schema per run.
// goose isn't involved in the test path.
func RequireSchemaApplied() error {
	if db == nil {
		return fmt.Errorf("database not initialised")
	}
	if db.Dialector.Name() != "postgres" {
		return nil
	}

	// goose_db_version stores one row per applied migration, not a single
	// "current version" row — so we look for the highest version_id with
	// is_applied=true. ORDER BY id DESC LIMIT 1 also catches the case where
	// the table exists but is empty (no rows returned, scan leaves Version
	// at zero).
	type migrationRow struct {
		VersionID int64 `gorm:"column:version_id"`
		IsApplied bool  `gorm:"column:is_applied"`
	}

	var row migrationRow
	err := db.Raw(`SELECT version_id, is_applied FROM goose_db_version ORDER BY id DESC LIMIT 1`).Scan(&row).Error
	if err != nil {
		return fmt.Errorf("goose_db_version check failed (run the migrate Job to bootstrap): %w", err)
	}
	if !row.IsApplied {
		return fmt.Errorf("goose_db_version latest row is_applied=false at version=%d — last migration was rolled back or aborted; investigate before starting", row.VersionID)
	}
	if row.VersionID < 1 {
		return fmt.Errorf("goose_db_version is empty — run goose up (or seed a row marking version 1 as applied if the schema already exists)")
	}
	log.Info().Int64("schema_version", row.VersionID).Msg("Schema precondition satisfied")
	return nil
}

// Migrate runs database migrations for all models
func Migrate() error {
	log.Info().Msg("Running database migrations...")

	// Migrate all models in order (respecting foreign key constraints)
	err := db.AutoMigrate(
		// Lookup tables first (no foreign keys)
		&models.ResidenceType{},
		&models.TaskCategory{},
		&models.TaskPriority{},
		&models.TaskFrequency{},
		&models.ContractorSpecialty{},
		&models.TaskTemplate{},     // Task templates reference category and frequency
		&models.ClimateRegion{},    // IECC climate regions for regional templates
		&models.ZipClimateRegion{}, // ZIP to climate region lookup

		// User and auth tables
		&models.User{},
		&models.UserProfile{},

		// Admin users (separate from app users)
		&models.AdminUser{},

		// Main entity tables (order matters for foreign keys!)
		&models.Residence{},
		&models.ResidenceShareCode{},
		&models.Contractor{},  // Contractor before Task (Task references Contractor)
		&models.Task{},
		&models.TaskCompletion{},
		&models.TaskCompletionImage{}, // Multiple images per completion
		&models.Document{},
		&models.DocumentImage{}, // Multiple images per document

		// Notification tables
		&models.Notification{},
		&models.NotificationPreference{},
		&models.APNSDevice{},
		&models.GCMDevice{},
		&models.TaskReminderLog{}, // Smart reminder tracking

		// Subscription tables
		&models.SubscriptionSettings{},
		&models.UserSubscription{},
		&models.UpgradeTrigger{},
		&models.FeatureBenefit{},
		&models.Promotion{},
		&models.TierLimits{},

		// Onboarding email tracking
		&models.OnboardingEmail{},
	)

	if err != nil {
		return fmt.Errorf("failed to run migrations: %w", err)
	}

	// Add unique constraint for onboarding emails (one email type per user)
	db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_onboarding_emails_user_type ON onboarding_emails(user_id, email_type)`)

	// Run GoAdmin migrations
	if err := migrateGoAdmin(); err != nil {
		return fmt.Errorf("failed to run GoAdmin migrations: %w", err)
	}

	// Run one-time data migrations (backfills, etc.)
	if err := RunDataMigrations(); err != nil {
		return fmt.Errorf("failed to run data migrations: %w", err)
	}

	log.Info().Msg("Database migrations completed successfully")
	return nil
}

// migrateGoAdmin creates GoAdmin required tables
func migrateGoAdmin() error {
	log.Info().Msg("Running GoAdmin migrations...")

	// GoAdmin session table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_session (
			id SERIAL PRIMARY KEY,
			sid VARCHAR(50) NOT NULL DEFAULT '',
			"values" TEXT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_session_sid ON goadmin_session(sid)`)

	// GoAdmin users table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_users (
			id SERIAL PRIMARY KEY,
			username VARCHAR(100) NOT NULL DEFAULT '',
			password VARCHAR(100) NOT NULL DEFAULT '',
			name VARCHAR(100) NOT NULL DEFAULT '',
			avatar VARCHAR(255) DEFAULT '',
			remember_token VARCHAR(100) DEFAULT '',
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_goadmin_users_username ON goadmin_users(username)`)

	// GoAdmin roles table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_roles (
			id SERIAL PRIMARY KEY,
			name VARCHAR(50) NOT NULL DEFAULT '',
			slug VARCHAR(50) NOT NULL DEFAULT '',
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_goadmin_roles_slug ON goadmin_roles(slug)`)

	// GoAdmin permissions table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_permissions (
			id SERIAL PRIMARY KEY,
			name VARCHAR(50) NOT NULL DEFAULT '',
			slug VARCHAR(50) NOT NULL DEFAULT '',
			http_method VARCHAR(255) DEFAULT '',
			http_path TEXT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_goadmin_permissions_slug ON goadmin_permissions(slug)`)

	// GoAdmin role_users table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_role_users (
			id SERIAL PRIMARY KEY,
			role_id INT NOT NULL,
			user_id INT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_users_role_id ON goadmin_role_users(role_id)`)
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_users_user_id ON goadmin_role_users(user_id)`)

	// GoAdmin user_permissions table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_user_permissions (
			id SERIAL PRIMARY KEY,
			user_id INT NOT NULL,
			permission_id INT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_user_permissions_user_id ON goadmin_user_permissions(user_id)`)
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_user_permissions_permission_id ON goadmin_user_permissions(permission_id)`)

	// GoAdmin role_permissions table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_role_permissions (
			id SERIAL PRIMARY KEY,
			role_id INT NOT NULL,
			permission_id INT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_permissions_role_id ON goadmin_role_permissions(role_id)`)
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_permissions_permission_id ON goadmin_role_permissions(permission_id)`)

	// GoAdmin menu table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_menu (
			id SERIAL PRIMARY KEY,
			parent_id INT NOT NULL DEFAULT 0,
			type INT NOT NULL DEFAULT 0,
			"order" INT NOT NULL DEFAULT 0,
			title VARCHAR(50) NOT NULL DEFAULT '',
			icon VARCHAR(50) NOT NULL DEFAULT '',
			uri VARCHAR(3000) NOT NULL DEFAULT '',
			header VARCHAR(150) DEFAULT '',
			plugin_name VARCHAR(150) NOT NULL DEFAULT '',
			uuid VARCHAR(150) DEFAULT '',
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_menu_parent_id ON goadmin_menu(parent_id)`)

	// GoAdmin role_menu table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_role_menu (
			id SERIAL PRIMARY KEY,
			role_id INT NOT NULL,
			menu_id INT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_menu_role_id ON goadmin_role_menu(role_id)`)
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_role_menu_menu_id ON goadmin_role_menu(menu_id)`)

	// GoAdmin operation_log table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_operation_log (
			id SERIAL PRIMARY KEY,
			user_id INT NOT NULL,
			path VARCHAR(255) NOT NULL DEFAULT '',
			method VARCHAR(10) NOT NULL DEFAULT '',
			ip VARCHAR(15) NOT NULL DEFAULT '',
			input TEXT NOT NULL,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_operation_log_user_id ON goadmin_operation_log(user_id)`)

	// GoAdmin site table
	if err := db.Exec(`
		CREATE TABLE IF NOT EXISTS goadmin_site (
			id SERIAL PRIMARY KEY,
			key VARCHAR(100) NOT NULL DEFAULT '',
			value TEXT NOT NULL,
			description VARCHAR(3000) DEFAULT '',
			state INT NOT NULL DEFAULT 0,
			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	`).Error; err != nil {
		return err
	}
	db.Exec(`CREATE INDEX IF NOT EXISTS idx_goadmin_site_key ON goadmin_site(key)`)

	// Seed default GoAdmin user only on first run (ON CONFLICT DO NOTHING).
	// Password is NOT reset on subsequent migrations to preserve operator changes.
	goAdminUsername := viper.GetString("GOADMIN_ADMIN_USERNAME")
	goAdminPassword := viper.GetString("GOADMIN_ADMIN_PASSWORD")
	if goAdminUsername == "" || goAdminPassword == "" {
		log.Warn().Msg("GOADMIN_ADMIN_USERNAME and/or GOADMIN_ADMIN_PASSWORD not set; skipping GoAdmin admin user seed")
	} else {
		goAdminHash, err := bcrypt.GenerateFromPassword([]byte(goAdminPassword), bcrypt.DefaultCost)
		if err != nil {
			return fmt.Errorf("failed to hash GoAdmin admin password: %w", err)
		}
		db.Exec(`
			INSERT INTO goadmin_users (username, password, name, avatar)
			VALUES (?, ?, 'Administrator', '')
			ON CONFLICT DO NOTHING
		`, goAdminUsername, string(goAdminHash))
	}

	// Seed default roles
	db.Exec(`INSERT INTO goadmin_roles (name, slug) VALUES ('Administrator', 'administrator') ON CONFLICT DO NOTHING`)
	db.Exec(`INSERT INTO goadmin_roles (name, slug) VALUES ('Operator', 'operator') ON CONFLICT DO NOTHING`)

	// Seed default permissions
	db.Exec(`INSERT INTO goadmin_permissions (name, slug, http_method, http_path) VALUES ('All permissions', '*', '', '*') ON CONFLICT DO NOTHING`)
	db.Exec(`INSERT INTO goadmin_permissions (name, slug, http_method, http_path) VALUES ('Dashboard', 'dashboard', 'GET', '/') ON CONFLICT DO NOTHING`)

	// Assign admin user to administrator role (if not already assigned)
	if goAdminUsername != "" {
		db.Exec(`
			INSERT INTO goadmin_role_users (role_id, user_id)
			SELECT r.id, u.id FROM goadmin_roles r, goadmin_users u
			WHERE r.slug = 'administrator' AND u.username = ?
			AND NOT EXISTS (
				SELECT 1 FROM goadmin_role_users ru
				WHERE ru.role_id = r.id AND ru.user_id = u.id
			)
		`, goAdminUsername)
	}

	// Assign all permissions to administrator role (if not already assigned)
	db.Exec(`
		INSERT INTO goadmin_role_permissions (role_id, permission_id)
		SELECT r.id, p.id FROM goadmin_roles r, goadmin_permissions p
		WHERE r.slug = 'administrator' AND p.slug = '*'
		AND NOT EXISTS (
			SELECT 1 FROM goadmin_role_permissions rp
			WHERE rp.role_id = r.id AND rp.permission_id = p.id
		)
	`)

	// Seed default menu items (only if menu is empty)
	var menuCount int64
	db.Raw(`SELECT COUNT(*) FROM goadmin_menu`).Scan(&menuCount)
	if menuCount == 0 {
		db.Exec(`
			INSERT INTO goadmin_menu (parent_id, type, "order", title, icon, uri, plugin_name) VALUES
			(0, 1, 1, 'Dashboard', 'fa-bar-chart', '/', ''),
			(0, 1, 2, 'Admin', 'fa-tasks', '', ''),
			(2, 1, 1, 'Users', 'fa-users', '/info/goadmin_users', ''),
			(2, 1, 2, 'Roles', 'fa-user', '/info/goadmin_roles', ''),
			(2, 1, 3, 'Permissions', 'fa-ban', '/info/goadmin_permissions', ''),
			(2, 1, 4, 'Menu', 'fa-bars', '/menu', ''),
			(2, 1, 5, 'Operation Log', 'fa-history', '/info/goadmin_operation_log', ''),
			(0, 1, 3, 'HoneyDue', 'fa-home', '', ''),
			(8, 1, 1, 'Users', 'fa-user', '/info/users', ''),
			(8, 1, 2, 'Residences', 'fa-building', '/info/residences', ''),
			(8, 1, 3, 'Tasks', 'fa-tasks', '/info/tasks', ''),
			(8, 1, 4, 'Contractors', 'fa-wrench', '/info/contractors', ''),
			(8, 1, 5, 'Documents', 'fa-file', '/info/documents', ''),
			(8, 1, 6, 'Notifications', 'fa-bell', '/info/notifications', '')
		`)

		// Assign all menus to administrator role
		db.Exec(`
			INSERT INTO goadmin_role_menu (role_id, menu_id)
			SELECT r.id, m.id FROM goadmin_roles r, goadmin_menu m
			WHERE r.slug = 'administrator'
		`)
	}

	log.Info().Msg("GoAdmin migrations completed")

	// Seed default Next.js admin user only on first run.
	// Password is NOT reset on subsequent migrations to preserve operator changes.
	adminEmail := viper.GetString("ADMIN_EMAIL")
	adminPassword := viper.GetString("ADMIN_PASSWORD")
	if adminEmail == "" || adminPassword == "" {
		log.Warn().Msg("ADMIN_EMAIL and/or ADMIN_PASSWORD not set; skipping Next.js admin user seed")
	} else {
		var adminCount int64
		db.Raw(`SELECT COUNT(*) FROM admin_users WHERE email = ?`, adminEmail).Scan(&adminCount)
		if adminCount == 0 {
			log.Info().Str("email", adminEmail).Msg("Seeding default admin user for Next.js admin panel...")
			adminHash, err := bcrypt.GenerateFromPassword([]byte(adminPassword), bcrypt.DefaultCost)
			if err != nil {
				return fmt.Errorf("failed to hash admin password: %w", err)
			}
			db.Exec(`
				INSERT INTO admin_users (email, password, first_name, last_name, role, is_active, created_at, updated_at)
				VALUES (?, ?, 'Admin', 'User', 'super_admin', true, NOW(), NOW())
			`, adminEmail, string(adminHash))
			log.Info().Str("email", adminEmail).Msg("Default admin user created")
		}
	}

	return nil
}