Add app auth, dashboard, scheduler, video management, and new scrapers

- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-16 07:48:10 -05:00
parent 4903b84aef
commit 236f36aae6
54 changed files with 9986 additions and 420 deletions
@@ -0,0 +1,95 @@
+import { getAuthConfig, getAutoDownloadUsers, updateAutoDownloadLastRun, getAutoScrapeJobs, updateAutoScrapeLastRun } from './db.js';
+import { runDownload } from './download.js';
+import { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, createJob } from './scrape.js';
+
+const INTERVAL = 12 * 60 * 60 * 1000; // 12 hours
+const STARTUP_DELAY = 30 * 1000; // 30 seconds
+
+async function runAutoDownloads() {
+  const users = getAutoDownloadUsers();
+  if (users.length === 0) return;
+
+  const authConfig = getAuthConfig();
+  if (!authConfig) {
+    console.log('[scheduler] Skipping auto-downloads: no auth config');
+    return;
+  }
+
+  console.log(`[scheduler] Starting auto-downloads for ${users.length} user(s)`);
+
+  for (const user of users) {
+    try {
+      console.log(`[scheduler] Downloading ${user.username} (${user.user_id})`);
+      await runDownload(user.user_id, authConfig, null, true, user.username);
+      updateAutoDownloadLastRun(user.user_id);
+      console.log(`[scheduler] Completed download for ${user.username}`);
+    } catch (err) {
+      console.error(`[scheduler] Error downloading ${user.username}:`, err.message);
+    }
+  }
+
+  console.log('[scheduler] Auto-downloads complete');
+}
+
+async function runAutoScrapes() {
+  const jobs = getAutoScrapeJobs();
+  if (jobs.length === 0) return;
+
+  console.log(`[scheduler] Starting auto-scrapes for ${jobs.length} job(s)`);
+
+  for (const savedJob of jobs) {
+    try {
+      const config = JSON.parse(savedJob.config);
+      config.folderName = savedJob.folder_name;
+      const job = createJob(savedJob.type, config);
+
+      console.log(`[scheduler] Running ${savedJob.type} scrape for ${savedJob.folder_name}`);
+
+      if (savedJob.type === 'forum') {
+        await runForumScrape(job);
+      } else if (savedJob.type === 'coomer') {
+        await runCoomerScrape(job);
+      } else if (savedJob.type === 'medialink') {
+        await runMediaLinkScrape(job);
+      } else if (savedJob.type === 'mega') {
+        await runMegaScrape(job);
+      }
+
+      updateAutoScrapeLastRun(savedJob.id);
+      console.log(`[scheduler] Completed scrape for ${savedJob.folder_name}`);
+    } catch (err) {
+      console.error(`[scheduler] Error scraping ${savedJob.folder_name}:`, err.message);
+    }
+  }
+
+  console.log('[scheduler] Auto-scrapes complete');
+}
+
+async function runAll() {
+  try {
+    await runAutoDownloads();
+  } catch (err) {
+    console.error('[scheduler] Auto-download batch failed:', err.message);
+  }
+  try {
+    await runAutoScrapes();
+  } catch (err) {
+    console.error('[scheduler] Auto-scrape batch failed:', err.message);
+  }
+}
+
+export function startScheduler() {
+  // Run once shortly after startup
+  setTimeout(() => {
+    console.log('[scheduler] Running initial auto-download/scrape check');
+    runAll();
+  }, STARTUP_DELAY);
+
+  // Then every 12 hours
+  setInterval(() => {
+    console.log('[scheduler] Running scheduled auto-download/scrape');
+    runAll();
+  }, INTERVAL);
+
+  console.log('[scheduler] Scheduler started (interval: 12h)');
+}