Add app auth, dashboard, scheduler, video management, and new scrapers

- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-16 07:48:10 -05:00
parent 4903b84aef
commit 236f36aae6
54 changed files with 9986 additions and 420 deletions
@@ -0,0 +1,434 @@
+import { Router } from 'express';
+import { join, extname } from 'path';
+import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
+import { execFile, spawn } from 'child_process';
+import { promisify } from 'util';
+import { getVideoById } from './db.js';
+
+const execFileAsync = promisify(execFile);
+const router = Router();
+
+const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
+const CACHE_DIR = join(VIDEOS_PATH, '.hls-cache');
+const SEGMENT_DURATION = 10;
+const MAX_CONCURRENT_TRANSCODES = 2;
+const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+// Ensure cache dir exists
+if (!existsSync(CACHE_DIR)) {
+  mkdirSync(CACHE_DIR, { recursive: true });
+}
+
+// Quality tiers (no "original" copy mode — always transcode for reliable HLS)
+const QUALITY_TIERS = {
+  '480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
+  '720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
+  '1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
+};
+
+// Compute output dimensions preserving source aspect ratio, clamped to maxW x maxH
+function fitDimensions(srcW, srcH, maxW, maxH) {
+  if (srcW <= maxW && srcH <= maxH) {
+    // Already fits — round to even
+    let w = srcW, h = srcH;
+    w += w % 2; h += h % 2;
+    return { w, h };
+  }
+  const scale = Math.min(maxW / srcW, maxH / srcH);
+  let w = Math.round(srcW * scale);
+  let h = Math.round(srcH * scale);
+  w += w % 2; h += h % 2; // ensure even (required for encoding)
+  return { w, h };
+}
+
+// Hardware acceleration detection: VAAPI > QSV > libx264
+// Alpine ships intel-media-driver (VA-API) but not oneVPL GPU runtime (QSV),
+// so VAAPI is the preferred HW accel path for Intel iGPUs.
+let hwAccel = null; // 'vaapi' | 'qsv' | null
+
+async function detectHwAccel() {
+  if (hwAccel !== null) return hwAccel;
+
+  // Try VAAPI first (works on Alpine with intel-media-driver)
+  try {
+    await execFileAsync('ffmpeg', [
+      '-hide_banner',
+      '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+      '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+      '-vf', 'format=nv12,hwupload',
+      '-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
+    ], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
+    hwAccel = 'vaapi';
+    console.log('[video-hls] Intel VAAPI hardware acceleration available');
+    return hwAccel;
+  } catch {
+    // VAAPI failed, try QSV
+  }
+
+  try {
+    await execFileAsync('ffmpeg', [
+      '-hide_banner', '-init_hw_device', 'qsv=hw',
+      '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+      '-vf', 'hwupload=extra_hw_frames=64,format=qsv',
+      '-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
+    ], { timeout: 10000 });
+    hwAccel = 'qsv';
+    console.log('[video-hls] Intel QSV hardware acceleration available');
+    return hwAccel;
+  } catch {
+    // QSV also failed
+  }
+
+  hwAccel = null;
+  console.log('[video-hls] No hardware acceleration available, using libx264 fallback');
+  return hwAccel;
+}
+
+// Detect on startup
+detectHwAccel();
+
+// --- Transcode semaphore ---
+
+let activeTranscodes = 0;
+const transcodeQueue = [];
+
+function acquireTranscodeSlot() {
+  return new Promise((resolve) => {
+    if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
+      activeTranscodes++;
+      resolve();
+    } else {
+      transcodeQueue.push(resolve);
+    }
+  });
+}
+
+function releaseTranscodeSlot() {
+  activeTranscodes--;
+  if (transcodeQueue.length > 0) {
+    activeTranscodes++;
+    transcodeQueue.shift()();
+  }
+}
+
+// --- Cache cleanup (hourly) ---
+
+function cleanupCache() {
+  try {
+    if (!existsSync(CACHE_DIR)) return;
+    const videoDirs = readdirSync(CACHE_DIR, { withFileTypes: true });
+    const now = Date.now();
+
+    for (const dir of videoDirs) {
+      if (!dir.isDirectory()) continue;
+      const videoCacheDir = join(CACHE_DIR, dir.name);
+
+      // Find newest segment mtime across all quality dirs
+      let newestMtime = 0;
+      try {
+        const qualityDirs = readdirSync(videoCacheDir, { withFileTypes: true });
+        for (const qDir of qualityDirs) {
+          if (!qDir.isDirectory()) continue;
+          const qPath = join(videoCacheDir, qDir.name);
+          const files = readdirSync(qPath);
+          for (const f of files) {
+            try {
+              const st = statSync(join(qPath, f));
+              if (st.mtimeMs > newestMtime) newestMtime = st.mtimeMs;
+            } catch { /* ignore */ }
+          }
+        }
+      } catch { continue; }
+
+      if (newestMtime > 0 && (now - newestMtime) > CACHE_MAX_AGE_MS) {
+        try {
+          rmSync(videoCacheDir, { recursive: true });
+          console.log(`[video-hls] Cleaned cache for video ${dir.name}`);
+        } catch { /* ignore */ }
+      }
+    }
+  } catch (err) {
+    console.error('[video-hls] Cache cleanup error:', err.message);
+  }
+}
+
+// Run cleanup every hour
+setInterval(cleanupCache, 60 * 60 * 1000);
+
+// --- Probe video duration ---
+
+async function getVideoDuration(filePath) {
+  const { stdout } = await execFileAsync('ffprobe', [
+    '-v', 'error', '-show_entries', 'format=duration',
+    '-of', 'csv=p=0', filePath,
+  ], { timeout: 15000 });
+  return parseFloat(stdout.trim()) || 0;
+}
+
+async function getVideoInfo(filePath) {
+  const { stdout } = await execFileAsync('ffprobe', [
+    '-v', 'error',
+    '-show_entries', 'stream=codec_type,width,height',
+    '-show_entries', 'format=duration',
+    '-of', 'json',
+    filePath,
+  ], { timeout: 15000 });
+  const info = JSON.parse(stdout);
+  const videoStream = info.streams?.find(s => s.codec_type === 'video');
+  return {
+    duration: parseFloat(info.format?.duration || '0'),
+    width: videoStream?.width || 0,
+    height: videoStream?.height || 0,
+    hasAudio: !!info.streams?.find(s => s.codec_type === 'audio'),
+  };
+}
+
+// --- Master playlist ---
+
+// GET /api/video-hls/:id/master.m3u8
+router.get('/api/video-hls/:id/master.m3u8', async (req, res) => {
+  try {
+    const id = parseInt(req.params.id, 10);
+    const video = getVideoById(id);
+    if (!video) return res.status(404).json({ error: 'Video not found' });
+    if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+    const info = await getVideoInfo(video.file_path);
+    const sourceWidth = info.width || video.width || 1920;
+    const sourceHeight = info.height || video.height || 1080;
+
+    let playlist = '#EXTM3U\n';
+
+    // Add quality tiers at or below source resolution (all transcoded, aspect-ratio preserved)
+    for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
+      if (tier.maxH <= sourceHeight) {
+        const { w, h } = fitDimensions(sourceWidth, sourceHeight, tier.maxW, tier.maxH);
+        const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
+        playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
+        playlist += `${name}/playlist.m3u8\n`;
+      }
+    }
+
+    res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.send(playlist);
+  } catch (err) {
+    console.error('[video-hls] Master playlist error:', err.message);
+    res.status(500).json({ error: 'Failed to generate master playlist' });
+  }
+});
+
+// --- Variant playlist ---
+
+// GET /api/video-hls/:id/:quality/playlist.m3u8
+router.get('/api/video-hls/:id/:quality/playlist.m3u8', async (req, res) => {
+  try {
+    const id = parseInt(req.params.id, 10);
+    const { quality } = req.params;
+
+    if (!QUALITY_TIERS[quality]) {
+      return res.status(400).json({ error: 'Invalid quality' });
+    }
+
+    const video = getVideoById(id);
+    if (!video) return res.status(404).json({ error: 'Video not found' });
+    if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+    const duration = await getVideoDuration(video.file_path);
+    if (!duration || duration <= 0) {
+      return res.status(500).json({ error: 'Could not determine video duration' });
+    }
+
+    const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
+
+    let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
+    playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
+    playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
+
+    for (let i = 0; i < segmentCount; i++) {
+      const remaining = duration - i * SEGMENT_DURATION;
+      const segDuration = Math.min(SEGMENT_DURATION, remaining);
+      playlist += `#EXTINF:${segDuration.toFixed(3)},\n`;
+      playlist += `segment-${i}.ts\n`;
+    }
+
+    playlist += '#EXT-X-ENDLIST\n';
+
+    res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.send(playlist);
+  } catch (err) {
+    console.error('[video-hls] Variant playlist error:', err.message);
+    res.status(500).json({ error: 'Failed to generate variant playlist' });
+  }
+});
+
+// --- Segment transcoding ---
+
+// GET /api/video-hls/:id/:quality/segment-:index.ts
+router.get('/api/video-hls/:id/:quality/segment-:index.ts', async (req, res) => {
+  try {
+    const id = parseInt(req.params.id, 10);
+    const { quality } = req.params;
+    const segIndex = parseInt(req.params.index, 10);
+
+    if (isNaN(segIndex) || segIndex < 0) {
+      return res.status(400).json({ error: 'Invalid segment index' });
+    }
+    if (!QUALITY_TIERS[quality]) {
+      return res.status(400).json({ error: 'Invalid quality' });
+    }
+
+    const video = getVideoById(id);
+    if (!video) return res.status(404).json({ error: 'Video not found' });
+    if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+    // Check cache first
+    const segmentCacheDir = join(CACHE_DIR, String(id), quality);
+    const segmentCachePath = join(segmentCacheDir, `segment-${segIndex}.ts`);
+
+    if (existsSync(segmentCachePath)) {
+      const stat = statSync(segmentCachePath);
+      res.writeHead(200, {
+        'Content-Type': 'video/MP2T',
+        'Content-Length': stat.size,
+        'Cache-Control': 'public, max-age=3600',
+      });
+      createReadStream(segmentCachePath).pipe(res);
+      return;
+    }
+
+    // Transcode on-demand
+    await acquireTranscodeSlot();
+
+    // Check cache again after acquiring slot (another request may have cached it)
+    if (existsSync(segmentCachePath)) {
+      releaseTranscodeSlot();
+      const stat = statSync(segmentCachePath);
+      res.writeHead(200, {
+        'Content-Type': 'video/MP2T',
+        'Content-Length': stat.size,
+        'Cache-Control': 'public, max-age=3600',
+      });
+      createReadStream(segmentCachePath).pipe(res);
+      return;
+    }
+
+    const offset = segIndex * SEGMENT_DURATION;
+    const accel = await detectHwAccel();
+    const tier = QUALITY_TIERS[quality];
+
+    // Compute output dimensions preserving source aspect ratio
+    const srcW = video.width || 1920;
+    const srcH = video.height || 1080;
+    const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
+
+    // -output_ts_offset ensures PTS continuity across segments (each segment's PTS
+    // starts where the previous one ended, required for smooth HLS playback)
+    let ffmpegArgs;
+    if (accel === 'vaapi') {
+      ffmpegArgs = [
+        '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+        '-filter_hw_device', 'va',
+        '-ss', String(offset),
+        '-i', video.file_path,
+        '-t', String(SEGMENT_DURATION),
+        '-output_ts_offset', String(offset),
+        '-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
+        '-c:v', 'h264_vaapi',
+        '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+        '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+        '-f', 'mpegts',
+        'pipe:1',
+      ];
+    } else if (accel === 'qsv') {
+      ffmpegArgs = [
+        '-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
+        '-ss', String(offset),
+        '-i', video.file_path,
+        '-t', String(SEGMENT_DURATION),
+        '-output_ts_offset', String(offset),
+        '-vf', `scale_qsv=w=${outW}:h=${outH}`,
+        '-c:v', 'h264_qsv',
+        '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+        '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+        '-f', 'mpegts',
+        'pipe:1',
+      ];
+    } else {
+      ffmpegArgs = [
+        '-ss', String(offset),
+        '-i', video.file_path,
+        '-t', String(SEGMENT_DURATION),
+        '-output_ts_offset', String(offset),
+        '-vf', `scale=${outW}:${outH}`,
+        '-c:v', 'libx264', '-preset', 'veryfast',
+        '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+        '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+        '-f', 'mpegts',
+        'pipe:1',
+      ];
+    }
+
+    const spawnEnv = accel === 'vaapi'
+      ? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
+      : undefined;
+
+    const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...(spawnEnv && { env: spawnEnv }),
+    });
+
+    // Stream directly to client while also collecting chunks for cache
+    res.setHeader('Content-Type', 'video/MP2T');
+    res.setHeader('Cache-Control', 'public, max-age=3600');
+
+    const cacheChunks = [];
+    let aborted = false;
+
+    ffmpeg.stdout.on('data', (chunk) => {
+      cacheChunks.push(chunk);
+      if (!res.destroyed) res.write(chunk);
+    });
+
+    req.on('close', () => {
+      if (!ffmpeg.killed) {
+        aborted = true;
+        ffmpeg.kill('SIGKILL');
+        releaseTranscodeSlot();
+      }
+    });
+
+    ffmpeg.on('close', (code) => {
+      if (aborted) return;
+      releaseTranscodeSlot();
+
+      // Write cache file on success
+      if (code === 0 && cacheChunks.length > 0) {
+        try {
+          if (!existsSync(segmentCacheDir)) mkdirSync(segmentCacheDir, { recursive: true });
+          const cacheStream = createWriteStream(segmentCachePath);
+          for (const chunk of cacheChunks) cacheStream.write(chunk);
+          cacheStream.end();
+        } catch { /* ignore cache write failure */ }
+      }
+
+      if (!res.destroyed) res.end();
+    });
+
+    ffmpeg.on('error', (err) => {
+      if (!aborted) releaseTranscodeSlot();
+      console.error('[video-hls] ffmpeg error:', err.message);
+      if (!res.headersSent) {
+        res.status(500).json({ error: 'Transcoding failed' });
+      }
+    });
+  } catch (err) {
+    console.error('[video-hls] Segment error:', err.message);
+    if (!res.headersSent) {
+      res.status(500).json({ error: err.message });
+    }
+  }
+});
+
+export default router;