Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
+351
-49
@@ -1,6 +1,6 @@
|
||||
import { Router } from 'express';
|
||||
import { join } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
|
||||
import { execFile, spawn } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { getSetting } from './db.js';
|
||||
@@ -8,7 +8,129 @@ import { getSetting } from './db.js';
|
||||
const execFileAsync = promisify(execFile);
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
const CACHE_DIR = join(MEDIA_PATH, '.hls-cache');
|
||||
const SEGMENT_DURATION = 10;
|
||||
const MAX_CONCURRENT_TRANSCODES = 2;
|
||||
const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
if (!existsSync(CACHE_DIR)) {
|
||||
mkdirSync(CACHE_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Quality tiers — always transcode for reliable HLS
|
||||
const QUALITY_TIERS = {
|
||||
'480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
|
||||
'720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
|
||||
'1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
|
||||
};
|
||||
|
||||
// Compute output dimensions preserving source aspect ratio
|
||||
function fitDimensions(srcW, srcH, maxW, maxH) {
|
||||
if (srcW <= maxW && srcH <= maxH) {
|
||||
let w = srcW, h = srcH;
|
||||
w += w % 2; h += h % 2;
|
||||
return { w, h };
|
||||
}
|
||||
const scale = Math.min(maxW / srcW, maxH / srcH);
|
||||
let w = Math.round(srcW * scale);
|
||||
let h = Math.round(srcH * scale);
|
||||
w += w % 2; h += h % 2;
|
||||
return { w, h };
|
||||
}
|
||||
|
||||
// --- Hardware acceleration detection (shared state with video-hls.js via same detection) ---
|
||||
let hwAccel = null; // 'vaapi' | 'qsv' | null
|
||||
let hwDetected = false;
|
||||
|
||||
async function detectHwAccel() {
|
||||
if (hwDetected) return hwAccel;
|
||||
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner',
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'format=nv12,hwupload',
|
||||
'-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
|
||||
hwAccel = 'vaapi';
|
||||
console.log('[hls] Intel VAAPI hardware acceleration available');
|
||||
} catch {
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner', '-init_hw_device', 'qsv=hw',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'hwupload=extra_hw_frames=64,format=qsv',
|
||||
'-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000 });
|
||||
hwAccel = 'qsv';
|
||||
console.log('[hls] Intel QSV hardware acceleration available');
|
||||
} catch {
|
||||
hwAccel = null;
|
||||
console.log('[hls] No hardware acceleration, using libx264');
|
||||
}
|
||||
}
|
||||
|
||||
hwDetected = true;
|
||||
return hwAccel;
|
||||
}
|
||||
|
||||
detectHwAccel();
|
||||
|
||||
// --- Transcode semaphore ---
|
||||
let activeTranscodes = 0;
|
||||
const transcodeQueue = [];
|
||||
|
||||
function acquireSlot() {
|
||||
return new Promise((resolve) => {
|
||||
if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
|
||||
activeTranscodes++;
|
||||
resolve();
|
||||
} else {
|
||||
transcodeQueue.push(resolve);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function releaseSlot() {
|
||||
activeTranscodes--;
|
||||
if (transcodeQueue.length > 0) {
|
||||
activeTranscodes++;
|
||||
transcodeQueue.shift()();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Cache cleanup (hourly) ---
|
||||
function cleanupCache() {
|
||||
try {
|
||||
if (!existsSync(CACHE_DIR)) return;
|
||||
const now = Date.now();
|
||||
const walk = (dir) => {
|
||||
let entries;
|
||||
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const e of entries) {
|
||||
const full = join(dir, e.name);
|
||||
if (e.isDirectory()) {
|
||||
walk(full);
|
||||
// Remove empty dirs
|
||||
try { if (readdirSync(full).length === 0) rmSync(full); } catch { /* ignore */ }
|
||||
} else if (e.name.endsWith('.ts')) {
|
||||
try {
|
||||
const st = statSync(full);
|
||||
if (now - st.mtimeMs > CACHE_MAX_AGE_MS) rmSync(full);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(CACHE_DIR);
|
||||
} catch (err) {
|
||||
console.error('[hls] Cache cleanup error:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(cleanupCache, 60 * 60 * 1000);
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function isHlsEnabled() {
|
||||
return (getSetting('hls_enabled') || process.env.HLS_ENABLED) === 'true';
|
||||
@@ -23,32 +145,89 @@ function validatePath(folder, filename) {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// GET /api/hls/:folder/:filename/master.m3u8
|
||||
// Sanitize folder+filename into a cache-safe directory name
|
||||
function cacheKey(folder, filename) {
|
||||
return join(folder, filename.replace(/[^a-zA-Z0-9._-]/g, '_'));
|
||||
}
|
||||
|
||||
async function probeVideo(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'stream=codec_type,width,height',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'json',
|
||||
filePath,
|
||||
], { timeout: 15000 });
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = info.streams?.find(s => s.codec_type === 'video');
|
||||
return {
|
||||
duration: parseFloat(info.format?.duration || '0'),
|
||||
width: videoStream?.width || 0,
|
||||
height: videoStream?.height || 0,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Master playlist ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
|
||||
if (!isHlsEnabled()) {
|
||||
return res.status(404).json({ error: 'HLS not enabled' });
|
||||
}
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename } = req.params;
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
}
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'csv=p=0',
|
||||
filePath,
|
||||
]);
|
||||
const info = await probeVideo(filePath);
|
||||
const srcW = info.width || 1920;
|
||||
const srcH = info.height || 1080;
|
||||
|
||||
const duration = parseFloat(stdout.trim());
|
||||
if (isNaN(duration) || duration <= 0) {
|
||||
let playlist = '#EXTM3U\n';
|
||||
|
||||
for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
|
||||
if (tier.maxH <= srcH) {
|
||||
const { w, h } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
|
||||
const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
|
||||
playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
|
||||
playlist += `${name}/playlist.m3u8\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// If source is too small for any tier, add a single tier at source resolution
|
||||
if (!playlist.includes('EXT-X-STREAM-INF')) {
|
||||
const { w, h } = fitDimensions(srcW, srcH, srcW, srcH);
|
||||
playlist += `#EXT-X-STREAM-INF:BANDWIDTH=1596000,RESOLUTION=${w}x${h},NAME="480p"\n`;
|
||||
playlist += `480p/playlist.m3u8\n`;
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[hls] Master playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate master playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Variant playlist ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/:quality/playlist.m3u8', async (req, res) => {
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename, quality } = req.params;
|
||||
if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
|
||||
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
try {
|
||||
const info = await probeVideo(filePath);
|
||||
const duration = info.duration;
|
||||
if (!duration || duration <= 0) {
|
||||
return res.status(500).json({ error: 'Could not determine video duration' });
|
||||
}
|
||||
|
||||
const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
|
||||
|
||||
let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
|
||||
playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
|
||||
playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
|
||||
@@ -63,54 +242,177 @@ router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
|
||||
playlist += '#EXT-X-ENDLIST\n';
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[hls] ffprobe error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to probe video' });
|
||||
console.error('[hls] Variant playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate variant playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/hls/:folder/:filename/segment-:index.ts
|
||||
router.get('/api/hls/:folder/:filename/segment-:index.ts', (req, res) => {
|
||||
if (!isHlsEnabled()) {
|
||||
return res.status(404).json({ error: 'HLS not enabled' });
|
||||
}
|
||||
// --- Segment transcoding ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/:quality/segment-:index.ts', async (req, res) => {
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename, quality } = req.params;
|
||||
const segIndex = parseInt(req.params.index, 10);
|
||||
|
||||
if (isNaN(segIndex) || segIndex < 0) return res.status(400).json({ error: 'Invalid segment index' });
|
||||
if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
|
||||
|
||||
const { folder, filename, index } = req.params;
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
// Check cache
|
||||
const key = cacheKey(folder, filename);
|
||||
const segCacheDir = join(CACHE_DIR, key, quality);
|
||||
const segCachePath = join(segCacheDir, `segment-${segIndex}.ts`);
|
||||
|
||||
if (existsSync(segCachePath)) {
|
||||
const stat = statSync(segCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const segIndex = parseInt(index, 10);
|
||||
if (isNaN(segIndex) || segIndex < 0) {
|
||||
return res.status(400).json({ error: 'Invalid segment index' });
|
||||
await acquireSlot();
|
||||
|
||||
// Double-check cache after acquiring slot
|
||||
if (existsSync(segCachePath)) {
|
||||
releaseSlot();
|
||||
const stat = statSync(segCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const offset = segIndex * SEGMENT_DURATION;
|
||||
try {
|
||||
const offset = segIndex * SEGMENT_DURATION;
|
||||
const accel = await detectHwAccel();
|
||||
const tier = QUALITY_TIERS[quality];
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', [
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-c', 'copy',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
// Probe source dimensions for aspect-ratio-aware scaling
|
||||
let srcW = 1920, srcH = 1080;
|
||||
try {
|
||||
const info = await probeVideo(filePath);
|
||||
srcW = info.width || 1920;
|
||||
srcH = info.height || 1080;
|
||||
} catch { /* use defaults */ }
|
||||
|
||||
res.setHeader('Content-Type', 'video/MP2T');
|
||||
ffmpeg.stdout.pipe(res);
|
||||
const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
|
||||
|
||||
req.on('close', () => {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
console.error('[hls] ffmpeg error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Transcoding failed' });
|
||||
let ffmpegArgs;
|
||||
if (accel === 'vaapi') {
|
||||
ffmpegArgs = [
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-filter_hw_device', 'va',
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_vaapi',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else if (accel === 'qsv') {
|
||||
ffmpegArgs = [
|
||||
'-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale_qsv=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_qsv',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else {
|
||||
ffmpegArgs = [
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale=${outW}:${outH}`,
|
||||
'-c:v', 'libx264', '-preset', 'veryfast',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
const spawnEnv = accel === 'vaapi'
|
||||
? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
|
||||
: undefined;
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
...(spawnEnv && { env: spawnEnv }),
|
||||
});
|
||||
|
||||
res.setHeader('Content-Type', 'video/MP2T');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
|
||||
const cacheChunks = [];
|
||||
let aborted = false;
|
||||
|
||||
ffmpeg.stdout.on('data', (chunk) => {
|
||||
cacheChunks.push(chunk);
|
||||
if (!res.destroyed) res.write(chunk);
|
||||
});
|
||||
|
||||
req.on('close', () => {
|
||||
if (!ffmpeg.killed) {
|
||||
aborted = true;
|
||||
ffmpeg.kill('SIGKILL');
|
||||
releaseSlot();
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (aborted) return;
|
||||
releaseSlot();
|
||||
|
||||
if (code === 0 && cacheChunks.length > 0) {
|
||||
try {
|
||||
if (!existsSync(segCacheDir)) mkdirSync(segCacheDir, { recursive: true });
|
||||
const ws = createWriteStream(segCachePath);
|
||||
for (const c of cacheChunks) ws.write(c);
|
||||
ws.end();
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
if (!res.destroyed) res.end();
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (!aborted) releaseSlot();
|
||||
console.error('[hls] ffmpeg error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Transcoding failed' });
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
releaseSlot();
|
||||
console.error('[hls] Segment error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
Reference in New Issue
Block a user