Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
+142
-2
@@ -3,7 +3,7 @@ import fetch from 'node-fetch';
|
||||
import { mkdirSync, createWriteStream, statSync } from 'fs';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { extname } from 'path';
|
||||
import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile } from './db.js';
|
||||
import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile, getAutoDownloadUsers, addAutoDownloadUser, removeAutoDownloadUser } from './db.js';
|
||||
import { createSignedHeaders, getRules } from './signing.js';
|
||||
import { downloadDrmMedia, hasCDM } from './drm-download.js';
|
||||
|
||||
@@ -15,6 +15,36 @@ const DOWNLOAD_DELAY = parseInt(process.env.DOWNLOAD_DELAY || '1000', 10);
|
||||
// In-memory progress: userId -> { total, completed, errors, running }
|
||||
const progressMap = new Map();
|
||||
|
||||
// In-memory download logs: userId -> last N file entries
|
||||
const downloadLogMap = new Map();
|
||||
const MAX_LOG_ENTRIES = 20;
|
||||
|
||||
function addDownloadLog(userId, entry) {
|
||||
const key = String(userId);
|
||||
if (!downloadLogMap.has(key)) downloadLogMap.set(key, []);
|
||||
const logs = downloadLogMap.get(key);
|
||||
logs.push({ ...entry, timestamp: new Date().toISOString() });
|
||||
if (logs.length > MAX_LOG_ENTRIES) logs.shift();
|
||||
}
|
||||
|
||||
export function getActiveDownloadCount() {
|
||||
let count = 0;
|
||||
for (const p of progressMap.values()) {
|
||||
if (p.running) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getActiveDownloadsList() {
|
||||
const list = [];
|
||||
for (const [userId, p] of progressMap.entries()) {
|
||||
if (p.running) {
|
||||
list.push({ userId, username: p.username, total: p.total, completed: p.completed, errors: p.errors });
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
function buildHeaders(authConfig, signedHeaders) {
|
||||
const rules = getRules();
|
||||
const headers = {
|
||||
@@ -83,8 +113,9 @@ async function downloadFile(url, dest) {
|
||||
}
|
||||
|
||||
async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const progress = { total: 0, completed: 0, errors: 0, running: true };
|
||||
const progress = { total: 0, completed: 0, errors: 0, running: true, username: username || null };
|
||||
progressMap.set(String(userId), progress);
|
||||
console.log(`[download] Starting download for user ${userId} (${username || 'unknown'})${postLimit ? ` limit=${postLimit}` : ' all posts'}${resume ? ' (resume)' : ''}`);
|
||||
|
||||
try {
|
||||
let beforePublishTime = null;
|
||||
@@ -156,6 +187,7 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
}
|
||||
|
||||
progress.total = allMedia.length;
|
||||
console.log(`[download] User ${userId}: found ${allMedia.length} media items across ${postsFetched} posts`);
|
||||
|
||||
// Phase 2: Download each media item
|
||||
for (const { postId, media, postDate } of allMedia) {
|
||||
@@ -203,9 +235,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const st = statSync(`${userDir}/${drmFilename}`);
|
||||
upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate);
|
||||
} catch { /* stat may fail if file was cleaned up */ }
|
||||
addDownloadLog(userId, { filename: drmFilename, mediaType: 'video', status: 'ok' });
|
||||
progress.completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
|
||||
addDownloadLog(userId, { filename: `${postId}_${mediaId}_video.mp4`, mediaType: 'video', status: 'error' });
|
||||
progress.errors++;
|
||||
progress.completed++;
|
||||
}
|
||||
@@ -234,9 +268,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
|
||||
if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
|
||||
} catch { /* ignore */ }
|
||||
addDownloadLog(userId, { filename, mediaType, status: 'ok' });
|
||||
progress.completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] Error downloading media ${media.id}:`, err.message);
|
||||
addDownloadLog(userId, { filename: `${postId}_${media.id}`, mediaType: media.type || 'unknown', status: 'error' });
|
||||
progress.errors++;
|
||||
progress.completed++;
|
||||
}
|
||||
@@ -251,6 +287,75 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
}
|
||||
}
|
||||
|
||||
// POST /api/download/post — download media from a single post
|
||||
router.post('/api/download/post', async (req, res, next) => {
|
||||
try {
|
||||
const { userId, username, postId, postedAt, media: mediaItems } = req.body;
|
||||
if (!userId || !postId || !Array.isArray(mediaItems) || mediaItems.length === 0) {
|
||||
return res.status(400).json({ error: 'userId, postId, and media[] are required' });
|
||||
}
|
||||
const postDate = postedAt || null;
|
||||
|
||||
const userDir = `${MEDIA_PATH}/${username || userId}`;
|
||||
mkdirSync(userDir, { recursive: true });
|
||||
let completed = 0, errors = 0;
|
||||
|
||||
console.log(`[download] Post ${postId}: downloading ${mediaItems.length} media items for ${username || userId}, postedAt=${postDate}`);
|
||||
|
||||
for (const media of mediaItems) {
|
||||
try {
|
||||
const mediaId = String(media.id);
|
||||
if (isMediaDownloaded(mediaId)) { completed++; continue; }
|
||||
if (media.canView === false) { completed++; continue; }
|
||||
|
||||
// DRM video
|
||||
const drm = media.files?.drm;
|
||||
if (drm?.manifest?.dash && drm?.signature?.dash) {
|
||||
if (!hasCDM()) { completed++; continue; }
|
||||
try {
|
||||
const sig = drm.signature.dash;
|
||||
const cfCookies = { cp: sig['CloudFront-Policy'], cs: sig['CloudFront-Signature'], ck: sig['CloudFront-Key-Pair-Id'] };
|
||||
const drmFilename = `${postId}_${mediaId}_video.mp4`;
|
||||
await downloadDrmMedia({ mpdUrl: drm.manifest.dash, cfCookies, mediaId, entityType: 'post', entityId: String(postId), outputDir: userDir, outputFilename: drmFilename });
|
||||
recordDownload(userId, String(postId), mediaId, 'video', drmFilename, postDate);
|
||||
try { const st = statSync(`${userDir}/${drmFilename}`); upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate); } catch {}
|
||||
completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
|
||||
errors++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = getMediaUrl(media);
|
||||
if (!url) { completed++; continue; }
|
||||
|
||||
const mediaType = media.type || 'unknown';
|
||||
const ext = getExtFromUrl(url);
|
||||
const filename = `${postId}_${mediaId}_${mediaType}${ext}`;
|
||||
const dest = `${userDir}/${filename}`;
|
||||
|
||||
await downloadFile(url, dest);
|
||||
recordDownload(userId, String(postId), mediaId, mediaType, filename, postDate);
|
||||
try {
|
||||
const st = statSync(dest);
|
||||
const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
|
||||
if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
|
||||
} catch {}
|
||||
completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] Error downloading media ${media.id}:`, err.message);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[download] Post ${postId}: done (${completed} downloaded, ${errors} errors)`);
|
||||
res.json({ status: 'done', completed, errors, total: mediaItems.length });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/download/:userId — start background download
|
||||
router.post('/api/download/:userId', (req, res, next) => {
|
||||
try {
|
||||
@@ -302,6 +407,21 @@ router.get('/api/download/active', (req, res) => {
|
||||
res.json(active);
|
||||
});
|
||||
|
||||
// GET /api/download/active/details — active downloads with recent file logs
|
||||
router.get('/api/download/active/details', (req, res) => {
|
||||
const active = [];
|
||||
for (const [userId, progress] of progressMap.entries()) {
|
||||
if (progress.running) {
|
||||
active.push({
|
||||
user_id: userId,
|
||||
...progress,
|
||||
recentFiles: downloadLogMap.get(String(userId))?.slice(-5) || [],
|
||||
});
|
||||
}
|
||||
}
|
||||
res.json(active);
|
||||
});
|
||||
|
||||
// GET /api/download/history
|
||||
router.get('/api/download/history', (req, res, next) => {
|
||||
try {
|
||||
@@ -312,4 +432,24 @@ router.get('/api/download/history', (req, res, next) => {
|
||||
}
|
||||
});
|
||||
|
||||
// --- Auto-download CRUD ---
|
||||
|
||||
router.get('/api/download/auto', (_req, res) => {
|
||||
res.json(getAutoDownloadUsers());
|
||||
});
|
||||
|
||||
router.post('/api/download/auto/:userId', (req, res) => {
|
||||
const { userId } = req.params;
|
||||
const { username } = req.body;
|
||||
if (!username) return res.status(400).json({ error: 'username is required' });
|
||||
addAutoDownloadUser(userId, username);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
router.delete('/api/download/auto/:userId', (req, res) => {
|
||||
removeAutoDownloadUser(req.params.userId);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
export { runDownload };
|
||||
export default router;
|
||||
|
||||
Reference in New Issue
Block a user