Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
+429
-22
@@ -1,9 +1,14 @@
|
||||
import { Router } from 'express';
|
||||
import { mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { scrapeForumPage, getPageUrl, detectMaxPage } from './scrapers/forum.js';
|
||||
import { parseUserUrl, fetchAllPosts, downloadFiles } from './scrapers/coomer.js';
|
||||
import { parseMediaUrl, fetchAllMedia, downloadMedia } from './scrapers/medialink.js';
|
||||
import { scrapeForumPage, getPageUrl, detectMaxPage, CookieExpiredError } from './scrapers/forum.js';
|
||||
import { refreshForumCookies } from './flaresolverr.js';
|
||||
import { parseUserUrl, fetchAllPosts, fetchSearchPosts, downloadFiles } from './scrapers/coomer.js';
|
||||
import { parseMediaUrl, fetchAllMedia, fetchAllMediaFromHtml, downloadMedia } from './scrapers/medialink.js';
|
||||
import { parseMegaUrl, listAllFiles, downloadMegaFiles } from './scrapers/mega.js';
|
||||
import { runYtdlp } from './scrapers/ytdlp.js';
|
||||
import { parseLeakGalleryUrl, fetchAllMedia as fetchLeakGalleryMedia, downloadMedia as downloadLeakGalleryMedia } from './scrapers/leakgallery.js';
|
||||
import { getAutoScrapeJobs, addAutoScrapeJob, removeAutoScrapeJob, getForumSites, getForumSiteById, createForumSite, updateForumSite, deleteForumSite } from './db.js';
|
||||
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
@@ -56,6 +61,8 @@ function jobToJson(job) {
|
||||
progress: job.progress,
|
||||
running: job.running,
|
||||
cancelled: job.cancelled,
|
||||
paused: job.paused || false,
|
||||
resumeAt: job.resumeAt || null,
|
||||
folderName: job.folderName,
|
||||
startedAt: job.startedAt,
|
||||
completedAt: job.completedAt,
|
||||
@@ -66,13 +73,38 @@ function jobToJson(job) {
|
||||
// --- Forum Scrape ---
|
||||
|
||||
async function runForumScrape(job) {
|
||||
const { url, startPage, endPage, delay, folderName } = job.config;
|
||||
let { url, startPage, endPage, delay, folderName, siteId, lastPageOnly } = job.config;
|
||||
let { cookies } = job.config;
|
||||
|
||||
// Load cookies from forum site record if siteId provided and no cookies passed
|
||||
if (!cookies && siteId) {
|
||||
const site = getForumSiteById(siteId);
|
||||
if (site && site.cookies) {
|
||||
cookies = site.cookies;
|
||||
job.config.cookies = cookies;
|
||||
addLog(job, `Loaded cookies from forum site: ${site.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const downloadedSet = new Set();
|
||||
let totalImages = 0;
|
||||
|
||||
// If lastPageOnly, detect the last page and only scrape that
|
||||
if (lastPageOnly) {
|
||||
addLog(job, 'Detecting last page...');
|
||||
const maxPage = await detectMaxPage(url, (msg) => addLog(job, msg), cookies);
|
||||
if (maxPage) {
|
||||
startPage = maxPage;
|
||||
endPage = maxPage;
|
||||
addLog(job, `Last page detected: ${maxPage}`);
|
||||
} else {
|
||||
addLog(job, 'Could not detect last page — falling back to page range');
|
||||
}
|
||||
}
|
||||
|
||||
addLog(job, `Starting forum scrape: pages ${startPage}-${endPage}`);
|
||||
addLog(job, `Output: ${outputDir}`);
|
||||
|
||||
@@ -88,7 +120,31 @@ async function runForumScrape(job) {
|
||||
const pageUrl = getPageUrl(url, page);
|
||||
addLog(job, `--- Page ${page}/${endPage} ---`);
|
||||
|
||||
const count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg));
|
||||
let count;
|
||||
try {
|
||||
count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
|
||||
} catch (err) {
|
||||
if (err instanceof CookieExpiredError && siteId) {
|
||||
addLog(job, `Cookie expired (HTTP ${err.statusCode}) — attempting auto-refresh via FlareSolverr...`);
|
||||
try {
|
||||
cookies = await refreshForumCookies(siteId);
|
||||
job.config.cookies = cookies;
|
||||
addLog(job, 'Cookies refreshed successfully — retrying page...');
|
||||
count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
|
||||
} catch (refreshErr) {
|
||||
addLog(job, `Cookie refresh failed: ${refreshErr.message}`);
|
||||
addLog(job, 'Stopping scrape — fix credentials or refresh cookies manually');
|
||||
break;
|
||||
}
|
||||
} else if (err instanceof CookieExpiredError) {
|
||||
addLog(job, `Cookie expired (HTTP ${err.statusCode}) — no siteId configured for auto-refresh`);
|
||||
addLog(job, 'Stopping scrape — refresh cookies manually and try again');
|
||||
break;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
totalImages += count;
|
||||
job.progress.completed = page - startPage + 1;
|
||||
|
||||
@@ -102,7 +158,7 @@ async function runForumScrape(job) {
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
addLog(job, `Done! ${totalImages} images saved to ${folderName}/`);
|
||||
addLog(job, `Done! ${totalImages} files saved to ${folderName}/`);
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
@@ -118,15 +174,24 @@ async function runCoomerScrape(job) {
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}`);
|
||||
|
||||
try {
|
||||
const { base, service, userId } = parseUserUrl(url);
|
||||
addLog(job, `Site: ${base}, Service: ${service}, User: ${userId}`);
|
||||
const parsed = parseUserUrl(url);
|
||||
let files;
|
||||
|
||||
// Phase 1: Collect files
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
const files = await fetchAllPosts(base, service, userId, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
if (parsed.mode === 'search') {
|
||||
addLog(job, `Site: ${parsed.base}, Search: "${parsed.query}"`);
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
files = await fetchSearchPosts(parsed.base, parsed.query, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
} else {
|
||||
addLog(job, `Site: ${parsed.base}, Service: ${parsed.service}, User: ${parsed.userId}`);
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
files = await fetchAllPosts(parsed.base, parsed.service, parsed.userId, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
}
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
@@ -174,12 +239,170 @@ async function runMediaLinkScrape(job) {
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
|
||||
|
||||
try {
|
||||
const { base, userId } = parseMediaUrl(url);
|
||||
addLog(job, `Site: ${base}, User ID: ${userId}`);
|
||||
const { base, userId, mode } = parseMediaUrl(url);
|
||||
addLog(job, `Site: ${base}, ${mode === 'html' ? 'Slug' : 'User ID'}: ${userId} (${mode} mode)`);
|
||||
|
||||
// Phase 1: Collect all media via JSON API
|
||||
// Phase 1: Collect all media
|
||||
let items;
|
||||
if (mode === 'html') {
|
||||
addLog(job, `Fetching up to ${pages} pages via HTML scraping...`);
|
||||
items = await fetchAllMediaFromHtml(base, userId, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
} else {
|
||||
addLog(job, `Fetching up to ${pages} pages from API...`);
|
||||
items = await fetchAllMedia(base, userId, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
}
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
addLog(job, 'No media found');
|
||||
return;
|
||||
}
|
||||
|
||||
job.progress.total = items.length;
|
||||
addLog(job, `Found ${items.length} media items. Downloading...`);
|
||||
|
||||
// Phase 2: Download all media files
|
||||
const result = await downloadMedia(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors = errors;
|
||||
job.progress.total = total;
|
||||
},
|
||||
() => job.cancelled,
|
||||
base + '/'
|
||||
);
|
||||
|
||||
addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Mega Scrape ---
|
||||
|
||||
async function runMegaScrape(job) {
|
||||
const { url, workers, folderName } = job.config;
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
addLog(job, `Starting mega.nz scrape: ${url}`);
|
||||
addLog(job, `Workers: ${workers}`);
|
||||
|
||||
try {
|
||||
parseMegaUrl(url);
|
||||
|
||||
// Phase 1: List all files
|
||||
const { folderName: megaName, items } = await listAllFiles(url,
|
||||
(msg) => addLog(job, msg)
|
||||
);
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
addLog(job, 'No files found in folder');
|
||||
return;
|
||||
}
|
||||
|
||||
job.progress.total = items.length;
|
||||
const totalSizeMb = (items.reduce((s, i) => s + i.size, 0) / (1024 * 1024)).toFixed(0);
|
||||
addLog(job, `Found ${items.length} files (${totalSizeMb} MB). Downloading...`);
|
||||
|
||||
// Phase 2: Download
|
||||
const result = await downloadMegaFiles(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors = errors;
|
||||
job.progress.total = total;
|
||||
},
|
||||
() => job.cancelled,
|
||||
(status) => {
|
||||
job.paused = status.paused;
|
||||
job.resumeAt = status.resumeAt;
|
||||
}
|
||||
);
|
||||
|
||||
addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- yt-dlp Scrape ---
|
||||
|
||||
async function runYtdlpScrape(job) {
|
||||
const config = job.config;
|
||||
addLog(job, `Starting yt-dlp download: ${config.url}`);
|
||||
addLog(job, `Quality: ${config.quality || 'best'}, Playlist: ${config.playlist ? 'yes' : 'no'}`);
|
||||
|
||||
try {
|
||||
const result = await runYtdlp(
|
||||
config,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors += errors;
|
||||
if (completed > job.progress.total) job.progress.total = completed;
|
||||
},
|
||||
() => job.cancelled
|
||||
);
|
||||
|
||||
if (result.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
} else {
|
||||
addLog(job, `Done! ${result.files} file${result.files !== 1 ? 's' : ''} downloaded`);
|
||||
}
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- LeakGallery Scrape ---
|
||||
|
||||
async function runLeakGalleryScrape(job) {
|
||||
const { url, pages, workers, delay, folderName } = job.config;
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
addLog(job, `Starting leakgallery scrape: ${url}`);
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
|
||||
|
||||
try {
|
||||
const { username } = parseLeakGalleryUrl(url);
|
||||
addLog(job, `Username: ${username}`);
|
||||
|
||||
// Phase 1: Collect all media
|
||||
addLog(job, `Fetching up to ${pages} pages from API...`);
|
||||
const items = await fetchAllMedia(base, userId, pages, delay,
|
||||
const items = await fetchLeakGalleryMedia(username, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
@@ -198,7 +421,7 @@ async function runMediaLinkScrape(job) {
|
||||
addLog(job, `Found ${items.length} media items. Downloading...`);
|
||||
|
||||
// Phase 2: Download all media files
|
||||
const result = await downloadMedia(items, outputDir, workers,
|
||||
const result = await downloadLeakGalleryMedia(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
@@ -222,7 +445,7 @@ async function runMediaLinkScrape(job) {
|
||||
// --- Endpoints ---
|
||||
|
||||
router.post('/api/scrape/forum', (req, res) => {
|
||||
const { url, folderName, startPage, endPage, delay } = req.body;
|
||||
const { url, folderName, startPage, endPage, delay, cookies, siteId, lastPageOnly } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
@@ -232,6 +455,9 @@ router.post('/api/scrape/forum', (req, res) => {
|
||||
startPage: parseInt(startPage) || 1,
|
||||
endPage: parseInt(endPage) || 10,
|
||||
delay: parseFloat(delay) || 1.0,
|
||||
cookies: cookies || '',
|
||||
siteId: siteId ? parseInt(siteId, 10) : null,
|
||||
lastPageOnly: !!lastPageOnly,
|
||||
};
|
||||
|
||||
const job = createJob('forum', config);
|
||||
@@ -289,6 +515,105 @@ router.post('/api/scrape/medialink', (req, res) => {
|
||||
res.json({ jobId: job.id, message: 'MediaLink scrape started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/mega', (req, res) => {
|
||||
const { url, folderName, workers } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
try {
|
||||
parseMegaUrl(url);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: err.message });
|
||||
}
|
||||
|
||||
const config = {
|
||||
url,
|
||||
folderName,
|
||||
workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
|
||||
};
|
||||
|
||||
const job = createJob('mega', config);
|
||||
runMegaScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'Mega scrape started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/ytdlp', (req, res) => {
|
||||
const { url, quality, customFormat, embedMetadata, embedThumbnail, embedSubs,
|
||||
writeSubs, subLangs, restrictFilenames, outputTemplate,
|
||||
playlist, maxDownloads, concurrentFragments, rateLimit,
|
||||
sponsorBlock, cookiesFile } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
|
||||
const config = {
|
||||
url,
|
||||
quality: quality || 'best',
|
||||
customFormat: customFormat || '',
|
||||
embedMetadata: embedMetadata !== false,
|
||||
embedThumbnail: embedThumbnail !== false,
|
||||
embedSubs: embedSubs !== false,
|
||||
writeSubs: writeSubs || false,
|
||||
subLangs: subLangs || 'en',
|
||||
restrictFilenames: restrictFilenames !== false,
|
||||
outputTemplate: outputTemplate || '%(title)s.%(ext)s',
|
||||
playlist: playlist || false,
|
||||
maxDownloads: parseInt(maxDownloads) || 0,
|
||||
concurrentFragments: Math.min(Math.max(parseInt(concurrentFragments) || 4, 1), 16),
|
||||
rateLimit: rateLimit || '',
|
||||
sponsorBlock: sponsorBlock || 'off',
|
||||
cookiesFile: cookiesFile || '',
|
||||
folderName: (() => {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
const path = u.pathname.replace(/^\//, '').replace(/\/$/, '');
|
||||
return path ? `${u.hostname}/${path}`.slice(0, 60) : u.hostname;
|
||||
} catch { return url.slice(0, 60); }
|
||||
})(),
|
||||
};
|
||||
|
||||
const job = createJob('ytdlp', config);
|
||||
runYtdlpScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'yt-dlp download started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/leakgallery', (req, res) => {
|
||||
const { url, folderName, pages, workers, delay } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
try {
|
||||
parseLeakGalleryUrl(url);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: err.message });
|
||||
}
|
||||
|
||||
const config = {
|
||||
url,
|
||||
folderName,
|
||||
pages: parseInt(pages) || 100,
|
||||
workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
|
||||
delay: parseInt(delay) || 300,
|
||||
};
|
||||
|
||||
const job = createJob('leakgallery', config);
|
||||
runLeakGalleryScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'LeakGallery scrape started' });
|
||||
});
|
||||
|
||||
router.get('/api/scrape/jobs', (_req, res) => {
|
||||
const jobs = [...jobsMap.values()].map(jobToJson);
|
||||
jobs.sort((a, b) => new Date(b.startedAt) - new Date(a.startedAt));
|
||||
@@ -310,13 +635,95 @@ router.post('/api/scrape/jobs/:jobId/cancel', (req, res) => {
|
||||
res.json({ message: 'Cancel requested' });
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/jobs/:jobId', (req, res) => {
|
||||
const job = jobsMap.get(req.params.jobId);
|
||||
if (!job) return res.status(404).json({ error: 'Job not found' });
|
||||
job.cancelled = true;
|
||||
job.running = false;
|
||||
jobsMap.delete(req.params.jobId);
|
||||
res.json({ message: 'Job removed' });
|
||||
});
|
||||
|
||||
// Auto-detect max page for forum URLs
|
||||
router.post('/api/scrape/forum/detect-pages', async (req, res) => {
|
||||
const { url } = req.body;
|
||||
const { url, cookies } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
const logs = [];
|
||||
const maxPage = await detectMaxPage(url, (msg) => logs.push(msg));
|
||||
const maxPage = await detectMaxPage(url, (msg) => logs.push(msg), cookies);
|
||||
res.json({ maxPage, logs });
|
||||
});
|
||||
|
||||
// --- Forum Sites CRUD ---
|
||||
|
||||
router.get('/api/scrape/forum-sites', (_req, res) => {
|
||||
res.json(getForumSites());
|
||||
});
|
||||
|
||||
router.post('/api/scrape/forum-sites', (req, res) => {
|
||||
const { name, baseUrl, cookies, username, password } = req.body;
|
||||
if (!name) return res.status(400).json({ error: 'Name is required' });
|
||||
const id = createForumSite(name, baseUrl, cookies, username, password);
|
||||
res.json(getForumSiteById(id));
|
||||
});
|
||||
|
||||
router.put('/api/scrape/forum-sites/:id', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const site = getForumSiteById(id);
|
||||
if (!site) return res.status(404).json({ error: 'Forum site not found' });
|
||||
const { name, baseUrl, cookies, username, password } = req.body;
|
||||
const fields = {};
|
||||
if (name !== undefined) fields.name = name;
|
||||
if (baseUrl !== undefined) fields.base_url = baseUrl;
|
||||
if (cookies !== undefined) fields.cookies = cookies;
|
||||
if (username !== undefined) fields.username = username;
|
||||
if (password !== undefined) fields.password = password;
|
||||
updateForumSite(id, fields);
|
||||
res.json(getForumSiteById(id));
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/forum-sites/:id', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
deleteForumSite(id);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// --- Auto-scrape CRUD ---
|
||||
|
||||
router.get('/api/scrape/auto', (_req, res) => {
|
||||
res.json(getAutoScrapeJobs());
|
||||
});
|
||||
|
||||
router.post('/api/scrape/auto', (req, res) => {
|
||||
const { type, url, folderName, config } = req.body;
|
||||
if (!type || !url || !folderName || !config) {
|
||||
return res.status(400).json({ error: 'type, url, folderName, and config are required' });
|
||||
}
|
||||
addAutoScrapeJob(type, url, folderName, config);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/auto/:id', (req, res) => {
|
||||
removeAutoScrapeJob(parseInt(req.params.id));
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
export function getActiveScrapeCount() {
|
||||
let count = 0;
|
||||
for (const job of jobsMap.values()) {
|
||||
if (job.running) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getActiveScrapesList() {
|
||||
const list = [];
|
||||
for (const job of jobsMap.values()) {
|
||||
if (job.running) {
|
||||
list.push({ type: job.type, folderName: job.folderName, progress: job.progress });
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
export { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, runYtdlpScrape, runLeakGalleryScrape, createJob };
|
||||
export default router;
|
||||
|
||||
Reference in New Issue
Block a user