Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
@@ -0,0 +1,219 @@
|
||||
import { File } from 'megajs';
|
||||
import { existsSync, mkdirSync, statSync, unlinkSync } from 'fs';
|
||||
import { createWriteStream } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
|
||||
const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);
|
||||
|
||||
export function parseMegaUrl(url) {
|
||||
// Validate it's a mega.nz folder URL
|
||||
const parsed = new URL(url);
|
||||
if (!parsed.hostname.includes('mega.nz') && !parsed.hostname.includes('mega.co.nz')) {
|
||||
throw new Error('Not a mega.nz URL');
|
||||
}
|
||||
if (!parsed.pathname.includes('/folder/')) {
|
||||
throw new Error('Expected a mega.nz folder URL (e.g. https://mega.nz/folder/ABC#key)');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// Load shared folder and list all files recursively
|
||||
export async function listAllFiles(url, logFn) {
|
||||
logFn('Loading shared folder...');
|
||||
const folder = File.fromURL(url);
|
||||
await folder.loadAttributes();
|
||||
|
||||
const folderName = folder.name || 'mega_folder';
|
||||
logFn(`Folder: ${folderName}`);
|
||||
|
||||
// Recursively get all non-directory files
|
||||
const allFiles = folder.filter(f => !f.directory, true);
|
||||
logFn(`Found ${allFiles.length} files across all subfolders`);
|
||||
|
||||
// Build items with subfolder paths
|
||||
const items = [];
|
||||
for (const file of allFiles) {
|
||||
const ext = extname(file.name).toLowerCase();
|
||||
let type = 'other';
|
||||
if (IMAGE_EXTS.has(ext)) type = 'image';
|
||||
else if (VIDEO_EXTS.has(ext)) type = 'video';
|
||||
|
||||
// Build relative path from parent folders
|
||||
let subfolder = '';
|
||||
let parent = file.parent;
|
||||
const parts = [];
|
||||
while (parent && parent !== folder) {
|
||||
parts.unshift(parent.name);
|
||||
parent = parent.parent;
|
||||
}
|
||||
subfolder = parts.join('/');
|
||||
|
||||
items.push({
|
||||
file,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type,
|
||||
subfolder,
|
||||
});
|
||||
}
|
||||
|
||||
return { folderName, items };
|
||||
}
|
||||
|
||||
// Parse bandwidth limit wait time from error message
|
||||
function parseBandwidthWait(errMsg) {
|
||||
const m = errMsg.match(/(\d+)\s*seconds?\s*until/i);
|
||||
if (m) return parseInt(m[1], 10);
|
||||
if (/bandwidth/i.test(errMsg)) return 3600; // default 1hr if can't parse
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Download all files with concurrency + bandwidth limit auto-retry
|
||||
export async function downloadMegaFiles(items, outputDir, workers, logFn, progressFn, checkCancelled, statusFn) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
let index = 0;
|
||||
let bandwidthPaused = false;
|
||||
|
||||
async function processNext() {
|
||||
while (index < items.length) {
|
||||
if (checkCancelled()) return;
|
||||
|
||||
// If another worker hit the bandwidth limit, wait for it to clear
|
||||
if (bandwidthPaused) return;
|
||||
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
|
||||
// All files go to root output dir (flatten subfolders)
|
||||
const filepath = join(outputDir, item.name);
|
||||
|
||||
// Skip if file exists AND is non-empty (0-byte = failed partial download)
|
||||
if (existsSync(filepath)) {
|
||||
try {
|
||||
const st = statSync(filepath);
|
||||
if (st.size > 0) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
// Remove 0-byte leftover from previous failed download
|
||||
unlinkSync(filepath);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = item.file.download();
|
||||
await pipeline(stream, createWriteStream(filepath));
|
||||
|
||||
// Verify the file was actually written
|
||||
let actualSize = item.size;
|
||||
try { actualSize = statSync(filepath).size; } catch {}
|
||||
|
||||
const folderName = basename(outputDir);
|
||||
const ext = extname(item.name).toLowerCase();
|
||||
const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
|
||||
try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
|
||||
|
||||
completed++;
|
||||
const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
|
||||
logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} catch (err) {
|
||||
// Clean up partial/empty file on any error
|
||||
try { unlinkSync(filepath); } catch {}
|
||||
|
||||
const waitSecs = parseBandwidthWait(err.message);
|
||||
if (waitSecs > 0) {
|
||||
// Bandwidth limit — put this item back and pause all workers
|
||||
index = current; // rewind so this file gets retried
|
||||
bandwidthPaused = true;
|
||||
const waitMins = Math.ceil(waitSecs / 60);
|
||||
const resumeAt = Date.now() + waitSecs * 1000;
|
||||
logFn(`Bandwidth limit reached — waiting ${waitMins} minutes for quota reset...`);
|
||||
if (statusFn) statusFn({ paused: true, resumeAt });
|
||||
await new Promise(r => setTimeout(r, waitSecs * 1000));
|
||||
if (checkCancelled()) return;
|
||||
if (statusFn) statusFn({ paused: false, resumeAt: null });
|
||||
logFn('Quota reset — resuming downloads...');
|
||||
bandwidthPaused = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
logFn(`FAILED: ${item.name} — ${err.message}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < Math.min(workers, items.length); i++) {
|
||||
workerPromises.push(processNext());
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
// If we paused for bandwidth and there are remaining files, run single-threaded to finish
|
||||
while (index < items.length && !checkCancelled()) {
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
const filepath = join(outputDir, item.name);
|
||||
|
||||
if (existsSync(filepath)) {
|
||||
try {
|
||||
const st = statSync(filepath);
|
||||
if (st.size > 0) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
unlinkSync(filepath);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = item.file.download();
|
||||
await pipeline(stream, createWriteStream(filepath));
|
||||
|
||||
let actualSize = item.size;
|
||||
try { actualSize = statSync(filepath).size; } catch {}
|
||||
|
||||
const folderName = basename(outputDir);
|
||||
const ext = extname(item.name).toLowerCase();
|
||||
const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
|
||||
try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
|
||||
|
||||
completed++;
|
||||
const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
|
||||
logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} catch (err) {
|
||||
try { unlinkSync(filepath); } catch {}
|
||||
|
||||
const waitSecs = parseBandwidthWait(err.message);
|
||||
if (waitSecs > 0) {
|
||||
index = current;
|
||||
const waitMins = Math.ceil(waitSecs / 60);
|
||||
const resumeAt = Date.now() + waitSecs * 1000;
|
||||
logFn(`Bandwidth limit reached — waiting ${waitMins} minutes...`);
|
||||
if (statusFn) statusFn({ paused: true, resumeAt });
|
||||
await new Promise(r => setTimeout(r, waitSecs * 1000));
|
||||
if (checkCancelled()) break;
|
||||
if (statusFn) statusFn({ paused: false, resumeAt: null });
|
||||
logFn('Quota reset — resuming...');
|
||||
continue;
|
||||
}
|
||||
logFn(`FAILED: ${item.name} — ${err.message}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
|
||||
return { completed, errors, skipped, total: items.length };
|
||||
}
|
||||
Reference in New Issue
Block a user