Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
@@ -0,0 +1,191 @@
|
||||
import { existsSync, writeFileSync, mkdirSync } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
|
||||
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
const API_BASE = 'https://api.leakgallery.com';
|
||||
const CDN_BASE = 'https://cdn.leakgallery.com';
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
|
||||
|
||||
export function parseLeakGalleryUrl(url) {
|
||||
const parsed = new URL(url);
|
||||
if (!parsed.hostname.includes('leakgallery.com')) {
|
||||
throw new Error('Not a leakgallery.com URL');
|
||||
}
|
||||
// URL format: https://leakgallery.com/{username}
|
||||
const m = parsed.pathname.match(/^\/([a-zA-Z0-9_.-]+)\/?$/);
|
||||
if (!m) throw new Error('Expected URL format: https://leakgallery.com/username');
|
||||
return { username: m[1] };
|
||||
}
|
||||
|
||||
async function fetchPage(username, page, logFn) {
|
||||
// Page 1: /profile/{username}?type=All&sort=MostRecent
|
||||
// Page 2+: /profile/{username}/{page}?type=All&sort=MostRecent
|
||||
const pagePath = page <= 1 ? '' : `/${page}`;
|
||||
const apiUrl = `${API_BASE}/profile/${username}${pagePath}?type=All&sort=MostRecent`;
|
||||
|
||||
try {
|
||||
const resp = await fetch(apiUrl, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Accept': 'application/json',
|
||||
'Origin': 'https://leakgallery.com',
|
||||
'Referer': 'https://leakgallery.com/',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
if (resp.status === 404) return null;
|
||||
logFn(`API error (${resp.status}): ${apiUrl}`);
|
||||
return null;
|
||||
}
|
||||
return await resp.json();
|
||||
} catch (err) {
|
||||
logFn(`API fetch error: ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchAllMedia(username, maxPages, delay, logFn, checkCancelled) {
|
||||
const allItems = [];
|
||||
const seen = new Set();
|
||||
let totalCount = 0;
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
if (checkCancelled()) break;
|
||||
|
||||
logFn(`Fetching page ${page}...`);
|
||||
const data = await fetchPage(username, page, logFn);
|
||||
|
||||
if (!data) {
|
||||
logFn(`Page ${page}: no data — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
// First page includes mediaCount
|
||||
if (page === 1 && data.mediaCount) {
|
||||
totalCount = data.mediaCount;
|
||||
logFn(`Profile has ${totalCount} total media items`);
|
||||
}
|
||||
|
||||
const medias = data.medias;
|
||||
if (!medias || !Array.isArray(medias) || medias.length === 0) {
|
||||
logFn(`Page ${page}: no more items — done`);
|
||||
break;
|
||||
}
|
||||
|
||||
let newCount = 0;
|
||||
for (const item of medias) {
|
||||
if (seen.has(item.id)) continue;
|
||||
seen.add(item.id);
|
||||
newCount++;
|
||||
|
||||
// file_path is relative, e.g. content4/username/watermark_hash__username__id_580px.webp
|
||||
// Full-size: remove _580px.webp suffix, use .jpg (or .mp4 for videos)
|
||||
const isVideo = !!item.is_video;
|
||||
let fullUrl;
|
||||
let filename;
|
||||
|
||||
if (isVideo) {
|
||||
// Videos: file_path is already the video file
|
||||
fullUrl = `${CDN_BASE}/${item.file_path}`;
|
||||
filename = basename(item.file_path);
|
||||
} else {
|
||||
// Images: thumbnail has _580px.webp — convert to full-size .jpg
|
||||
const filePath = item.file_path || item.thumbnail_path || '';
|
||||
const fullPath = filePath
|
||||
.replace(/_580px\.webp$/, '.jpg')
|
||||
.replace(/_300px\.webp$/, '.jpg');
|
||||
fullUrl = `${CDN_BASE}/${fullPath}`;
|
||||
filename = basename(fullPath);
|
||||
}
|
||||
|
||||
allItems.push({
|
||||
id: item.id,
|
||||
url: fullUrl,
|
||||
filename,
|
||||
type: isVideo ? 'video' : 'image',
|
||||
});
|
||||
}
|
||||
|
||||
if (newCount === 0) {
|
||||
logFn(`Page ${page}: all duplicates — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
logFn(`Page ${page}: ${medias.length} items (${newCount} new, ${allItems.length} total)`);
|
||||
|
||||
if (page < maxPages && !checkCancelled()) {
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
async function tryFetch(url) {
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Referer': 'https://leakgallery.com/',
|
||||
},
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!resp.ok) return null;
|
||||
const buf = Buffer.from(await resp.arrayBuffer());
|
||||
if (buf.length < 500) return null;
|
||||
return buf;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
let index = 0;
|
||||
|
||||
async function processNext() {
|
||||
while (index < items.length) {
|
||||
if (checkCancelled()) return;
|
||||
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
const filename = item.filename || `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
|
||||
const filepath = join(outputDir, filename);
|
||||
|
||||
if (existsSync(filepath)) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
const buf = await tryFetch(item.url);
|
||||
if (buf) {
|
||||
writeFileSync(filepath, buf);
|
||||
const folderName = basename(outputDir);
|
||||
const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
|
||||
try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
|
||||
completed++;
|
||||
logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} else {
|
||||
logFn(`FAILED: ${filename}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < Math.min(workers, items.length); i++) {
|
||||
workerPromises.push(processNext());
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
return { completed, errors, skipped, total: items.length };
|
||||
}
|
||||
Reference in New Issue
Block a user