import { existsSync, writeFileSync, mkdirSync } from 'fs'; import { basename, join, extname } from 'path'; import { upsertMediaFile } from '../db.js'; const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; const API_BASE = 'https://api.leakgallery.com'; const CDN_BASE = 'https://cdn.leakgallery.com'; const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']); export function parseLeakGalleryUrl(url) { const parsed = new URL(url); if (!parsed.hostname.includes('leakgallery.com')) { throw new Error('Not a leakgallery.com URL'); } // URL format: https://leakgallery.com/{username} const m = parsed.pathname.match(/^\/([a-zA-Z0-9_.-]+)\/?$/); if (!m) throw new Error('Expected URL format: https://leakgallery.com/username'); return { username: m[1] }; } async function fetchPage(username, page, logFn) { // Page 1: /profile/{username}?type=All&sort=MostRecent // Page 2+: /profile/{username}/{page}?type=All&sort=MostRecent const pagePath = page <= 1 ? '' : `/${page}`; const apiUrl = `${API_BASE}/profile/${username}${pagePath}?type=All&sort=MostRecent`; try { const resp = await fetch(apiUrl, { headers: { 'User-Agent': UA, 'Accept': 'application/json', 'Origin': 'https://leakgallery.com', 'Referer': 'https://leakgallery.com/', }, signal: AbortSignal.timeout(15000), }); if (!resp.ok) { if (resp.status === 404) return null; logFn(`API error (${resp.status}): ${apiUrl}`); return null; } return await resp.json(); } catch (err) { logFn(`API fetch error: ${err.message}`); return null; } } export async function fetchAllMedia(username, maxPages, delay, logFn, checkCancelled) { const allItems = []; const seen = new Set(); let totalCount = 0; for (let page = 1; page <= maxPages; page++) { if (checkCancelled()) break; logFn(`Fetching page ${page}...`); const data = await fetchPage(username, page, logFn); if (!data) { logFn(`Page ${page}: no data — stopping`); break; } // First page includes mediaCount if (page === 1 && data.mediaCount) { totalCount = data.mediaCount; logFn(`Profile has ${totalCount} total media items`); } const medias = data.medias; if (!medias || !Array.isArray(medias) || medias.length === 0) { logFn(`Page ${page}: no more items — done`); break; } let newCount = 0; for (const item of medias) { if (seen.has(item.id)) continue; seen.add(item.id); newCount++; // file_path is relative, e.g. content4/username/watermark_hash__username__id_580px.webp // Full-size: remove _580px.webp suffix, use .jpg (or .mp4 for videos) const isVideo = !!item.is_video; let fullUrl; let filename; if (isVideo) { // Videos: file_path is already the video file fullUrl = `${CDN_BASE}/${item.file_path}`; filename = basename(item.file_path); } else { // Images: thumbnail has _580px.webp — convert to full-size .jpg const filePath = item.file_path || item.thumbnail_path || ''; const fullPath = filePath .replace(/_580px\.webp$/, '.jpg') .replace(/_300px\.webp$/, '.jpg'); fullUrl = `${CDN_BASE}/${fullPath}`; filename = basename(fullPath); } allItems.push({ id: item.id, url: fullUrl, filename, type: isVideo ? 'video' : 'image', }); } if (newCount === 0) { logFn(`Page ${page}: all duplicates — stopping`); break; } logFn(`Page ${page}: ${medias.length} items (${newCount} new, ${allItems.length} total)`); if (page < maxPages && !checkCancelled()) { await new Promise(r => setTimeout(r, delay)); } } return allItems; } async function tryFetch(url) { try { const resp = await fetch(url, { headers: { 'User-Agent': UA, 'Referer': 'https://leakgallery.com/', }, signal: AbortSignal.timeout(60000), }); if (!resp.ok) return null; const buf = Buffer.from(await resp.arrayBuffer()); if (buf.length < 500) return null; return buf; } catch { return null; } } export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) { mkdirSync(outputDir, { recursive: true }); let completed = 0; let errors = 0; let skipped = 0; let index = 0; async function processNext() { while (index < items.length) { if (checkCancelled()) return; const current = index++; const item = items[current]; const filename = item.filename || `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`; const filepath = join(outputDir, filename); if (existsSync(filepath)) { skipped++; progressFn(completed + skipped, errors, items.length); continue; } const buf = await tryFetch(item.url); if (buf) { writeFileSync(filepath, buf); const folderName = basename(outputDir); const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image'; try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {} completed++; logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`); progressFn(completed + skipped, errors, items.length); } else { logFn(`FAILED: ${filename}`); errors++; progressFn(completed + skipped, errors, items.length); } } } const workerPromises = []; for (let i = 0; i < Math.min(workers, items.length); i++) { workerPromises.push(processNext()); } await Promise.all(workerPromises); return { completed, errors, skipped, total: items.length }; }