OFApp/server/scrapers/leakgallery.js

import { existsSync, writeFileSync, mkdirSync } from 'fs';
import { basename, join, extname } from 'path';
import { upsertMediaFile } from '../db.js';

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const API_BASE = 'https://api.leakgallery.com';
const CDN_BASE = 'https://cdn.leakgallery.com';
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);

export function parseLeakGalleryUrl(url) {
  const parsed = new URL(url);
  if (!parsed.hostname.includes('leakgallery.com')) {
    throw new Error('Not a leakgallery.com URL');
  }
  // URL format: https://leakgallery.com/{username}
  const m = parsed.pathname.match(/^\/([a-zA-Z0-9_.-]+)\/?$/);
  if (!m) throw new Error('Expected URL format: https://leakgallery.com/username');
  return { username: m[1] };
}

async function fetchPage(username, page, logFn) {
  // Page 1: /profile/{username}?type=All&sort=MostRecent
  // Page 2+: /profile/{username}/{page}?type=All&sort=MostRecent
  const pagePath = page <= 1 ? '' : `/${page}`;
  const apiUrl = `${API_BASE}/profile/${username}${pagePath}?type=All&sort=MostRecent`;

  try {
    const resp = await fetch(apiUrl, {
      headers: {
        'User-Agent': UA,
        'Accept': 'application/json',
        'Origin': 'https://leakgallery.com',
        'Referer': 'https://leakgallery.com/',
      },
      signal: AbortSignal.timeout(15000),
    });
    if (!resp.ok) {
      if (resp.status === 404) return null;
      logFn(`API error (${resp.status}): ${apiUrl}`);
      return null;
    }
    return await resp.json();
  } catch (err) {
    logFn(`API fetch error: ${err.message}`);
    return null;
  }
}

export async function fetchAllMedia(username, maxPages, delay, logFn, checkCancelled) {
  const allItems = [];
  const seen = new Set();
  let totalCount = 0;

  for (let page = 1; page <= maxPages; page++) {
    if (checkCancelled()) break;

    logFn(`Fetching page ${page}...`);
    const data = await fetchPage(username, page, logFn);

    if (!data) {
      logFn(`Page ${page}: no data — stopping`);
      break;
    }

    // First page includes mediaCount
    if (page === 1 && data.mediaCount) {
      totalCount = data.mediaCount;
      logFn(`Profile has ${totalCount} total media items`);
    }

    const medias = data.medias;
    if (!medias || !Array.isArray(medias) || medias.length === 0) {
      logFn(`Page ${page}: no more items — done`);
      break;
    }

    let newCount = 0;
    for (const item of medias) {
      if (seen.has(item.id)) continue;
      seen.add(item.id);
      newCount++;

      // file_path is relative, e.g. content4/username/watermark_hash__username__id_580px.webp
      // Full-size: remove _580px.webp suffix, use .jpg (or .mp4 for videos)
      const isVideo = !!item.is_video;
      let fullUrl;
      let filename;

      if (isVideo) {
        // Videos: file_path is already the video file
        fullUrl = `${CDN_BASE}/${item.file_path}`;
        filename = basename(item.file_path);
      } else {
        // Images: thumbnail has _580px.webp — convert to full-size .jpg
        const filePath = item.file_path || item.thumbnail_path || '';
        const fullPath = filePath
          .replace(/_580px\.webp$/, '.jpg')
          .replace(/_300px\.webp$/, '.jpg');
        fullUrl = `${CDN_BASE}/${fullPath}`;
        filename = basename(fullPath);
      }

      allItems.push({
        id: item.id,
        url: fullUrl,
        filename,
        type: isVideo ? 'video' : 'image',
      });
    }

    if (newCount === 0) {
      logFn(`Page ${page}: all duplicates — stopping`);
      break;
    }

    logFn(`Page ${page}: ${medias.length} items (${newCount} new, ${allItems.length} total)`);

    if (page < maxPages && !checkCancelled()) {
      await new Promise(r => setTimeout(r, delay));
    }
  }

  return allItems;
}

async function tryFetch(url) {
  try {
    const resp = await fetch(url, {
      headers: {
        'User-Agent': UA,
        'Referer': 'https://leakgallery.com/',
      },
      signal: AbortSignal.timeout(60000),
    });
    if (!resp.ok) return null;
    const buf = Buffer.from(await resp.arrayBuffer());
    if (buf.length < 500) return null;
    return buf;
  } catch {
    return null;
  }
}

export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
  mkdirSync(outputDir, { recursive: true });

  let completed = 0;
  let errors = 0;
  let skipped = 0;
  let index = 0;

  async function processNext() {
    while (index < items.length) {
      if (checkCancelled()) return;

      const current = index++;
      const item = items[current];
      const filename = item.filename || `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
      const filepath = join(outputDir, filename);

      if (existsSync(filepath)) {
        skipped++;
        progressFn(completed + skipped, errors, items.length);
        continue;
      }

      const buf = await tryFetch(item.url);
      if (buf) {
        writeFileSync(filepath, buf);
        const folderName = basename(outputDir);
        const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
        try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
        completed++;
        logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
        progressFn(completed + skipped, errors, items.length);
      } else {
        logFn(`FAILED: ${filename}`);
        errors++;
        progressFn(completed + skipped, errors, items.length);
      }
    }
  }

  const workerPromises = [];
  for (let i = 0; i < Math.min(workers, items.length); i++) {
    workerPromises.push(processNext());
  }
  await Promise.all(workerPromises);

  return { completed, errors, skipped, total: items.length };
}