OFApp/server/scrapers/mega.js

import { File } from 'megajs';
import { existsSync, mkdirSync, statSync, unlinkSync } from 'fs';
import { createWriteStream } from 'fs';
import { basename, join, extname } from 'path';
import { pipeline } from 'stream/promises';
import { upsertMediaFile } from '../db.js';

const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);

export function parseMegaUrl(url) {
  // Validate it's a mega.nz folder URL
  const parsed = new URL(url);
  if (!parsed.hostname.includes('mega.nz') && !parsed.hostname.includes('mega.co.nz')) {
    throw new Error('Not a mega.nz URL');
  }
  if (!parsed.pathname.includes('/folder/')) {
    throw new Error('Expected a mega.nz folder URL (e.g. https://mega.nz/folder/ABC#key)');
  }
  return url;
}

// Load shared folder and list all files recursively
export async function listAllFiles(url, logFn) {
  logFn('Loading shared folder...');
  const folder = File.fromURL(url);
  await folder.loadAttributes();

  const folderName = folder.name || 'mega_folder';
  logFn(`Folder: ${folderName}`);

  // Recursively get all non-directory files
  const allFiles = folder.filter(f => !f.directory, true);
  logFn(`Found ${allFiles.length} files across all subfolders`);

  // Build items with subfolder paths
  const items = [];
  for (const file of allFiles) {
    const ext = extname(file.name).toLowerCase();
    let type = 'other';
    if (IMAGE_EXTS.has(ext)) type = 'image';
    else if (VIDEO_EXTS.has(ext)) type = 'video';

    // Build relative path from parent folders
    let subfolder = '';
    let parent = file.parent;
    const parts = [];
    while (parent && parent !== folder) {
      parts.unshift(parent.name);
      parent = parent.parent;
    }
    subfolder = parts.join('/');

    items.push({
      file,
      name: file.name,
      size: file.size,
      type,
      subfolder,
    });
  }

  return { folderName, items };
}

// Parse bandwidth limit wait time from error message
function parseBandwidthWait(errMsg) {
  const m = errMsg.match(/(\d+)\s*seconds?\s*until/i);
  if (m) return parseInt(m[1], 10);
  if (/bandwidth/i.test(errMsg)) return 3600; // default 1hr if can't parse
  return 0;
}

// Download all files with concurrency + bandwidth limit auto-retry
export async function downloadMegaFiles(items, outputDir, workers, logFn, progressFn, checkCancelled, statusFn) {
  mkdirSync(outputDir, { recursive: true });

  let completed = 0;
  let errors = 0;
  let skipped = 0;
  let index = 0;
  let bandwidthPaused = false;

  async function processNext() {
    while (index < items.length) {
      if (checkCancelled()) return;

      // If another worker hit the bandwidth limit, wait for it to clear
      if (bandwidthPaused) return;

      const current = index++;
      const item = items[current];

      // All files go to root output dir (flatten subfolders)
      const filepath = join(outputDir, item.name);

      // Skip if file exists AND is non-empty (0-byte = failed partial download)
      if (existsSync(filepath)) {
        try {
          const st = statSync(filepath);
          if (st.size > 0) {
            skipped++;
            progressFn(completed + skipped, errors, items.length);
            continue;
          }
          // Remove 0-byte leftover from previous failed download
          unlinkSync(filepath);
        } catch {}
      }

      try {
        const stream = item.file.download();
        await pipeline(stream, createWriteStream(filepath));

        // Verify the file was actually written
        let actualSize = item.size;
        try { actualSize = statSync(filepath).size; } catch {}

        const folderName = basename(outputDir);
        const ext = extname(item.name).toLowerCase();
        const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
        try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}

        completed++;
        const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
        logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
        progressFn(completed + skipped, errors, items.length);
      } catch (err) {
        // Clean up partial/empty file on any error
        try { unlinkSync(filepath); } catch {}

        const waitSecs = parseBandwidthWait(err.message);
        if (waitSecs > 0) {
          // Bandwidth limit — put this item back and pause all workers
          index = current; // rewind so this file gets retried
          bandwidthPaused = true;
          const waitMins = Math.ceil(waitSecs / 60);
          const resumeAt = Date.now() + waitSecs * 1000;
          logFn(`Bandwidth limit reached — waiting ${waitMins} minutes for quota reset...`);
          if (statusFn) statusFn({ paused: true, resumeAt });
          await new Promise(r => setTimeout(r, waitSecs * 1000));
          if (checkCancelled()) return;
          if (statusFn) statusFn({ paused: false, resumeAt: null });
          logFn('Quota reset — resuming downloads...');
          bandwidthPaused = false;
          continue;
        }

        logFn(`FAILED: ${item.name} — ${err.message}`);
        errors++;
        progressFn(completed + skipped, errors, items.length);
      }
    }
  }

  const workerPromises = [];
  for (let i = 0; i < Math.min(workers, items.length); i++) {
    workerPromises.push(processNext());
  }
  await Promise.all(workerPromises);

  // If we paused for bandwidth and there are remaining files, run single-threaded to finish
  while (index < items.length && !checkCancelled()) {
    const current = index++;
    const item = items[current];
    const filepath = join(outputDir, item.name);

    if (existsSync(filepath)) {
      try {
        const st = statSync(filepath);
        if (st.size > 0) {
          skipped++;
          progressFn(completed + skipped, errors, items.length);
          continue;
        }
        unlinkSync(filepath);
      } catch {}
    }

    try {
      const stream = item.file.download();
      await pipeline(stream, createWriteStream(filepath));

      let actualSize = item.size;
      try { actualSize = statSync(filepath).size; } catch {}

      const folderName = basename(outputDir);
      const ext = extname(item.name).toLowerCase();
      const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
      try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}

      completed++;
      const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
      logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
      progressFn(completed + skipped, errors, items.length);
    } catch (err) {
      try { unlinkSync(filepath); } catch {}

      const waitSecs = parseBandwidthWait(err.message);
      if (waitSecs > 0) {
        index = current;
        const waitMins = Math.ceil(waitSecs / 60);
        const resumeAt = Date.now() + waitSecs * 1000;
        logFn(`Bandwidth limit reached — waiting ${waitMins} minutes...`);
        if (statusFn) statusFn({ paused: true, resumeAt });
        await new Promise(r => setTimeout(r, waitSecs * 1000));
        if (checkCancelled()) break;
        if (statusFn) statusFn({ paused: false, resumeAt: null });
        logFn('Quota reset — resuming...');
        continue;
      }
      logFn(`FAILED: ${item.name} — ${err.message}`);
      errors++;
      progressFn(completed + skipped, errors, items.length);
    }
  }

  return { completed, errors, skipped, total: items.length };
}