import { File } from 'megajs'; import { existsSync, mkdirSync, statSync, unlinkSync } from 'fs'; import { createWriteStream } from 'fs'; import { basename, join, extname } from 'path'; import { pipeline } from 'stream/promises'; import { upsertMediaFile } from '../db.js'; const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']); const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']); export function parseMegaUrl(url) { // Validate it's a mega.nz folder URL const parsed = new URL(url); if (!parsed.hostname.includes('mega.nz') && !parsed.hostname.includes('mega.co.nz')) { throw new Error('Not a mega.nz URL'); } if (!parsed.pathname.includes('/folder/')) { throw new Error('Expected a mega.nz folder URL (e.g. https://mega.nz/folder/ABC#key)'); } return url; } // Load shared folder and list all files recursively export async function listAllFiles(url, logFn) { logFn('Loading shared folder...'); const folder = File.fromURL(url); await folder.loadAttributes(); const folderName = folder.name || 'mega_folder'; logFn(`Folder: ${folderName}`); // Recursively get all non-directory files const allFiles = folder.filter(f => !f.directory, true); logFn(`Found ${allFiles.length} files across all subfolders`); // Build items with subfolder paths const items = []; for (const file of allFiles) { const ext = extname(file.name).toLowerCase(); let type = 'other'; if (IMAGE_EXTS.has(ext)) type = 'image'; else if (VIDEO_EXTS.has(ext)) type = 'video'; // Build relative path from parent folders let subfolder = ''; let parent = file.parent; const parts = []; while (parent && parent !== folder) { parts.unshift(parent.name); parent = parent.parent; } subfolder = parts.join('/'); items.push({ file, name: file.name, size: file.size, type, subfolder, }); } return { folderName, items }; } // Parse bandwidth limit wait time from error message function parseBandwidthWait(errMsg) { const m = errMsg.match(/(\d+)\s*seconds?\s*until/i); if (m) return parseInt(m[1], 10); if (/bandwidth/i.test(errMsg)) return 3600; // default 1hr if can't parse return 0; } // Download all files with concurrency + bandwidth limit auto-retry export async function downloadMegaFiles(items, outputDir, workers, logFn, progressFn, checkCancelled, statusFn) { mkdirSync(outputDir, { recursive: true }); let completed = 0; let errors = 0; let skipped = 0; let index = 0; let bandwidthPaused = false; async function processNext() { while (index < items.length) { if (checkCancelled()) return; // If another worker hit the bandwidth limit, wait for it to clear if (bandwidthPaused) return; const current = index++; const item = items[current]; // All files go to root output dir (flatten subfolders) const filepath = join(outputDir, item.name); // Skip if file exists AND is non-empty (0-byte = failed partial download) if (existsSync(filepath)) { try { const st = statSync(filepath); if (st.size > 0) { skipped++; progressFn(completed + skipped, errors, items.length); continue; } // Remove 0-byte leftover from previous failed download unlinkSync(filepath); } catch {} } try { const stream = item.file.download(); await pipeline(stream, createWriteStream(filepath)); // Verify the file was actually written let actualSize = item.size; try { actualSize = statSync(filepath).size; } catch {} const folderName = basename(outputDir); const ext = extname(item.name).toLowerCase(); const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other'; try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {} completed++; const sizeMb = (item.size / (1024 * 1024)).toFixed(1); logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`); progressFn(completed + skipped, errors, items.length); } catch (err) { // Clean up partial/empty file on any error try { unlinkSync(filepath); } catch {} const waitSecs = parseBandwidthWait(err.message); if (waitSecs > 0) { // Bandwidth limit — put this item back and pause all workers index = current; // rewind so this file gets retried bandwidthPaused = true; const waitMins = Math.ceil(waitSecs / 60); const resumeAt = Date.now() + waitSecs * 1000; logFn(`Bandwidth limit reached — waiting ${waitMins} minutes for quota reset...`); if (statusFn) statusFn({ paused: true, resumeAt }); await new Promise(r => setTimeout(r, waitSecs * 1000)); if (checkCancelled()) return; if (statusFn) statusFn({ paused: false, resumeAt: null }); logFn('Quota reset — resuming downloads...'); bandwidthPaused = false; continue; } logFn(`FAILED: ${item.name} — ${err.message}`); errors++; progressFn(completed + skipped, errors, items.length); } } } const workerPromises = []; for (let i = 0; i < Math.min(workers, items.length); i++) { workerPromises.push(processNext()); } await Promise.all(workerPromises); // If we paused for bandwidth and there are remaining files, run single-threaded to finish while (index < items.length && !checkCancelled()) { const current = index++; const item = items[current]; const filepath = join(outputDir, item.name); if (existsSync(filepath)) { try { const st = statSync(filepath); if (st.size > 0) { skipped++; progressFn(completed + skipped, errors, items.length); continue; } unlinkSync(filepath); } catch {} } try { const stream = item.file.download(); await pipeline(stream, createWriteStream(filepath)); let actualSize = item.size; try { actualSize = statSync(filepath).size; } catch {} const folderName = basename(outputDir); const ext = extname(item.name).toLowerCase(); const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other'; try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {} completed++; const sizeMb = (item.size / (1024 * 1024)).toFixed(1); logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`); progressFn(completed + skipped, errors, items.length); } catch (err) { try { unlinkSync(filepath); } catch {} const waitSecs = parseBandwidthWait(err.message); if (waitSecs > 0) { index = current; const waitMins = Math.ceil(waitSecs / 60); const resumeAt = Date.now() + waitSecs * 1000; logFn(`Bandwidth limit reached — waiting ${waitMins} minutes...`); if (statusFn) statusFn({ paused: true, resumeAt }); await new Promise(r => setTimeout(r, waitSecs * 1000)); if (checkCancelled()) break; if (statusFn) statusFn({ paused: false, resumeAt: null }); logFn('Quota reset — resuming...'); continue; } logFn(`FAILED: ${item.name} — ${err.message}`); errors++; progressFn(completed + skipped, errors, items.length); } } return { completed, errors, skipped, total: items.length }; }