OFApp/server/scrapers/turbo.js

import { writeFileSync, existsSync } from 'fs';
import { join, basename } from 'path';
import { fsCreateSession, fsDestroySession, fsGet } from '../flaresolverr.js';
import { upsertMediaFile } from '../db.js';

const TURBO_HOST_RE = /^https?:\/\/(?:www\.)?turbo\.\w+\//i;
const TURBO_BASE = 'https://turbo.cr';
const DEFAULT_UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

export function isTurboUrl(url) {
  return TURBO_HOST_RE.test(url);
}

function unescapeHtml(s) {
  return s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
}

function extractMp4FromHtml(html) {
  // Plyr renders the resolved URL into <video src="..."> after WASM runs
  const m = html.match(/<video[^>]+\bsrc=["']([^"']+\.mp4[^"']*)["']/i);
  if (m) return unescapeHtml(m[1]);
  // Fallback: any direct turbocdn mp4
  const m2 = html.match(/https?:\/\/[^"'\s<>]*turbocdn[^"'\s<>]*\.mp4[^"'\s<>]*/i);
  if (m2) return unescapeHtml(m2[0]);
  return null;
}

function turboFilename(mp4Url, fallbackId) {
  try {
    const u = new URL(mp4Url);
    const fn = u.searchParams.get('fn');
    if (fn) return fn;
    const base = basename(u.pathname);
    if (base) return base;
  } catch {}
  return (fallbackId || 'turbo') + '.mp4';
}

async function downloadVideo(url, dest, ua) {
  try {
    const r = await fetch(url, {
      headers: { 'User-Agent': ua, 'Referer': TURBO_BASE + '/' },
      signal: AbortSignal.timeout(600000), // 10 min for big videos
    });
    if (!r.ok) return { ok: false, status: r.status };
    const buf = Buffer.from(await r.arrayBuffer());
    if (buf.length < 10000) return { ok: false, reason: 'too small ' + buf.length };
    writeFileSync(dest, buf);
    return { ok: true, size: buf.length };
  } catch (e) {
    return { ok: false, error: e.message };
  }
}

async function resolveEmbed(sessionId, embedUrl) {
  const r = await fsGet(sessionId, embedUrl, '');
  if (r.status !== 200) return null;
  return extractMp4FromHtml(r.html);
}

async function resolveAlbumIds(sessionId, albumUrl) {
  const r = await fsGet(sessionId, albumUrl, '');
  if (r.status !== 200) return [];
  // Album page: each video tile has data-id="<videoId>"
  const ids = new Set();
  const re = /data-id=["']([A-Za-z0-9_-]{6,})["']/g;
  let m;
  while ((m = re.exec(r.html)) !== null) ids.add(m[1]);
  // Also handle direct embed links if present
  const re2 = /turbo\.[a-z]+\/embed\/([A-Za-z0-9_-]+)/g;
  while ((m = re2.exec(r.html)) !== null) ids.add(m[1]);
  return [...ids];
}

/**
 * Resolve a turbo.cr URL (embed or album) and download all videos found.
 * Pass an existing FlareSolverr sessionId to reuse it across many calls;
 * otherwise one is created and destroyed per call.
 *
 * Returns the count of videos successfully downloaded.
 */
export async function downloadTurbo(url, outputDir, logFn, userAgent, fsSession) {
  const ua = userAgent || DEFAULT_UA;
  const folderName = basename(outputDir);
  let ownSession = false;
  let sessionId = fsSession;

  try {
    if (!sessionId) {
      sessionId = await fsCreateSession(TURBO_BASE + '/');
      ownSession = true;
    }

    let embedIds = [];
    let mEmbed = url.match(/turbo\.[a-z]+\/embed\/([A-Za-z0-9_-]+)/i);
    if (mEmbed) {
      embedIds = [mEmbed[1]];
    } else if (/\/a\//i.test(url)) {
      logFn(`turbo: resolving album ${url}`);
      embedIds = await resolveAlbumIds(sessionId, url);
      logFn(`turbo: album has ${embedIds.length} video(s)`);
    } else {
      logFn(`turbo: unrecognized URL ${url}`);
      return 0;
    }

    let count = 0;
    for (const id of embedIds) {
      const embedUrl = `${TURBO_BASE}/embed/${id}`;
      try {
        const mp4 = await resolveEmbed(sessionId, embedUrl);
        if (!mp4) {
          logFn(`turbo: could not resolve mp4 for ${id}`);
          continue;
        }
        const filename = turboFilename(mp4, id);
        const dest = join(outputDir, filename);
        if (existsSync(dest)) {
          logFn(`turbo: already have ${filename}`);
          continue;
        }
        const dl = await downloadVideo(mp4, dest, ua);
        if (dl.ok) {
          try { upsertMediaFile(folderName, filename, 'video', dl.size, Date.now(), null); } catch {}
          logFn(`Downloaded: ${filename} (${(dl.size / (1024 * 1024)).toFixed(1)} MB) [video]`);
          count++;
        } else {
          logFn(`turbo: download failed ${filename} - ${dl.status || dl.error || dl.reason}`);
        }
      } catch (e) {
        logFn(`turbo: error for ${id}: ${e.message}`);
      }
    }
    return count;
  } finally {
    if (ownSession && sessionId) await fsDestroySession(sessionId);
  }
}