Add DRM downloads, scrapers, gallery index, and UI improvements

- DRM video download pipeline with pywidevine subprocess for Widevine key acquisition
- Scraper system: forum threads, Coomer/Kemono API, and MediaLink (Fapello) scrapers
- SQLite-backed media index for instant gallery loads with startup scan
- Duplicate detection and gallery filtering/sorting
- HLS video component, log viewer, and scrape management UI
- Dockerfile updated for Python/pywidevine, docker-compose volume for CDM

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

server/download.js

@@ -1,10 +1,11 @@
 import { Router } from 'express';
 import fetch from 'node-fetch';
-import { mkdirSync, createWriteStream } from 'fs';
+import { mkdirSync, createWriteStream, statSync } from 'fs';
 import { pipeline } from 'stream/promises';
 import { extname } from 'path';
-import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor } from './db.js';
+import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile } from './db.js';
 import { createSignedHeaders, getRules } from './signing.js';
+import { downloadDrmMedia, hasCDM } from './drm-download.js';
 
 const router = Router();
 const OF_BASE = 'https://onlyfans.com';
@@ -111,21 +112,32 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
       }
 
       const data = await fetchOF(ofPath, authConfig);
-      const mediaList = Array.isArray(data) ? data : (data.list || []);
-      postsFetched += mediaList.length;
+      const rawList = Array.isArray(data) ? data : (data.list || []);
+      postsFetched += rawList.length;
 
-      for (const media of mediaList) {
-        const postDate = media.postedAt || media.createdAt || media.publishedAt || null;
-        const postId = media.postId || media.post_id || media.id;
-        allMedia.push({ postId, media, postDate });
+      // The /posts/medias endpoint returns post objects with nested media[].
+      // Flatten into individual media items.
+      for (const item of rawList) {
+        const postDate = item.postedAt || item.createdAt || item.publishedAt || null;
+        const postId = item.id;
+
+        if (Array.isArray(item.media) && item.media.length > 0) {
+          for (const m of item.media) {
+            allMedia.push({ postId, media: m, postDate });
+          }
+        } else {
+          // Fallback: treat the item itself as a media object
+          const pid = item.postId || item.post_id || item.id;
+          allMedia.push({ postId: pid, media: item, postDate });
+        }
       }
 
       hasMore = Array.isArray(data) ? data.length === batchSize : !!data.hasMore;
       if (!Array.isArray(data)) {
         beforePublishTime = data.tailMarker || null;
-      } else if (mediaList.length > 0) {
+      } else if (rawList.length > 0) {
         // For flat array responses, use the last item's date as cursor
-        const last = mediaList[mediaList.length - 1];
+        const last = rawList[rawList.length - 1];
         beforePublishTime = last.postedAt || last.createdAt || null;
       }
 
@@ -160,8 +172,50 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
           continue;
         }
 
+        // Check for DRM-protected video
+        const drm = media.files?.drm;
+        if (drm?.manifest?.dash && drm?.signature?.dash) {
+          if (!hasCDM()) {
+            console.log(`[download] Skipping DRM media ${mediaId} (no WVD file configured)`);
+            progress.completed++;
+            continue;
+          }
+          try {
+            const sig = drm.signature.dash;
+            const cfCookies = {
+              cp: sig['CloudFront-Policy'],
+              cs: sig['CloudFront-Signature'],
+              ck: sig['CloudFront-Key-Pair-Id'],
+            };
+            const drmFilename = `${postId}_${mediaId}_video.mp4`;
+            const userDir = `${MEDIA_PATH}/${username || userId}`;
+            await downloadDrmMedia({
+              mpdUrl: drm.manifest.dash,
+              cfCookies,
+              mediaId,
+              entityType: 'post',
+              entityId: String(postId),
+              outputDir: userDir,
+              outputFilename: drmFilename,
+            });
+            recordDownload(userId, String(postId), mediaId, 'video', drmFilename, postDate);
+            try {
+              const st = statSync(`${userDir}/${drmFilename}`);
+              upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate);
+            } catch { /* stat may fail if file was cleaned up */ }
+            progress.completed++;
+          } catch (err) {
+            console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
+            progress.errors++;
+            progress.completed++;
+          }
+          await sleep(DOWNLOAD_DELAY);
+          continue;
+        }
+
         const url = getMediaUrl(media);
         if (!url) {
+          console.log(`[download] Skipping media ${mediaId} (no URL)`);
           progress.completed++;
           continue;
         }
@@ -175,6 +229,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
 
         await downloadFile(url, dest);
         recordDownload(userId, String(postId), mediaId, mediaType, filename, postDate);
+        try {
+          const st = statSync(dest);
+          const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
+          if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
+        } catch { /* ignore */ }
         progress.completed++;
       } catch (err) {
         console.error(`[download] Error downloading media ${media.id}:`, err.message);