Add DRM downloads, scrapers, gallery index, and UI improvements

- DRM video download pipeline with pywidevine subprocess for Widevine key acquisition
- Scraper system: forum threads, Coomer/Kemono API, and MediaLink (Fapello) scrapers
- SQLite-backed media index for instant gallery loads with startup scan
- Duplicate detection and gallery filtering/sorting
- HLS video component, log viewer, and scrape management UI
- Dockerfile updated for Python/pywidevine, docker-compose volume for CDM

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Trey t
2026-02-16 11:29:11 -06:00
parent c60de19348
commit 1e5f54f60b
28 changed files with 4736 additions and 203 deletions

View File

@@ -1,10 +1,11 @@
import { useState, useEffect, useRef } from 'react'
import { getDownloadHistory, getActiveDownloads, getUser } from '../api'
import { getDownloadHistory, getActiveDownloads, getUser, getScrapeJobs } from '../api'
import Spinner from '../components/Spinner'
export default function Downloads() {
const [history, setHistory] = useState([])
const [active, setActive] = useState([])
const [scrapeJobs, setScrapeJobs] = useState([])
const [loading, setLoading] = useState(true)
const [error, setError] = useState(null)
const pollRef = useRef(null)
@@ -23,9 +24,10 @@ export default function Downloads() {
setLoading(true)
setError(null)
const [histData, activeData] = await Promise.all([
const [histData, activeData, scrapeData] = await Promise.all([
getDownloadHistory(),
getActiveDownloads(),
getScrapeJobs(),
])
if (histData.error) {
@@ -37,25 +39,31 @@ export default function Downloads() {
const histList = Array.isArray(histData) ? histData : histData.list || []
setHistory(histList)
setActive(Array.isArray(activeData) ? activeData : [])
setScrapeJobs(Array.isArray(scrapeData) ? scrapeData.filter(j => j.running) : [])
setLoading(false)
resolveUsernames(histList)
}
const startPolling = () => {
pollRef.current = setInterval(async () => {
const activeData = await getActiveDownloads()
if (activeData.error) return
const list = Array.isArray(activeData) ? activeData : []
setActive((prev) => {
// If something just finished, refresh history
if (prev.length > 0 && list.length < prev.length) {
getDownloadHistory().then((h) => {
if (!h.error) setHistory(Array.isArray(h) ? h : h.list || [])
})
}
return list
})
const [activeData, scrapeData] = await Promise.all([
getActiveDownloads(),
getScrapeJobs(),
])
if (!activeData.error) {
const list = Array.isArray(activeData) ? activeData : []
setActive((prev) => {
if (prev.length > 0 && list.length < prev.length) {
getDownloadHistory().then((h) => {
if (!h.error) setHistory(Array.isArray(h) ? h : h.list || [])
})
}
return list
})
}
if (!scrapeData.error) {
setScrapeJobs(Array.isArray(scrapeData) ? scrapeData.filter(j => j.running) : [])
}
}, 2000)
}
@@ -159,6 +167,50 @@ export default function Downloads() {
</div>
)}
{/* Scrape Jobs */}
{scrapeJobs.length > 0 && (
<div className="mb-8">
<h2 className="text-sm font-semibold text-gray-300 uppercase tracking-wider mb-3">
Scrape Jobs
</h2>
<div className="space-y-3">
{scrapeJobs.map((job) => {
const progress = job.progress.total > 0
? Math.round((job.progress.completed / job.progress.total) * 100)
: 0
return (
<div key={job.id} className="bg-[#161616] border border-[#222] rounded-lg p-4">
<div className="flex items-center justify-between mb-3">
<div className="flex items-center gap-2">
<span className={`inline-block px-2 py-0.5 rounded text-xs font-medium ${
job.type === 'forum' ? 'bg-purple-500/10 text-purple-400' : 'bg-orange-500/10 text-orange-400'
}`}>
{job.type}
</span>
<p className="text-sm font-medium text-white">{job.folderName}</p>
</div>
<span className="text-xs text-[#0095f6] font-medium">{progress}%</span>
</div>
<p className="text-xs text-gray-500 mb-2">
{job.progress.completed} / {job.progress.total} {job.type === 'forum' ? 'pages' : 'files'}
{job.progress.errors > 0 && (
<span className="text-red-400 ml-2">({job.progress.errors} errors)</span>
)}
</p>
<div className="w-full bg-[#1a1a1a] rounded-full h-1.5">
<div
className="bg-[#0095f6] h-1.5 rounded-full transition-all duration-500"
style={{ width: `${progress}%` }}
/>
</div>
</div>
)
})}
</div>
</div>
)}
{/* Download History */}
<div>
<h2 className="text-sm font-semibold text-gray-300 uppercase tracking-wider mb-3">