diff --git a/client/src/pages/VideoDetail.jsx b/client/src/pages/VideoDetail.jsx
new file mode 100644
index 0000000..d485d9d
--- /dev/null
+++ b/client/src/pages/VideoDetail.jsx
@@ -0,0 +1,287 @@
+import { useState, useEffect, useRef } from 'react'
+import { useParams, useNavigate } from 'react-router-dom'
+import { getVideo, updateVideoMeta, deleteVideo } from '../api'
+import HlsVideo from '../components/HlsVideo'
+import TagInput from '../components/TagInput'
+import Spinner from '../components/Spinner'
+
+function formatDuration(seconds) {
+ if (!seconds) return '—'
+ const h = Math.floor(seconds / 3600)
+ const m = Math.floor((seconds % 3600) / 60)
+ const s = Math.floor(seconds % 60)
+ if (h > 0) return `${h}h ${m}m ${s}s`
+ if (m > 0) return `${m}m ${s}s`
+ return `${s}s`
+}
+
+function formatBytes(bytes) {
+ if (!bytes) return '—'
+ if (bytes >= 1073741824) return `${(bytes / 1073741824).toFixed(1)} GB`
+ if (bytes >= 1048576) return `${(bytes / 1048576).toFixed(1)} MB`
+ return `${(bytes / 1024).toFixed(1)} KB`
+}
+
+function formatDate(dateStr) {
+ if (!dateStr) return '—'
+ return new Date(dateStr).toLocaleDateString('en-US', {
+ month: 'short', day: 'numeric', year: 'numeric', hour: 'numeric', minute: '2-digit',
+ })
+}
+
+export default function VideoDetail() {
+ const { id } = useParams()
+ const navigate = useNavigate()
+ const [video, setVideo] = useState(null)
+ const [loading, setLoading] = useState(true)
+ const [error, setError] = useState(null)
+
+ const [editingTitle, setEditingTitle] = useState(false)
+ const [titleDraft, setTitleDraft] = useState('')
+ const [editingDesc, setEditingDesc] = useState(false)
+ const [descDraft, setDescDraft] = useState('')
+ const [saving, setSaving] = useState(false)
+ const [confirmDelete, setConfirmDelete] = useState(false)
+ const videoRef = useRef(null)
+
+ useEffect(() => {
+ setLoading(true)
+ getVideo(id).then(data => {
+ if (data.error) {
+ setError(data.error)
+ } else {
+ setVideo(data)
+ setTitleDraft(data.title)
+ setDescDraft(data.description || '')
+ }
+ setLoading(false)
+ })
+ }, [id])
+
+ const saveTitle = async () => {
+ if (!titleDraft.trim() || titleDraft === video.title) {
+ setEditingTitle(false)
+ return
+ }
+ setSaving(true)
+ const res = await updateVideoMeta(id, { title: titleDraft.trim() })
+ if (!res.error) {
+ setVideo(res)
+ setTitleDraft(res.title)
+ }
+ setSaving(false)
+ setEditingTitle(false)
+ }
+
+ const saveDesc = async () => {
+ if (descDraft === (video.description || '')) {
+ setEditingDesc(false)
+ return
+ }
+ setSaving(true)
+ const res = await updateVideoMeta(id, { description: descDraft })
+ if (!res.error) {
+ setVideo(res)
+ setDescDraft(res.description || '')
+ }
+ setSaving(false)
+ setEditingDesc(false)
+ }
+
+ const handleTagsChange = async (newTags) => {
+ setSaving(true)
+ const res = await updateVideoMeta(id, { tags: newTags })
+ if (!res.error) {
+ setVideo(res)
+ }
+ setSaving(false)
+ }
+
+ const handleDelete = async () => {
+ const res = await deleteVideo(id)
+ if (!res.error) {
+ navigate('/videos', { replace: true })
+ }
+ }
+
+ if (loading) return
+ )
+ if (!video) return null
+
+ const hlsSrc = `/api/video-hls/${video.id}/master.m3u8`
+ const tags = (video.tags || []).map(t => t.name)
+
+ const skipForward = () => {
+ if (videoRef.current) {
+ videoRef.current.currentTime = Math.min(videoRef.current.currentTime + 10, videoRef.current.duration || Infinity)
+ }
+ }
+
+ return (
+
+ )
+}
diff --git a/client/src/pages/VideoUpload.jsx b/client/src/pages/VideoUpload.jsx
new file mode 100644
index 0000000..dfd3c97
--- /dev/null
+++ b/client/src/pages/VideoUpload.jsx
@@ -0,0 +1,310 @@
+import { useState, useRef, useCallback } from 'react'
+import { useNavigate } from 'react-router-dom'
+import { uploadVideo, scanVideos, getVideoScanStatus } from '../api'
+
+export default function VideoUpload() {
+ const navigate = useNavigate()
+ const fileInputRef = useRef(null)
+ const [uploads, setUploads] = useState([]) // { id, file, progress, status, error, video }
+ const [dragging, setDragging] = useState(false)
+ const [scanning, setScanning] = useState(false)
+ const [scanStatus, setScanStatus] = useState(null)
+ const pollRef = useRef(null)
+
+ const processFiles = useCallback((files) => {
+ const videoFiles = Array.from(files).filter(f => {
+ const ext = f.name.split('.').pop().toLowerCase()
+ return ['mp4', 'mov', 'avi', 'webm', 'mkv', 'm4v', 'wmv', 'flv', 'ts'].includes(ext)
+ })
+
+ if (videoFiles.length === 0) return
+
+ const newUploads = videoFiles.map((file, i) => ({
+ id: `${Date.now()}-${i}`,
+ file,
+ progress: 0,
+ status: 'pending',
+ error: null,
+ video: null,
+ }))
+
+ setUploads(prev => [...prev, ...newUploads])
+
+ // Start uploads sequentially
+ ;(async () => {
+ for (const item of newUploads) {
+ setUploads(prev => prev.map(u =>
+ u.id === item.id ? { ...u, status: 'uploading' } : u
+ ))
+
+ try {
+ const result = await uploadVideo(item.file, (progress) => {
+ setUploads(prev => prev.map(u =>
+ u.id === item.id ? { ...u, progress } : u
+ ))
+ })
+
+ if (result.error) {
+ setUploads(prev => prev.map(u =>
+ u.id === item.id ? { ...u, status: 'error', error: result.error } : u
+ ))
+ } else {
+ setUploads(prev => prev.map(u =>
+ u.id === item.id ? { ...u, status: 'done', progress: 1, video: result.video } : u
+ ))
+ }
+ } catch (err) {
+ setUploads(prev => prev.map(u =>
+ u.id === item.id ? { ...u, status: 'error', error: err.message } : u
+ ))
+ }
+ }
+ })()
+ }, [])
+
+ const handleDrop = (e) => {
+ e.preventDefault()
+ setDragging(false)
+ processFiles(e.dataTransfer.files)
+ }
+
+ const handleFileSelect = (e) => {
+ processFiles(e.target.files)
+ e.target.value = ''
+ }
+
+ const handleScan = async () => {
+ setScanning(true)
+ setScanStatus(null)
+ const res = await scanVideos()
+ if (res.error) {
+ setScanStatus({ error: res.error })
+ setScanning(false)
+ return
+ }
+
+ // Poll for status
+ pollRef.current = setInterval(async () => {
+ const status = await getVideoScanStatus()
+ setScanStatus(status)
+ if (!status.running) {
+ clearInterval(pollRef.current)
+ setScanning(false)
+ }
+ }, 2000)
+ }
+
+ const doneCount = uploads.filter(u => u.status === 'done').length
+ const errorCount = uploads.filter(u => u.status === 'error').length
+
+ return (
+
+ {/* Back button */}
+
+
+
Add Videos
+
+
+ {/* Upload Zone */}
+
+
Upload Files
+
{ e.preventDefault(); setDragging(true) }}
+ onDragLeave={() => setDragging(false)}
+ onDrop={handleDrop}
+ onClick={() => fileInputRef.current?.click()}
+ className={`border-2 border-dashed rounded-xl p-12 text-center cursor-pointer transition-all ${
+ dragging
+ ? 'border-[#0095f6] bg-[#0095f6]/5'
+ : 'border-[#333] hover:border-[#555] bg-[#111]'
+ }`}
+ >
+
+
+ Drag & drop video files here
+
+
+ or click to browse — MP4, MKV, MOV, AVI, WebM
+
+
+
+
+
+ {/* Folder Scan */}
+
+
Scan Folder
+
+
+ Scan the server's video directory for new files.
+ Videos already indexed will be skipped.
+
+
+
+ {scanStatus && (
+
+ {scanStatus.error ? (
+
{scanStatus.error}
+ ) : (
+ <>
+ {scanStatus.running && (
+
+
+ Progress
+ {scanStatus.done} / {scanStatus.total}
+
+
+
0 ? (scanStatus.done / scanStatus.total * 100) : 0}%` }}
+ />
+
+
+ )}
+ {!scanStatus.running && scanStatus.total > 0 && (
+
+
{scanStatus.added} new video{scanStatus.added !== 1 ? 's' : ''} added
+ {scanStatus.skipped > 0 && (
+
{scanStatus.skipped} already indexed
+ )}
+ {scanStatus.errors > 0 && (
+
{scanStatus.errors} error{scanStatus.errors !== 1 ? 's' : ''}
+ )}
+
+ )}
+ >
+ )}
+
+ )}
+
+
+
+
+ {/* Upload List */}
+ {uploads.length > 0 && (
+
+
+
+ Uploads
+ {doneCount > 0 && {doneCount} done}
+ {errorCount > 0 && {errorCount} failed}
+
+ {uploads.every(u => u.status === 'done' || u.status === 'error') && (
+
+ )}
+
+
+
+ {uploads.map(item => (
+
+ {/* Status icon */}
+
+ {item.status === 'done' ? (
+
+ ) : item.status === 'error' ? (
+
+ ) : (
+
+ )}
+
+
+ {/* File info */}
+
+
{item.file.name}
+ {item.status === 'uploading' && (
+
+ )}
+ {item.error && (
+
{item.error}
+ )}
+
+
+ {/* Progress / action */}
+
+ {item.status === 'uploading' && `${Math.round(item.progress * 100)}%`}
+ {item.status === 'done' && item.video && (
+
+ )}
+
+
+ ))}
+
+
+ )}
+
+ )
+}
+
+function UploadIcon({ className }) {
+ return (
+
+ )
+}
+
+function ScanIcon({ className }) {
+ return (
+
+ )
+}
diff --git a/client/src/pages/Videos.jsx b/client/src/pages/Videos.jsx
new file mode 100644
index 0000000..55c517d
--- /dev/null
+++ b/client/src/pages/Videos.jsx
@@ -0,0 +1,473 @@
+import { useState, useEffect, useCallback, useRef } from 'react'
+import { useNavigate } from 'react-router-dom'
+import { getVideos, getVideoTags } from '../api'
+import VideoCard from '../components/VideoCard'
+import LoadMoreButton from '../components/LoadMoreButton'
+import Spinner from '../components/Spinner'
+import GridWall, { GridWallPicker } from '../components/GridWall'
+
+const PAGE_SIZE = 48
+
+const SORT_OPTIONS = [
+ { value: 'latest', label: 'Latest' },
+ { value: 'oldest', label: 'Oldest' },
+ { value: 'longest', label: 'Longest' },
+ { value: 'shortest', label: 'Shortest' },
+ { value: 'largest', label: 'Largest' },
+ { value: 'title', label: 'Title' },
+]
+
+const DURATION_OPTIONS = [
+ { value: '', label: 'Any Length' },
+ { value: '0-300', label: 'Under 5 min' },
+ { value: '300-1200', label: '5–20 min' },
+ { value: '1200-3600', label: '20–60 min' },
+ { value: '3600-', label: 'Over 1 hour' },
+]
+
+const RESOLUTION_OPTIONS = [
+ { value: '', label: 'All' },
+ { value: '480', label: '480p+' },
+ { value: '720', label: '720p+' },
+ { value: '1080', label: '1080p+' },
+]
+
+export default function Videos() {
+ const navigate = useNavigate()
+ const [videos, setVideos] = useState([])
+ const [total, setTotal] = useState(0)
+ const [loading, setLoading] = useState(true)
+ const [loadingMore, setLoadingMore] = useState(false)
+ const [error, setError] = useState(null)
+
+ const [search, setSearch] = useState('')
+ const [sortOption, setSortOption] = useState('latest')
+ const [duration, setDuration] = useState('')
+ const [resolution, setResolution] = useState('')
+ const [selectedTags, setSelectedTags] = useState([])
+ const [tagFilterOpen, setTagFilterOpen] = useState(false)
+ const [tagSearch, setTagSearch] = useState('')
+ const [allTags, setAllTags] = useState([])
+ const [gridWallLayout, setGridWallLayout] = useState(null)
+ const [gridPickerOpen, setGridPickerOpen] = useState(false)
+ const gridPickerRef = useRef(null)
+ const tagRef = useRef(null)
+
+ // Load tags for filter
+ useEffect(() => {
+ getVideoTags().then(data => {
+ if (Array.isArray(data)) setAllTags(data)
+ })
+ }, [])
+
+ // Close tag filter on click outside
+ useEffect(() => {
+ if (!tagFilterOpen) return
+ const handleClick = (e) => {
+ if (tagRef.current && !tagRef.current.contains(e.target)) {
+ setTagFilterOpen(false)
+ setTagSearch('')
+ }
+ }
+ document.addEventListener('mousedown', handleClick)
+ return () => document.removeEventListener('mousedown', handleClick)
+ }, [tagFilterOpen])
+
+ const loadVideos = useCallback(async (reset = true) => {
+ if (reset) {
+ setLoading(true)
+ setError(null)
+ } else {
+ setLoadingMore(true)
+ }
+
+ const offset = reset ? 0 : videos.length
+ const [minDuration, maxDuration] = duration ? duration.split('-') : ['', '']
+
+ const data = await getVideos({
+ search: search || undefined,
+ tags: selectedTags.length > 0 ? selectedTags : undefined,
+ minDuration: minDuration || undefined,
+ maxDuration: maxDuration || undefined,
+ minWidth: resolution || undefined,
+ sort: sortOption,
+ offset,
+ limit: PAGE_SIZE,
+ })
+
+ if (data.error) {
+ setError(data.error)
+ } else {
+ setVideos(prev => reset ? data.videos : [...prev, ...data.videos])
+ setTotal(data.total)
+ }
+
+ setLoading(false)
+ setLoadingMore(false)
+ }, [search, sortOption, duration, resolution, selectedTags, videos.length])
+
+ useEffect(() => {
+ loadVideos(true)
+ }, [search, sortOption, duration, resolution, selectedTags])
+
+ const toggleTag = (name) => {
+ setSelectedTags(prev =>
+ prev.includes(name) ? prev.filter(t => t !== name) : [...prev, name]
+ )
+ }
+
+ // Grid wall: fetch shuffled videos from current filter
+ const fetchGridItems = useCallback(async (limit) => {
+ const [minDuration, maxDuration] = duration ? duration.split('-') : ['', '']
+ const data = await getVideos({
+ search: search || undefined,
+ tags: selectedTags.length > 0 ? selectedTags : undefined,
+ minDuration: minDuration || undefined,
+ maxDuration: maxDuration || undefined,
+ minWidth: resolution || undefined,
+ sort: 'shuffle',
+ limit,
+ })
+ if (data.error) return []
+ // Normalize video items to have type='video' for GridCell
+ return data.videos.map(v => ({ ...v, type: 'video' }))
+ }, [search, selectedTags, duration, resolution])
+
+ const hasMore = videos.length < total
+
+ return (
+
+ {/* Header */}
+
+
+
Videos
+
+
+ {total} video{total !== 1 ? 's' : ''}
+
+
+
+
+
+
+ {/* Filters */}
+
+ {/* Tag Filter */}
+
+
+
+ {tagFilterOpen && (
+
+
+ setTagSearch(e.target.value)}
+ placeholder="Search tags..."
+ autoFocus
+ className="w-full px-3 py-1.5 bg-[#111] border border-[#333] rounded-md text-sm text-white placeholder-gray-600 focus:outline-none focus:border-[#0095f6]"
+ />
+
+
+ {allTags
+ .filter(t => t.name.toLowerCase().includes(tagSearch.toLowerCase()))
+ .map(t => (
+
+ ))}
+ {allTags.length === 0 && (
+
No tags yet
+ )}
+
+ {selectedTags.length > 0 && (
+
+
+
+ )}
+
+ )}
+
+
+ {/* Selected tag pills */}
+ {selectedTags.length > 0 && selectedTags.length <= 5 && (
+ selectedTags.map(name => (
+
+ {name}
+
+
+ ))
+ )}
+
+ {/* Duration Dropdown */}
+
+
+ {/* Resolution Segmented */}
+
+ {RESOLUTION_OPTIONS.map(opt => (
+
+ ))}
+
+
+ {/* Sort Dropdown */}
+
+
+ {/* Grid Wall Button */}
+
+
+ {gridPickerOpen && (
+ { setGridWallLayout(layout); setGridPickerOpen(false) }}
+ onClose={() => setGridPickerOpen(false)}
+ />
+ )}
+
+
+ {/* Search */}
+
+ setSearch(e.target.value)}
+ placeholder="Search videos..."
+ className="w-full px-3 py-2 text-sm rounded-lg border border-[#333] bg-[#161616] text-white placeholder-gray-600 focus:outline-none focus:border-[#0095f6] transition-colors"
+ />
+
+
+
+ {/* Content */}
+ {loading ? (
+
+ ) : error ? (
+
+ ) : videos.length === 0 ? (
+
+
+
No videos found
+
+ Upload videos or scan a folder to get started
+
+
+
+ ) : (
+ <>
+
+ {videos.map(v => (
+
+ ))}
+
+
+
+ loadVideos(false)}
+ loading={loadingMore}
+ hasMore={hasMore}
+ />
+
+ >
+ )}
+
+ {/* Grid Wall */}
+ {gridWallLayout && (
+
setGridWallLayout(null)}
+ />
+ )}
+
+ )
+}
+
+function GridWallIcon({ className }) {
+ return (
+
+ )
+}
+
+function SortDropdown({ value, onChange }) {
+ const [open, setOpen] = useState(false)
+ const ref = useRef(null)
+ const current = SORT_OPTIONS.find(o => o.value === value)
+
+ useEffect(() => {
+ if (!open) return
+ const handleClick = (e) => {
+ if (ref.current && !ref.current.contains(e.target)) setOpen(false)
+ }
+ document.addEventListener('mousedown', handleClick)
+ return () => document.removeEventListener('mousedown', handleClick)
+ }, [open])
+
+ return (
+
+
+ {open && (
+
+ {SORT_OPTIONS.map(opt => (
+
+ ))}
+
+ )}
+
+ )
+}
+
+function DurationDropdown({ value, onChange }) {
+ const [open, setOpen] = useState(false)
+ const ref = useRef(null)
+ const current = DURATION_OPTIONS.find(o => o.value === value)
+
+ useEffect(() => {
+ if (!open) return
+ const handleClick = (e) => {
+ if (ref.current && !ref.current.contains(e.target)) setOpen(false)
+ }
+ document.addEventListener('mousedown', handleClick)
+ return () => document.removeEventListener('mousedown', handleClick)
+ }, [open])
+
+ return (
+
+
+ {open && (
+
+ {DURATION_OPTIONS.map(opt => (
+
+ ))}
+
+ )}
+
+ )
+}
+
+function TagIcon({ className }) {
+ return (
+
+ )
+}
+
+function VideoIcon({ className }) {
+ return (
+
+ )
+}
diff --git a/docker-compose.yml b/docker-compose.yml
index 3eb45fc..ea2ae00 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,6 +4,8 @@ services:
build: .
container_name: ofapp
restart: unless-stopped
+ depends_on:
+ - flaresolverr
ports:
- "3002:3001"
- "3003:3443"
@@ -11,9 +13,26 @@ services:
- /mnt/user/downloads/OFApp/db:/data/db
- /mnt/user/downloads/OFApp/media:/data/media
- /mnt/user/downloads/OFApp/cdm:/data/cdm
+ - /mnt/user/downloads/OFApp/videos:/data/videos
+ devices:
+ - /dev/dri:/dev/dri
environment:
- PORT=3001
- DB_PATH=/data/db/ofapp.db
- MEDIA_PATH=/data/media
+ - VIDEOS_PATH=/data/videos
- DOWNLOAD_DELAY=1000
- HLS_ENABLED=false
+ - LIBVA_DRIVER_NAME=iHD
+ - TZ=America/Chicago
+ - FLARESOLVERR_URL=http://flaresolverr:8191
+
+ flaresolverr:
+ image: ghcr.io/flaresolverr/flaresolverr:latest
+ container_name: flaresolverr
+ restart: unless-stopped
+ environment:
+ - LOG_LEVEL=info
+ - HEADLESS=true
+ ports:
+ - "8191:8191"
diff --git a/package-lock.json b/package-lock.json
index 2ee11f3..7064ea2 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -7,6 +7,9 @@
"": {
"name": "ofapp",
"version": "1.0.0",
+ "dependencies": {
+ "megajs": "^1.3.9"
+ },
"devDependencies": {
"concurrently": "^8.2.0"
}
@@ -157,6 +160,18 @@
"url": "https://opencollective.com/date-fns"
}
},
+ "node_modules/duplexify": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",
+ "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==",
+ "license": "MIT",
+ "dependencies": {
+ "end-of-stream": "^1.4.1",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.1.1",
+ "stream-shift": "^1.0.2"
+ }
+ },
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -164,6 +179,15 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/end-of-stream": {
+ "version": "1.4.5",
+ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
+ "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
+ "license": "MIT",
+ "dependencies": {
+ "once": "^1.4.0"
+ }
+ },
"node_modules/escalade": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
@@ -194,6 +218,12 @@
"node": ">=8"
}
},
+ "node_modules/inherits": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+ "license": "ISC"
+ },
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -211,6 +241,60 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/megajs": {
+ "version": "1.3.9",
+ "resolved": "https://registry.npmjs.org/megajs/-/megajs-1.3.9.tgz",
+ "integrity": "sha512-91GGJbUfUu9z/KFORHcn4bugVILWcGahaoy07Q7M5GLzT6zOsrpusxkjEvEys9XCXbxntg0v+f2JN6sITrEkPQ==",
+ "license": "MIT",
+ "dependencies": {
+ "pumpify": "^2.0.1",
+ "stream-skip": "^1.0.3"
+ }
+ },
+ "node_modules/once": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+ "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+ "license": "ISC",
+ "dependencies": {
+ "wrappy": "1"
+ }
+ },
+ "node_modules/pump": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz",
+ "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==",
+ "license": "MIT",
+ "dependencies": {
+ "end-of-stream": "^1.1.0",
+ "once": "^1.3.1"
+ }
+ },
+ "node_modules/pumpify": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz",
+ "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==",
+ "license": "MIT",
+ "dependencies": {
+ "duplexify": "^4.1.1",
+ "inherits": "^2.0.3",
+ "pump": "^3.0.0"
+ }
+ },
+ "node_modules/readable-stream": {
+ "version": "3.6.2",
+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+ "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+ "license": "MIT",
+ "dependencies": {
+ "inherits": "^2.0.3",
+ "string_decoder": "^1.1.1",
+ "util-deprecate": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -231,6 +315,26 @@
"tslib": "^2.1.0"
}
},
+ "node_modules/safe-buffer": {
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "license": "MIT"
+ },
"node_modules/shell-quote": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
@@ -250,6 +354,27 @@
"integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==",
"dev": true
},
+ "node_modules/stream-shift": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",
+ "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==",
+ "license": "MIT"
+ },
+ "node_modules/stream-skip": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/stream-skip/-/stream-skip-1.0.3.tgz",
+ "integrity": "sha512-2rB0uBiOnYSQwJxJ3wZLher+fz0yyXQxKuKnVTsidHmkqvC8rWZ2AbX50ZVdz7fsL6zkYkqaN/pPD0RldKIbpQ==",
+ "license": "MIT"
+ },
+ "node_modules/string_decoder": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
+ "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
+ "license": "MIT",
+ "dependencies": {
+ "safe-buffer": "~5.2.0"
+ }
+ },
"node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
@@ -311,6 +436,12 @@
"dev": true,
"license": "0BSD"
},
+ "node_modules/util-deprecate": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
+ "license": "MIT"
+ },
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -329,6 +460,12 @@
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
+ "node_modules/wrappy": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+ "license": "ISC"
+ },
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
diff --git a/package.json b/package.json
index e3eeb11..9ea46e3 100644
--- a/package.json
+++ b/package.json
@@ -12,5 +12,8 @@
},
"devDependencies": {
"concurrently": "^8.2.0"
+ },
+ "dependencies": {
+ "megajs": "^1.3.9"
}
}
diff --git a/server/auth.js b/server/auth.js
new file mode 100644
index 0000000..f5d0926
--- /dev/null
+++ b/server/auth.js
@@ -0,0 +1,355 @@
+import { Router } from 'express';
+import bcrypt from 'bcryptjs';
+import jwt from 'jsonwebtoken';
+import crypto from 'crypto';
+import {
+ getSetting, setSetting,
+ getAppUserCount, createAppUser, getAppUserByUsername, getAppUserById,
+ getAllAppUsers, updateAppUser, deleteAppUser,
+ getUserFolderAccess, setUserFolderAccess,
+ getUserRouteAccess, setUserRouteAccess,
+ getAllIndexedFolders,
+} from './db.js';
+
+const router = Router();
+const TOKEN_COOKIE = 'ofapp_token';
+const TOKEN_EXPIRY = undefined; // no expiration
+
+function getJwtSecret() {
+ let secret = getSetting('jwt_secret');
+ if (!secret) {
+ secret = crypto.randomBytes(48).toString('hex');
+ setSetting('jwt_secret', secret);
+ console.log('[auth] Generated new JWT secret');
+ }
+ return secret;
+}
+
+function signToken(userId) {
+ return jwt.sign({ userId }, getJwtSecret(), { expiresIn: TOKEN_EXPIRY });
+}
+
+function setTokenCookie(res, token) {
+ res.cookie(TOKEN_COOKIE, token, {
+ httpOnly: true,
+ sameSite: 'lax',
+ maxAge: 10 * 365 * 24 * 60 * 60 * 1000, // ~10 years
+ secure: false, // allow HTTP (self-signed HTTPS + local network)
+ });
+}
+
+function userPayload(user, routes, folders) {
+ return {
+ id: user.id,
+ username: user.username,
+ display_name: user.display_name,
+ role: user.role,
+ enabled: user.enabled,
+ routes,
+ folders: user.role === 'admin' ? null : folders, // null = all access
+ };
+}
+
+// --- Middleware ---
+
+export function requireAuth(req, res, next) {
+ // Setup mode: if no users exist, allow all requests as synthetic admin
+ const userCount = getAppUserCount();
+ if (userCount === 0) {
+ req.user = { id: 0, username: 'setup', role: 'admin', enabled: 1 };
+ return next();
+ }
+
+ const token = req.cookies?.[TOKEN_COOKIE];
+ if (!token) {
+ return res.status(401).json({ error: 'Authentication required' });
+ }
+
+ try {
+ const decoded = jwt.verify(token, getJwtSecret());
+ const user = getAppUserById(decoded.userId);
+ if (!user || !user.enabled) {
+ return res.status(401).json({ error: 'Account disabled or not found' });
+ }
+
+ req.user = user;
+
+ next();
+ } catch (err) {
+ return res.status(401).json({ error: 'Invalid or expired token' });
+ }
+}
+
+export function requireAdmin(req, res, next) {
+ if (req.user?.role !== 'admin') {
+ return res.status(403).json({ error: 'Admin access required' });
+ }
+ next();
+}
+
+// Route permission map: API path prefix -> route key
+const ROUTE_PERMISSION_MAP = {
+ '/api/feed': 'feed',
+ '/api/subscriptions': 'users',
+ '/api/users': 'users',
+ '/api/download': 'downloads',
+ '/api/gallery': 'gallery',
+ '/api/hls': 'gallery',
+ '/api/scrape': 'scrape',
+ '/api/settings': 'settings',
+ '/api/auth': 'settings',
+ '/api/dashboard': 'dashboard',
+ '/api/health': 'dashboard',
+ '/api/videos': 'videos',
+ '/api/video-hls': 'videos',
+ '/api/drm': null,
+ '/api/media-proxy': null,
+ '/api/me': null,
+ '/api/admin': null, // handled by requireAdmin
+ '/api/app-auth': null, // public
+};
+
+export function checkRoutePermission(req, res, next) {
+ // Admins bypass all route checks
+ if (req.user?.role === 'admin') return next();
+
+ // Find matching route key
+ const path = req.path;
+ let routeKey = undefined;
+ for (const [prefix, key] of Object.entries(ROUTE_PERMISSION_MAP)) {
+ if (path.startsWith(prefix)) {
+ routeKey = key;
+ break;
+ }
+ }
+
+ // null = always allowed for authenticated users
+ if (routeKey === null || routeKey === undefined) return next();
+
+ // Check user's route access
+ const userRoutes = getUserRouteAccess(req.user.id);
+ if (!userRoutes.includes(routeKey)) {
+ return res.status(403).json({ error: 'Access denied' });
+ }
+
+ next();
+}
+
+// --- Public endpoints (no auth required) ---
+
+// GET /api/app-auth/status — check if setup is needed
+router.get('/api/app-auth/status', (req, res) => {
+ const count = getAppUserCount();
+ res.json({ setupRequired: count === 0 });
+});
+
+// POST /api/app-auth/setup — create initial admin (only when 0 users exist)
+router.post('/api/app-auth/setup', (req, res) => {
+ const count = getAppUserCount();
+ if (count > 0) {
+ return res.status(400).json({ error: 'Setup already completed' });
+ }
+
+ const { username, password } = req.body;
+ if (!username || !password) {
+ return res.status(400).json({ error: 'Username and password required' });
+ }
+ if (password.length < 4) {
+ return res.status(400).json({ error: 'Password must be at least 4 characters' });
+ }
+
+ const hash = bcrypt.hashSync(password, 10);
+ const userId = createAppUser(username.trim(), hash, 'admin', username.trim());
+ const user = getAppUserById(userId);
+ const token = signToken(userId);
+ setTokenCookie(res, token);
+
+ res.json({
+ ok: true,
+ user: userPayload(user, [], null),
+ });
+});
+
+// POST /api/app-auth/login — validate creds, set JWT cookie
+router.post('/api/app-auth/login', (req, res) => {
+ const { username, password } = req.body;
+ if (!username || !password) {
+ return res.status(400).json({ error: 'Username and password required' });
+ }
+
+ const user = getAppUserByUsername(username.trim());
+ if (!user) {
+ return res.status(401).json({ error: 'Invalid username or password' });
+ }
+ if (!user.enabled) {
+ return res.status(401).json({ error: 'Account is disabled' });
+ }
+ if (!bcrypt.compareSync(password, user.password_hash)) {
+ return res.status(401).json({ error: 'Invalid username or password' });
+ }
+
+ const routes = getUserRouteAccess(user.id);
+ const folders = getUserFolderAccess(user.id);
+ const token = signToken(user.id);
+ setTokenCookie(res, token);
+
+ res.json({
+ ok: true,
+ user: userPayload(user, routes, folders),
+ });
+});
+
+// POST /api/app-auth/logout — clear JWT cookie
+router.post('/api/app-auth/logout', (req, res) => {
+ res.clearCookie(TOKEN_COOKIE);
+ res.json({ ok: true });
+});
+
+// --- Authenticated endpoints ---
+
+// GET /api/app-auth/me — current user + permissions
+router.get('/api/app-auth/me', requireAuth, (req, res) => {
+ if (!req.user || req.user.id === 0) {
+ return res.json({ setupRequired: true });
+ }
+
+ const routes = getUserRouteAccess(req.user.id);
+ const folders = getUserFolderAccess(req.user.id);
+ res.json({ user: userPayload(req.user, routes, folders) });
+});
+
+// --- Admin-only endpoints ---
+
+// GET /api/admin/users — list all users with permissions
+router.get('/api/admin/users', requireAuth, requireAdmin, (req, res) => {
+ const users = getAllAppUsers();
+ const result = users.map((u) => ({
+ ...u,
+ routes: getUserRouteAccess(u.id),
+ folders: getUserFolderAccess(u.id),
+ }));
+ res.json(result);
+});
+
+// POST /api/admin/users — create user
+router.post('/api/admin/users', requireAuth, requireAdmin, (req, res) => {
+ const { username, password, role, display_name, routes, folders } = req.body;
+ if (!username || !password) {
+ return res.status(400).json({ error: 'Username and password required' });
+ }
+ if (password.length < 4) {
+ return res.status(400).json({ error: 'Password must be at least 4 characters' });
+ }
+
+ const existing = getAppUserByUsername(username.trim());
+ if (existing) {
+ return res.status(400).json({ error: 'Username already exists' });
+ }
+
+ const hash = bcrypt.hashSync(password, 10);
+ const userId = createAppUser(username.trim(), hash, role || 'user', display_name || username.trim());
+
+ if (routes && Array.isArray(routes)) {
+ setUserRouteAccess(userId, routes);
+ }
+ if (folders && Array.isArray(folders)) {
+ setUserFolderAccess(userId, folders);
+ }
+
+ const user = getAppUserById(userId);
+ res.json({
+ ...user,
+ routes: getUserRouteAccess(userId),
+ folders: getUserFolderAccess(userId),
+ });
+});
+
+// PUT /api/admin/users/:id — update user
+router.put('/api/admin/users/:id', requireAuth, requireAdmin, (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ const target = getAppUserById(id);
+ if (!target) {
+ return res.status(404).json({ error: 'User not found' });
+ }
+
+ const { username, password, role, display_name, enabled, routes, folders } = req.body;
+
+ // Prevent demoting/disabling last admin
+ if (target.role === 'admin' && (role === 'user' || enabled === 0)) {
+ const allUsers = getAllAppUsers();
+ const adminCount = allUsers.filter(u => u.role === 'admin' && u.enabled).length;
+ if (adminCount <= 1) {
+ return res.status(400).json({ error: 'Cannot demote or disable the last admin' });
+ }
+ }
+
+ const fields = {};
+ if (username !== undefined) fields.username = username.trim();
+ if (display_name !== undefined) fields.display_name = display_name;
+ if (role !== undefined) fields.role = role;
+ if (enabled !== undefined) fields.enabled = enabled;
+ if (password) {
+ if (password.length < 4) {
+ return res.status(400).json({ error: 'Password must be at least 4 characters' });
+ }
+ fields.password_hash = bcrypt.hashSync(password, 10);
+ }
+
+ if (Object.keys(fields).length > 0) {
+ updateAppUser(id, fields);
+ }
+
+ if (routes !== undefined && Array.isArray(routes)) {
+ setUserRouteAccess(id, routes);
+ }
+ if (folders !== undefined && Array.isArray(folders)) {
+ setUserFolderAccess(id, folders);
+ }
+
+ const updated = getAppUserById(id);
+ res.json({
+ id: updated.id,
+ username: updated.username,
+ display_name: updated.display_name,
+ role: updated.role,
+ enabled: updated.enabled,
+ created_at: updated.created_at,
+ updated_at: updated.updated_at,
+ routes: getUserRouteAccess(id),
+ folders: getUserFolderAccess(id),
+ });
+});
+
+// DELETE /api/admin/users/:id — delete user
+router.delete('/api/admin/users/:id', requireAuth, requireAdmin, (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ const target = getAppUserById(id);
+ if (!target) {
+ return res.status(404).json({ error: 'User not found' });
+ }
+
+ // Cannot delete self
+ if (req.user.id === id) {
+ return res.status(400).json({ error: 'Cannot delete your own account' });
+ }
+
+ // Cannot delete last admin
+ if (target.role === 'admin') {
+ const allUsers = getAllAppUsers();
+ const adminCount = allUsers.filter(u => u.role === 'admin').length;
+ if (adminCount <= 1) {
+ return res.status(400).json({ error: 'Cannot delete the last admin' });
+ }
+ }
+
+ deleteAppUser(id);
+ res.json({ ok: true });
+});
+
+// GET /api/admin/available-folders — all gallery folders
+router.get('/api/admin/available-folders', requireAuth, requireAdmin, (req, res) => {
+ const folders = getAllIndexedFolders();
+ res.json(folders);
+});
+
+export default router;
diff --git a/server/dashboard.js b/server/dashboard.js
new file mode 100644
index 0000000..d14880f
--- /dev/null
+++ b/server/dashboard.js
@@ -0,0 +1,51 @@
+import { Router } from 'express';
+import {
+ getStorageStats, getTotalStorageSize, getDownloadsToday,
+ getRecentDownloads, getMediaFileCount, getAutoDownloadUsers, getAutoScrapeJobs,
+ getAuthConfig,
+} from './db.js';
+import { getActiveDownloadCount, getActiveDownloadsList } from './download.js';
+import { getActiveScrapeCount, getActiveScrapesList } from './scrape.js';
+
+const router = Router();
+
+router.get('/api/dashboard', (req, res) => {
+ try {
+ const storageStats = getStorageStats();
+ const totalStorage = getTotalStorageSize();
+ const totalFiles = getMediaFileCount();
+ const downloadsToday = getDownloadsToday();
+ const recentDownloads = getRecentDownloads(10);
+ const autoDownloads = getAutoDownloadUsers();
+ const autoScrapes = getAutoScrapeJobs();
+
+ res.json({
+ stats: {
+ totalFiles,
+ totalStorage,
+ totalFolders: storageStats.length,
+ downloadsToday,
+ },
+ topFolders: storageStats.slice(0, 10),
+ activeJobs: {
+ downloads: getActiveDownloadCount(),
+ scrapes: getActiveScrapeCount(),
+ downloadList: getActiveDownloadsList(),
+ scrapeList: getActiveScrapesList(),
+ },
+ auth: {
+ configured: !!getAuthConfig(),
+ },
+ scheduler: {
+ autoDownloadCount: autoDownloads.length,
+ autoScrapeCount: autoScrapes.length,
+ },
+ recentDownloads,
+ });
+ } catch (err) {
+ console.error('[dashboard] Error:', err.message);
+ res.status(500).json({ error: err.message });
+ }
+});
+
+export default router;
diff --git a/server/db.js b/server/db.js
index f3475aa..054c068 100644
--- a/server/db.js
+++ b/server/db.js
@@ -60,6 +60,64 @@ db.exec(`
CREATE INDEX IF NOT EXISTS idx_media_type ON media_files(type);
CREATE INDEX IF NOT EXISTS idx_media_modified ON media_files(modified);
CREATE INDEX IF NOT EXISTS idx_media_posted_at ON media_files(posted_at);
+
+ CREATE TABLE IF NOT EXISTS auto_download_users (
+ user_id TEXT PRIMARY KEY,
+ username TEXT NOT NULL,
+ enabled INTEGER DEFAULT 1,
+ last_run TEXT,
+ created_at TEXT DEFAULT (datetime('now'))
+ );
+
+ CREATE TABLE IF NOT EXISTS auto_scrape_jobs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ type TEXT NOT NULL,
+ url TEXT NOT NULL,
+ folder_name TEXT NOT NULL,
+ config TEXT NOT NULL,
+ enabled INTEGER DEFAULT 1,
+ last_run TEXT,
+ created_at TEXT DEFAULT (datetime('now'))
+ );
+
+ CREATE TABLE IF NOT EXISTS app_users (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ username TEXT NOT NULL UNIQUE COLLATE NOCASE,
+ password_hash TEXT NOT NULL,
+ display_name TEXT DEFAULT '',
+ role TEXT NOT NULL DEFAULT 'user',
+ enabled INTEGER DEFAULT 1,
+ created_at TEXT DEFAULT (datetime('now')),
+ updated_at TEXT DEFAULT (datetime('now'))
+ );
+
+ CREATE TABLE IF NOT EXISTS user_folder_access (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ user_id INTEGER NOT NULL,
+ folder TEXT NOT NULL,
+ UNIQUE(user_id, folder),
+ FOREIGN KEY (user_id) REFERENCES app_users(id) ON DELETE CASCADE
+ );
+
+ CREATE TABLE IF NOT EXISTS user_route_access (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ user_id INTEGER NOT NULL,
+ route_key TEXT NOT NULL,
+ UNIQUE(user_id, route_key),
+ FOREIGN KEY (user_id) REFERENCES app_users(id) ON DELETE CASCADE
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_user_folder_user ON user_folder_access(user_id);
+ CREATE INDEX IF NOT EXISTS idx_user_route_user ON user_route_access(user_id);
+
+ CREATE TABLE IF NOT EXISTS forum_sites (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL,
+ base_url TEXT DEFAULT '',
+ cookies TEXT DEFAULT '',
+ created_at TEXT DEFAULT (datetime('now')),
+ updated_at TEXT DEFAULT (datetime('now'))
+ );
`);
// Migration: add posted_at column if missing
@@ -68,6 +126,18 @@ if (!cols.includes('posted_at')) {
db.exec('ALTER TABLE download_history ADD COLUMN posted_at TEXT');
}
+// Migration: add username, password, cookie_expires_at columns to forum_sites
+const forumCols = db.prepare("PRAGMA table_info(forum_sites)").all().map((c) => c.name);
+if (!forumCols.includes('username')) {
+ db.exec("ALTER TABLE forum_sites ADD COLUMN username TEXT DEFAULT ''");
+}
+if (!forumCols.includes('password')) {
+ db.exec("ALTER TABLE forum_sites ADD COLUMN password TEXT DEFAULT ''");
+}
+if (!forumCols.includes('cookie_expires_at')) {
+ db.exec('ALTER TABLE forum_sites ADD COLUMN cookie_expires_at TEXT');
+}
+
export function getAuthConfig() {
const row = db.prepare('SELECT * FROM auth_config LIMIT 1').get();
return row || null;
@@ -177,15 +247,19 @@ export function getMediaFolders() {
SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) AS images,
SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) AS videos
FROM media_files
+ WHERE folder NOT LIKE '\\_%' ESCAPE '\\'
GROUP BY folder
ORDER BY folder
`).all();
}
-export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
+export function getMediaFiles({ folder, folders, type, sort, offset, limit, dateFrom, dateTo, minSize, maxSize, search }) {
const conditions = [];
const params = [];
+ // Always exclude folders starting with _
+ conditions.push("folder NOT LIKE '\\_%' ESCAPE '\\'");
+
if (folder) {
conditions.push('folder = ?');
params.push(folder);
@@ -199,17 +273,89 @@ export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
params.push(type);
}
+ if (dateFrom) {
+ conditions.push("COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) >= ?");
+ params.push(dateFrom);
+ }
+ if (dateTo) {
+ conditions.push("COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) <= ?");
+ params.push(dateTo + 'T23:59:59');
+ }
+ if (minSize) {
+ conditions.push('size >= ?');
+ params.push(parseInt(minSize, 10));
+ }
+ if (maxSize) {
+ conditions.push('size <= ?');
+ params.push(parseInt(maxSize, 10));
+ }
+ if (search) {
+ conditions.push('(filename LIKE ? OR folder LIKE ?)');
+ params.push(`%${search}%`, `%${search}%`);
+ }
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
const countRow = db.prepare(`SELECT COUNT(*) AS total FROM media_files ${where}`).get(...params);
const total = countRow.total;
- let orderBy;
+ const effectiveLimit = limit || 50;
+ const effectiveOffset = offset || 0;
+
+ // Equal-mix shuffle: when shuffling with multiple folders, sample equally from each
if (sort === 'shuffle') {
- orderBy = 'ORDER BY RANDOM()';
- } else {
- // 'latest' — prefer posted_at, fall back to modified
- orderBy = 'ORDER BY COALESCE(posted_at, datetime(modified / 1000, \'unixepoch\')) DESC';
+ // Count distinct folders in the result set
+ const folderCountRow = db.prepare(
+ `SELECT COUNT(DISTINCT folder) AS cnt FROM media_files ${where}`
+ ).get(...params);
+ const numFolders = folderCountRow?.cnt || 1;
+
+ if (numFolders > 1) {
+ // Use ROW_NUMBER to pick equal random samples per folder
+ const perFolder = Math.ceil(effectiveLimit / numFolders);
+ const rows = db.prepare(`
+ WITH ranked AS (
+ SELECT folder, filename, type, size, modified, posted_at,
+ ROW_NUMBER() OVER (PARTITION BY folder ORDER BY RANDOM()) AS rn
+ FROM media_files
+ ${where}
+ )
+ SELECT folder, filename, type, size, modified, posted_at
+ FROM ranked
+ WHERE rn <= ?
+ ORDER BY RANDOM()
+ LIMIT ? OFFSET ?
+ `).all(...params, perFolder, effectiveLimit, effectiveOffset);
+ return { total, rows };
+ }
+
+ // Single folder or no folder filter — plain random
+ const rows = db.prepare(`
+ SELECT folder, filename, type, size, modified, posted_at
+ FROM media_files
+ ${where}
+ ORDER BY RANDOM()
+ LIMIT ? OFFSET ?
+ `).all(...params, effectiveLimit, effectiveOffset);
+ return { total, rows };
+ }
+
+ let orderBy;
+ switch (sort) {
+ case 'oldest':
+ orderBy = "ORDER BY COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) ASC";
+ break;
+ case 'largest':
+ orderBy = 'ORDER BY size DESC';
+ break;
+ case 'smallest':
+ orderBy = 'ORDER BY size ASC';
+ break;
+ case 'name':
+ orderBy = 'ORDER BY filename ASC';
+ break;
+ default:
+ orderBy = "ORDER BY COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) DESC";
}
const rows = db.prepare(`
@@ -218,7 +364,7 @@ export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
${where}
${orderBy}
LIMIT ? OFFSET ?
- `).all(...params, limit || 50, offset || 0);
+ `).all(...params, effectiveLimit, effectiveOffset);
return { total, rows };
}
@@ -244,3 +390,399 @@ export function removeStaleFiles(folder, existingFilenames) {
export function getMediaFileCount() {
return db.prepare('SELECT COUNT(*) AS count FROM media_files').get().count;
}
+
+export function getNewMediaCount(since) {
+ return db.prepare('SELECT COUNT(*) AS count FROM media_files WHERE created_at > ?').get(since).count;
+}
+
+// --- auto_download_users helpers ---
+
+export function getAutoDownloadUsers() {
+ return db.prepare('SELECT * FROM auto_download_users WHERE enabled = 1').all();
+}
+
+export function addAutoDownloadUser(userId, username) {
+ db.prepare(
+ 'INSERT INTO auto_download_users (user_id, username) VALUES (?, ?) ON CONFLICT(user_id) DO UPDATE SET username = excluded.username, enabled = 1'
+ ).run(String(userId), username);
+}
+
+export function removeAutoDownloadUser(userId) {
+ db.prepare('DELETE FROM auto_download_users WHERE user_id = ?').run(String(userId));
+}
+
+export function isAutoDownloadUser(userId) {
+ return !!db.prepare('SELECT 1 FROM auto_download_users WHERE user_id = ? AND enabled = 1').get(String(userId));
+}
+
+export function updateAutoDownloadLastRun(userId) {
+ db.prepare('UPDATE auto_download_users SET last_run = datetime(\'now\') WHERE user_id = ?').run(String(userId));
+}
+
+// --- auto_scrape_jobs helpers ---
+
+export function getAutoScrapeJobs() {
+ return db.prepare('SELECT * FROM auto_scrape_jobs WHERE enabled = 1').all();
+}
+
+export function addAutoScrapeJob(type, url, folderName, config) {
+ db.prepare(
+ 'INSERT INTO auto_scrape_jobs (type, url, folder_name, config) VALUES (?, ?, ?, ?)'
+ ).run(type, url, folderName, JSON.stringify(config));
+}
+
+export function removeAutoScrapeJob(id) {
+ db.prepare('DELETE FROM auto_scrape_jobs WHERE id = ?').run(id);
+}
+
+export function updateAutoScrapeLastRun(id) {
+ db.prepare('UPDATE auto_scrape_jobs SET last_run = datetime(\'now\') WHERE id = ?').run(id);
+}
+
+// --- Dashboard / stats helpers ---
+
+export function getStorageStats() {
+ return db.prepare(`
+ SELECT folder,
+ COUNT(*) AS file_count,
+ SUM(size) AS total_size,
+ SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) AS images,
+ SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) AS videos
+ FROM media_files
+ GROUP BY folder
+ ORDER BY SUM(size) DESC
+ `).all();
+}
+
+export function getTotalStorageSize() {
+ const row = db.prepare('SELECT SUM(size) AS total FROM media_files').get();
+ return row?.total || 0;
+}
+
+// --- videos tables ---
+
+db.exec(`
+ CREATE TABLE IF NOT EXISTS videos (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ title TEXT NOT NULL,
+ description TEXT DEFAULT '',
+ filename TEXT NOT NULL,
+ file_path TEXT NOT NULL,
+ file_size INTEGER NOT NULL DEFAULT 0,
+ duration REAL,
+ width INTEGER,
+ height INTEGER,
+ fps REAL,
+ codec TEXT,
+ bitrate INTEGER,
+ has_audio INTEGER DEFAULT 1,
+ thumbnail_path TEXT,
+ status TEXT DEFAULT 'pending',
+ error_message TEXT,
+ created_at TEXT DEFAULT (datetime('now')),
+ updated_at TEXT DEFAULT (datetime('now')),
+ UNIQUE(file_path)
+ );
+
+ CREATE TABLE IF NOT EXISTS tags (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ name TEXT NOT NULL UNIQUE COLLATE NOCASE,
+ created_at TEXT DEFAULT (datetime('now'))
+ );
+
+ CREATE TABLE IF NOT EXISTS video_tags (
+ video_id INTEGER NOT NULL,
+ tag_id INTEGER NOT NULL,
+ PRIMARY KEY (video_id, tag_id),
+ FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE,
+ FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE
+ );
+`);
+
+// --- video helpers ---
+
+export function insertVideo(data) {
+ const stmt = db.prepare(`
+ INSERT INTO videos (title, description, filename, file_path, file_size, duration, width, height, fps, codec, bitrate, has_audio, thumbnail_path, status, error_message)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `);
+ const result = stmt.run(
+ data.title, data.description || '', data.filename, data.file_path, data.file_size || 0,
+ data.duration || null, data.width || null, data.height || null, data.fps || null,
+ data.codec || null, data.bitrate || null, data.has_audio ?? 1,
+ data.thumbnail_path || null, data.status || 'pending', data.error_message || null
+ );
+ return result.lastInsertRowid;
+}
+
+export function getVideoById(id) {
+ return db.prepare('SELECT * FROM videos WHERE id = ?').get(id) || null;
+}
+
+export function getVideoByPath(filePath) {
+ return db.prepare('SELECT * FROM videos WHERE file_path = ?').get(filePath) || null;
+}
+
+export function updateVideo(id, data) {
+ const fields = [];
+ const values = [];
+ for (const [key, val] of Object.entries(data)) {
+ if (key === 'id') continue;
+ fields.push(`${key} = ?`);
+ values.push(val);
+ }
+ if (fields.length === 0) return;
+ fields.push("updated_at = datetime('now')");
+ values.push(id);
+ db.prepare(`UPDATE videos SET ${fields.join(', ')} WHERE id = ?`).run(...values);
+}
+
+export function deleteVideoById(id) {
+ db.prepare('DELETE FROM video_tags WHERE video_id = ?').run(id);
+ db.prepare('DELETE FROM videos WHERE id = ?').run(id);
+}
+
+export function searchVideos({ search, tags, minDuration, maxDuration, minWidth, sort, offset, limit }) {
+ const conditions = [];
+ const params = [];
+
+ if (search) {
+ conditions.push('(v.title LIKE ? OR v.description LIKE ? OR v.filename LIKE ?)');
+ params.push(`%${search}%`, `%${search}%`, `%${search}%`);
+ }
+ if (minDuration) {
+ conditions.push('v.duration >= ?');
+ params.push(parseFloat(minDuration));
+ }
+ if (maxDuration) {
+ conditions.push('v.duration <= ?');
+ params.push(parseFloat(maxDuration));
+ }
+ if (minWidth) {
+ conditions.push('v.width >= ?');
+ params.push(parseInt(minWidth, 10));
+ }
+ if (tags && tags.length > 0) {
+ conditions.push(`v.id IN (
+ SELECT vt.video_id FROM video_tags vt
+ JOIN tags t ON t.id = vt.tag_id
+ WHERE t.name IN (${tags.map(() => '?').join(',')})
+ )`);
+ params.push(...tags);
+ }
+
+ conditions.push("v.status = 'ready'");
+ const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+
+ const countRow = db.prepare(`SELECT COUNT(*) AS total FROM videos v ${where}`).get(...params);
+ const total = countRow.total;
+
+ let orderBy;
+ switch (sort) {
+ case 'oldest': orderBy = 'ORDER BY v.created_at ASC'; break;
+ case 'longest': orderBy = 'ORDER BY v.duration DESC'; break;
+ case 'shortest': orderBy = 'ORDER BY v.duration ASC'; break;
+ case 'largest': orderBy = 'ORDER BY v.file_size DESC'; break;
+ case 'title': orderBy = 'ORDER BY v.title ASC'; break;
+ case 'shuffle': orderBy = 'ORDER BY RANDOM()'; break;
+ default: orderBy = 'ORDER BY v.created_at DESC';
+ }
+
+ const effectiveLimit = limit || 48;
+ const effectiveOffset = offset || 0;
+
+ const rows = db.prepare(`
+ SELECT v.* FROM videos v ${where} ${orderBy} LIMIT ? OFFSET ?
+ `).all(...params, effectiveLimit, effectiveOffset);
+
+ return { total, rows };
+}
+
+export function getOrCreateTag(name) {
+ const trimmed = name.trim();
+ if (!trimmed) return null;
+ let row = db.prepare('SELECT id FROM tags WHERE name = ? COLLATE NOCASE').get(trimmed);
+ if (!row) {
+ const result = db.prepare('INSERT INTO tags (name) VALUES (?)').run(trimmed);
+ return result.lastInsertRowid;
+ }
+ return row.id;
+}
+
+export function getAllTags(search) {
+ if (search) {
+ return db.prepare(`
+ SELECT t.id, t.name, COUNT(vt.video_id) AS count
+ FROM tags t
+ LEFT JOIN video_tags vt ON vt.tag_id = t.id
+ WHERE t.name LIKE ?
+ GROUP BY t.id
+ ORDER BY count DESC, t.name ASC
+ `).all(`%${search}%`);
+ }
+ return db.prepare(`
+ SELECT t.id, t.name, COUNT(vt.video_id) AS count
+ FROM tags t
+ LEFT JOIN video_tags vt ON vt.tag_id = t.id
+ GROUP BY t.id
+ ORDER BY count DESC, t.name ASC
+ `).all();
+}
+
+export function setVideoTags(videoId, tagNames) {
+ const setTags = db.transaction((names) => {
+ db.prepare('DELETE FROM video_tags WHERE video_id = ?').run(videoId);
+ for (const name of names) {
+ const tagId = getOrCreateTag(name);
+ if (tagId) {
+ db.prepare('INSERT OR IGNORE INTO video_tags (video_id, tag_id) VALUES (?, ?)').run(videoId, tagId);
+ }
+ }
+ });
+ setTags(tagNames);
+}
+
+export function getVideoTags(videoId) {
+ return db.prepare(`
+ SELECT t.id, t.name FROM tags t
+ JOIN video_tags vt ON vt.tag_id = t.id
+ WHERE vt.video_id = ?
+ ORDER BY t.name ASC
+ `).all(videoId);
+}
+
+export function getDownloadsToday() {
+ // created_at is stored in UTC via SQLite datetime('now').
+ // Compute local midnight in UTC-relative format so "today" matches the server's local day.
+ const now = new Date();
+ const localMidnight = new Date(now.getFullYear(), now.getMonth(), now.getDate());
+ // Format as SQLite-compatible "YYYY-MM-DD HH:MM:SS"
+ const y = localMidnight.getUTCFullYear();
+ const mo = String(localMidnight.getUTCMonth() + 1).padStart(2, '0');
+ const d = String(localMidnight.getUTCDate()).padStart(2, '0');
+ const h = String(localMidnight.getUTCHours()).padStart(2, '0');
+ const mi = String(localMidnight.getUTCMinutes()).padStart(2, '0');
+ const s = String(localMidnight.getUTCSeconds()).padStart(2, '0');
+ const todayUtc = `${y}-${mo}-${d} ${h}:${mi}:${s}`;
+ const row = db.prepare('SELECT COUNT(*) AS count FROM media_files WHERE created_at >= ?').get(todayUtc);
+ return row?.count || 0;
+}
+
+export function getRecentDownloads(limit = 10) {
+ // Merge recent items from both download_history and media_files (for scrapes)
+ return db.prepare(`
+ SELECT filename, type AS media_type, folder AS user_id, created_at AS downloaded_at
+ FROM media_files
+ ORDER BY created_at DESC
+ LIMIT ?
+ `).all(limit);
+}
+
+// --- app_users helpers ---
+
+export function createAppUser(username, passwordHash, role = 'user', displayName = '') {
+ const result = db.prepare(
+ 'INSERT INTO app_users (username, password_hash, role, display_name) VALUES (?, ?, ?, ?)'
+ ).run(username, passwordHash, role, displayName);
+ return result.lastInsertRowid;
+}
+
+export function getAppUserByUsername(username) {
+ return db.prepare('SELECT * FROM app_users WHERE username = ?').get(username) || null;
+}
+
+export function getAppUserById(id) {
+ return db.prepare('SELECT * FROM app_users WHERE id = ?').get(id) || null;
+}
+
+export function getAllAppUsers() {
+ return db.prepare('SELECT id, username, display_name, role, enabled, created_at, updated_at FROM app_users ORDER BY id').all();
+}
+
+export function updateAppUser(id, fields) {
+ const allowed = ['username', 'password_hash', 'display_name', 'role', 'enabled'];
+ const sets = [];
+ const values = [];
+ for (const [key, val] of Object.entries(fields)) {
+ if (!allowed.includes(key)) continue;
+ sets.push(`${key} = ?`);
+ values.push(val);
+ }
+ if (sets.length === 0) return;
+ sets.push("updated_at = datetime('now')");
+ values.push(id);
+ db.prepare(`UPDATE app_users SET ${sets.join(', ')} WHERE id = ?`).run(...values);
+}
+
+export function deleteAppUser(id) {
+ db.prepare('DELETE FROM app_users WHERE id = ?').run(id);
+}
+
+export function getAppUserCount() {
+ return db.prepare('SELECT COUNT(*) AS count FROM app_users').get().count;
+}
+
+export function getUserFolderAccess(userId) {
+ return db.prepare('SELECT folder FROM user_folder_access WHERE user_id = ?').all(userId).map(r => r.folder);
+}
+
+export function setUserFolderAccess(userId, folders) {
+ const update = db.transaction((flds) => {
+ db.prepare('DELETE FROM user_folder_access WHERE user_id = ?').run(userId);
+ const ins = db.prepare('INSERT INTO user_folder_access (user_id, folder) VALUES (?, ?)');
+ for (const f of flds) {
+ ins.run(userId, f);
+ }
+ });
+ update(folders);
+}
+
+export function getUserRouteAccess(userId) {
+ return db.prepare('SELECT route_key FROM user_route_access WHERE user_id = ?').all(userId).map(r => r.route_key);
+}
+
+export function setUserRouteAccess(userId, routes) {
+ const update = db.transaction((rts) => {
+ db.prepare('DELETE FROM user_route_access WHERE user_id = ?').run(userId);
+ const ins = db.prepare('INSERT INTO user_route_access (user_id, route_key) VALUES (?, ?)');
+ for (const r of rts) {
+ ins.run(userId, r);
+ }
+ });
+ update(routes);
+}
+
+// --- Forum Sites ---
+
+export function getForumSites() {
+ return db.prepare('SELECT * FROM forum_sites ORDER BY name').all();
+}
+
+export function getForumSiteById(id) {
+ return db.prepare('SELECT * FROM forum_sites WHERE id = ?').get(id);
+}
+
+export function createForumSite(name, baseUrl, cookies, username, password) {
+ const result = db.prepare('INSERT INTO forum_sites (name, base_url, cookies, username, password) VALUES (?, ?, ?, ?, ?)').run(name, baseUrl || '', cookies || '', username || '', password || '');
+ return result.lastInsertRowid;
+}
+
+export function updateForumSite(id, fields) {
+ const allowed = ['name', 'base_url', 'cookies', 'username', 'password', 'cookie_expires_at'];
+ const sets = [];
+ const vals = [];
+ for (const [k, v] of Object.entries(fields)) {
+ if (allowed.includes(k)) {
+ sets.push(`${k} = ?`);
+ vals.push(v);
+ }
+ }
+ if (sets.length === 0) return;
+ sets.push("updated_at = datetime('now')");
+ vals.push(id);
+ db.prepare(`UPDATE forum_sites SET ${sets.join(', ')} WHERE id = ?`).run(...vals);
+}
+
+export function deleteForumSite(id) {
+ db.prepare('DELETE FROM forum_sites WHERE id = ?').run(id);
+}
diff --git a/server/download.js b/server/download.js
index 94714e3..ae02c5e 100644
--- a/server/download.js
+++ b/server/download.js
@@ -3,7 +3,7 @@ import fetch from 'node-fetch';
import { mkdirSync, createWriteStream, statSync } from 'fs';
import { pipeline } from 'stream/promises';
import { extname } from 'path';
-import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile } from './db.js';
+import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile, getAutoDownloadUsers, addAutoDownloadUser, removeAutoDownloadUser } from './db.js';
import { createSignedHeaders, getRules } from './signing.js';
import { downloadDrmMedia, hasCDM } from './drm-download.js';
@@ -15,6 +15,36 @@ const DOWNLOAD_DELAY = parseInt(process.env.DOWNLOAD_DELAY || '1000', 10);
// In-memory progress: userId -> { total, completed, errors, running }
const progressMap = new Map();
+// In-memory download logs: userId -> last N file entries
+const downloadLogMap = new Map();
+const MAX_LOG_ENTRIES = 20;
+
+function addDownloadLog(userId, entry) {
+ const key = String(userId);
+ if (!downloadLogMap.has(key)) downloadLogMap.set(key, []);
+ const logs = downloadLogMap.get(key);
+ logs.push({ ...entry, timestamp: new Date().toISOString() });
+ if (logs.length > MAX_LOG_ENTRIES) logs.shift();
+}
+
+export function getActiveDownloadCount() {
+ let count = 0;
+ for (const p of progressMap.values()) {
+ if (p.running) count++;
+ }
+ return count;
+}
+
+export function getActiveDownloadsList() {
+ const list = [];
+ for (const [userId, p] of progressMap.entries()) {
+ if (p.running) {
+ list.push({ userId, username: p.username, total: p.total, completed: p.completed, errors: p.errors });
+ }
+ }
+ return list;
+}
+
function buildHeaders(authConfig, signedHeaders) {
const rules = getRules();
const headers = {
@@ -83,8 +113,9 @@ async function downloadFile(url, dest) {
}
async function runDownload(userId, authConfig, postLimit, resume, username) {
- const progress = { total: 0, completed: 0, errors: 0, running: true };
+ const progress = { total: 0, completed: 0, errors: 0, running: true, username: username || null };
progressMap.set(String(userId), progress);
+ console.log(`[download] Starting download for user ${userId} (${username || 'unknown'})${postLimit ? ` limit=${postLimit}` : ' all posts'}${resume ? ' (resume)' : ''}`);
try {
let beforePublishTime = null;
@@ -156,6 +187,7 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
}
progress.total = allMedia.length;
+ console.log(`[download] User ${userId}: found ${allMedia.length} media items across ${postsFetched} posts`);
// Phase 2: Download each media item
for (const { postId, media, postDate } of allMedia) {
@@ -203,9 +235,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
const st = statSync(`${userDir}/${drmFilename}`);
upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate);
} catch { /* stat may fail if file was cleaned up */ }
+ addDownloadLog(userId, { filename: drmFilename, mediaType: 'video', status: 'ok' });
progress.completed++;
} catch (err) {
console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
+ addDownloadLog(userId, { filename: `${postId}_${mediaId}_video.mp4`, mediaType: 'video', status: 'error' });
progress.errors++;
progress.completed++;
}
@@ -234,9 +268,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
} catch { /* ignore */ }
+ addDownloadLog(userId, { filename, mediaType, status: 'ok' });
progress.completed++;
} catch (err) {
console.error(`[download] Error downloading media ${media.id}:`, err.message);
+ addDownloadLog(userId, { filename: `${postId}_${media.id}`, mediaType: media.type || 'unknown', status: 'error' });
progress.errors++;
progress.completed++;
}
@@ -251,6 +287,75 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
}
}
+// POST /api/download/post — download media from a single post
+router.post('/api/download/post', async (req, res, next) => {
+ try {
+ const { userId, username, postId, postedAt, media: mediaItems } = req.body;
+ if (!userId || !postId || !Array.isArray(mediaItems) || mediaItems.length === 0) {
+ return res.status(400).json({ error: 'userId, postId, and media[] are required' });
+ }
+ const postDate = postedAt || null;
+
+ const userDir = `${MEDIA_PATH}/${username || userId}`;
+ mkdirSync(userDir, { recursive: true });
+ let completed = 0, errors = 0;
+
+ console.log(`[download] Post ${postId}: downloading ${mediaItems.length} media items for ${username || userId}, postedAt=${postDate}`);
+
+ for (const media of mediaItems) {
+ try {
+ const mediaId = String(media.id);
+ if (isMediaDownloaded(mediaId)) { completed++; continue; }
+ if (media.canView === false) { completed++; continue; }
+
+ // DRM video
+ const drm = media.files?.drm;
+ if (drm?.manifest?.dash && drm?.signature?.dash) {
+ if (!hasCDM()) { completed++; continue; }
+ try {
+ const sig = drm.signature.dash;
+ const cfCookies = { cp: sig['CloudFront-Policy'], cs: sig['CloudFront-Signature'], ck: sig['CloudFront-Key-Pair-Id'] };
+ const drmFilename = `${postId}_${mediaId}_video.mp4`;
+ await downloadDrmMedia({ mpdUrl: drm.manifest.dash, cfCookies, mediaId, entityType: 'post', entityId: String(postId), outputDir: userDir, outputFilename: drmFilename });
+ recordDownload(userId, String(postId), mediaId, 'video', drmFilename, postDate);
+ try { const st = statSync(`${userDir}/${drmFilename}`); upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate); } catch {}
+ completed++;
+ } catch (err) {
+ console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
+ errors++;
+ }
+ continue;
+ }
+
+ const url = getMediaUrl(media);
+ if (!url) { completed++; continue; }
+
+ const mediaType = media.type || 'unknown';
+ const ext = getExtFromUrl(url);
+ const filename = `${postId}_${mediaId}_${mediaType}${ext}`;
+ const dest = `${userDir}/${filename}`;
+
+ await downloadFile(url, dest);
+ recordDownload(userId, String(postId), mediaId, mediaType, filename, postDate);
+ try {
+ const st = statSync(dest);
+ const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
+ if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
+ } catch {}
+ completed++;
+ } catch (err) {
+ console.error(`[download] Error downloading media ${media.id}:`, err.message);
+ errors++;
+ }
+ }
+
+ console.log(`[download] Post ${postId}: done (${completed} downloaded, ${errors} errors)`);
+ res.json({ status: 'done', completed, errors, total: mediaItems.length });
+ } catch (err) {
+ next(err);
+ }
+});
+
// POST /api/download/:userId — start background download
router.post('/api/download/:userId', (req, res, next) => {
try {
@@ -302,6 +407,21 @@ router.get('/api/download/active', (req, res) => {
res.json(active);
});
+// GET /api/download/active/details — active downloads with recent file logs
+router.get('/api/download/active/details', (req, res) => {
+ const active = [];
+ for (const [userId, progress] of progressMap.entries()) {
+ if (progress.running) {
+ active.push({
+ user_id: userId,
+ ...progress,
+ recentFiles: downloadLogMap.get(String(userId))?.slice(-5) || [],
+ });
+ }
+ }
+ res.json(active);
+});
+
// GET /api/download/history
router.get('/api/download/history', (req, res, next) => {
try {
@@ -312,4 +432,24 @@ router.get('/api/download/history', (req, res, next) => {
}
});
+// --- Auto-download CRUD ---
+
+router.get('/api/download/auto', (_req, res) => {
+ res.json(getAutoDownloadUsers());
+});
+
+router.post('/api/download/auto/:userId', (req, res) => {
+ const { userId } = req.params;
+ const { username } = req.body;
+ if (!username) return res.status(400).json({ error: 'username is required' });
+ addAutoDownloadUser(userId, username);
+ res.json({ ok: true });
+});
+
+router.delete('/api/download/auto/:userId', (req, res) => {
+ removeAutoDownloadUser(req.params.userId);
+ res.json({ ok: true });
+});
+
+export { runDownload };
export default router;
diff --git a/server/flaresolverr.js b/server/flaresolverr.js
new file mode 100644
index 0000000..89367e7
--- /dev/null
+++ b/server/flaresolverr.js
@@ -0,0 +1,128 @@
+import { Router } from 'express';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { getForumSiteById, updateForumSite } from './db.js';
+
+const execAsync = promisify(exec);
+const router = Router();
+const FLARESOLVERR_URL = process.env.FLARESOLVERR_URL || 'http://localhost:8191';
+const CHROMIUM_PATH = process.env.CHROMIUM_PATH || '/usr/bin/chromium-browser';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Refresh forum cookies using undetected_chromedriver (Python).
+ * Runs login_helper.py via xvfb-run so Chrome runs in headed mode
+ * with a virtual display — this is what lets Turnstile auto-solve.
+ */
+export async function refreshForumCookies(siteId) {
+ const site = getForumSiteById(siteId);
+ if (!site) throw new Error(`Forum site ${siteId} not found`);
+ if (!site.username || !site.password) {
+ throw new Error('Forum site has no saved credentials — set username and password first');
+ }
+
+ const baseUrl = site.base_url || 'https://simpcity.su';
+ const loginUrl = `${baseUrl}/login/`;
+ const helperPath = path.join(__dirname, 'login_helper.py');
+
+ console.log(`[flaresolverr] Refreshing cookies for site ${siteId} (${site.name})`);
+ console.log(`[flaresolverr] Login URL: ${loginUrl}`);
+
+ // Run the Python helper with xvfb-run for virtual display
+ // Escape arguments for shell safety
+ const escapedUrl = loginUrl.replace(/'/g, "'\\''");
+ const escapedUser = site.username.replace(/'/g, "'\\''");
+ const escapedPass = site.password.replace(/'/g, "'\\''");
+
+ const cmd = `xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' python3 '${helperPath}' '${escapedUrl}' '${escapedUser}' '${escapedPass}'`;
+
+ try {
+ const { stdout, stderr } = await execAsync(cmd, {
+ timeout: 120000, // 2 minutes
+ maxBuffer: 10 * 1024 * 1024,
+ env: { ...process.env, CHROMIUM_PATH },
+ });
+
+ // Log stderr (debug output from login_helper.py)
+ if (stderr) {
+ for (const line of stderr.split('\n').filter(Boolean)) {
+ console.log(`[flaresolverr] ${line}`);
+ }
+ }
+
+ // Parse JSON from stdout
+ const result = JSON.parse(stdout.trim());
+
+ if (!result.ok) {
+ throw new Error(result.error || 'Login failed');
+ }
+
+ // Update DB with new cookies
+ const expiresAt = new Date(Date.now() + 25 * 24 * 60 * 60 * 1000).toISOString();
+ updateForumSite(siteId, {
+ cookies: result.cookies,
+ cookie_expires_at: expiresAt,
+ });
+
+ console.log(`[flaresolverr] Cookie refresh successful for site ${siteId}`);
+ return result.cookies;
+ } catch (err) {
+ // If execAsync fails, the error might have stderr info
+ if (err.stderr) {
+ for (const line of err.stderr.split('\n').filter(Boolean)) {
+ console.error(`[flaresolverr] ${line}`);
+ }
+ }
+ // Try to parse stdout for a structured error
+ if (err.stdout) {
+ try {
+ const result = JSON.parse(err.stdout.trim());
+ if (result.error) throw new Error(result.error);
+ } catch (parseErr) {
+ // Not JSON, use original error
+ }
+ }
+ throw new Error(`Cookie refresh failed: ${err.message}`);
+ }
+}
+
+// --- API Endpoints ---
+
+// Manual cookie refresh
+router.post('/api/flaresolverr/refresh/:siteId', async (req, res) => {
+ const siteId = parseInt(req.params.siteId, 10);
+ try {
+ const cookieStr = await refreshForumCookies(siteId);
+ res.json({ ok: true, cookies: cookieStr });
+ } catch (err) {
+ console.error(`[flaresolverr] Refresh failed for site ${siteId}:`, err.message);
+ res.status(500).json({ error: err.message });
+ }
+});
+
+// Check if cookie refresh is available (Chromium + xvfb-run installed)
+router.get('/api/flaresolverr/status', async (_req, res) => {
+ try {
+ // Check for xvfb-run and chromium
+ await execAsync('which xvfb-run && which chromium-browser || which chromium', { timeout: 5000 });
+ // Check for undetected_chromedriver python package
+ await execAsync('python3 -c "import undetected_chromedriver"', { timeout: 5000 });
+ res.json({ available: true });
+ } catch {
+ // Fallback: check FlareSolverr service
+ try {
+ const resp = await fetch(`${FLARESOLVERR_URL}/health`, {
+ signal: AbortSignal.timeout(5000),
+ });
+ res.json({ available: resp.ok });
+ } catch {
+ res.json({ available: false, error: 'Neither undetected_chromedriver nor FlareSolverr available' });
+ }
+ }
+});
+
+export default router;
diff --git a/server/gallery.js b/server/gallery.js
index d6f97b1..d8e77dd 100644
--- a/server/gallery.js
+++ b/server/gallery.js
@@ -8,6 +8,7 @@ import {
getPostDateByFilename, getSetting,
upsertMediaFileBatch, removeMediaFile, removeStaleFiles,
getMediaFolders, getMediaFiles, getMediaFileCount, getAllIndexedFolders,
+ getNewMediaCount, getUserFolderAccess,
} from './db.js';
const execFileAsync = promisify(execFile);
@@ -107,20 +108,49 @@ export function scanMediaFiles() {
console.log(`[gallery] Index scan complete: ${totalFiles} files in ${scannedFolders.size} folders (${elapsed}s). DB total: ${dbCount}`);
}
+// Helper: get allowed folders for current user (null = all access)
+function getAllowedFolders(req) {
+ if (!req.user || req.user.role === 'admin') return null;
+ return getUserFolderAccess(req.user.id);
+}
+
+function checkFolderAccess(req, folder) {
+ const allowed = getAllowedFolders(req);
+ if (allowed === null) return true; // admin or no restrictions
+ return allowed.includes(folder);
+}
+
+// GET /api/gallery/new-count — count of media added since last gallery visit
+router.get('/api/gallery/new-count', (req, res, next) => {
+ try {
+ const lastSeen = getSetting('gallery_last_seen');
+ if (!lastSeen) return res.json({ count: 0 });
+ const count = getNewMediaCount(lastSeen);
+ res.json({ count });
+ } catch (err) {
+ next(err);
+ }
+});
+
// GET /api/gallery/folders — list all folders with file counts (from DB index)
router.get('/api/gallery/folders', (req, res, next) => {
try {
- const folders = getMediaFolders();
+ let folders = getMediaFolders();
+ const allowed = getAllowedFolders(req);
+ if (allowed !== null) {
+ const set = new Set(allowed);
+ folders = folders.filter(f => set.has(f.name));
+ }
res.json(folders);
} catch (err) {
next(err);
}
});
-// GET /api/gallery/files?folder=&type=&sort=&offset=&limit= (from DB index)
+// GET /api/gallery/files?folder=&type=&sort=&offset=&limit=&dateFrom=&dateTo=&minSize=&maxSize=&search= (from DB index)
router.get('/api/gallery/files', (req, res, next) => {
try {
- const { folder, type, sort, offset, limit } = req.query;
+ const { folder, type, sort, offset, limit, dateFrom, dateTo, minSize, maxSize, search } = req.query;
const foldersParam = req.query.folders;
const foldersArr = foldersParam
? foldersParam.split(',').map((f) => f.trim()).filter(Boolean)
@@ -130,13 +160,45 @@ router.get('/api/gallery/files', (req, res, next) => {
const limitNum = parseInt(limit || '50', 10);
const hlsEnabled = (getSetting('hls_enabled') || process.env.HLS_ENABLED) === 'true';
+ // Enforce folder access for non-admin users
+ const allowed = getAllowedFolders(req);
+ let effectiveFolder = folder || undefined;
+ let effectiveFolders = foldersArr;
+
+ if (allowed !== null) {
+ const allowedSet = new Set(allowed);
+ if (effectiveFolder) {
+ // Requested specific folder — must be allowed
+ if (!allowedSet.has(effectiveFolder)) {
+ return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
+ }
+ } else if (effectiveFolders && effectiveFolders.length > 0) {
+ // Requested multiple folders — intersect with allowed
+ effectiveFolders = effectiveFolders.filter(f => allowedSet.has(f));
+ if (effectiveFolders.length === 0) {
+ return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
+ }
+ } else {
+ // No folder filter — restrict to allowed folders only
+ effectiveFolders = allowed;
+ if (effectiveFolders.length === 0) {
+ return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
+ }
+ }
+ }
+
const { total, rows } = getMediaFiles({
- folder: folder || undefined,
- folders: foldersArr,
+ folder: effectiveFolder,
+ folders: effectiveFolders,
type: type || 'all',
sort: sort || 'latest',
offset: offsetNum,
limit: limitNum,
+ dateFrom: dateFrom || undefined,
+ dateTo: dateTo || undefined,
+ minSize: minSize || undefined,
+ maxSize: maxSize || undefined,
+ search: search || undefined,
});
const files = rows.map((r) => {
@@ -197,6 +259,10 @@ router.get('/api/gallery/media/:folder/:filename', (req, res) => {
return res.status(400).json({ error: 'Invalid path' });
}
+ if (!checkFolderAccess(req, folder)) {
+ return res.status(403).json({ error: 'Access denied' });
+ }
+
const filePath = join(MEDIA_PATH, folder, filename);
res.sendFile(filePath, { root: '/' }, (err) => {
if (err && !res.headersSent) {
@@ -225,19 +291,54 @@ async function generateThumb(folder, filename) {
const promise = (async () => {
try {
+ // Skip corrupt/empty files
+ try {
+ const st = statSync(videoPath);
+ if (st.size < 1000) return null;
+ } catch { return null; }
+
if (!existsSync(thumbDir)) mkdirSync(thumbDir, { recursive: true });
+
+ // Check if file has a video stream and get duration
+ let hasVideo = false;
+ let probeFailed = false;
+ let duration = 0;
+ try {
+ const probe = await execFileAsync('ffprobe', [
+ '-v', 'error',
+ '-select_streams', 'v',
+ '-show_entries', 'stream=codec_type',
+ '-show_entries', 'format=duration',
+ '-of', 'json',
+ videoPath,
+ ], { timeout: 15000 });
+ const info = JSON.parse(probe.stdout);
+ hasVideo = info.streams && info.streams.length > 0;
+ duration = parseFloat(info.format?.duration || '0');
+ } catch {
+ probeFailed = true;
+ }
+
+ if (!hasVideo && !probeFailed) return 'audio-only'; // confirmed audio-only, skip
+ if (!hasVideo && probeFailed) return null; // probe failed, count as error
+
+ const seekTime = duration > 1.5 ? '1' : '0';
+
await execFileAsync('ffmpeg', [
- '-ss', '1',
+ '-ss', seekTime,
'-i', videoPath,
'-frames:v', '1',
'-vf', 'scale=320:-1',
+ '-pix_fmt', 'yuvj420p',
'-q:v', '6',
'-y',
+ '-update', '1',
thumbPath,
- ], { timeout: 10000 });
+ ], { timeout: 30000 });
return thumbPath;
} catch (err) {
console.error(`[gallery] thumb failed for ${key}:`, err.message);
+ if (err.stderr) console.error(`[gallery] ffmpeg stderr:`, err.stderr.trim());
return null;
} finally {
thumbInFlight.delete(key);
@@ -248,44 +349,116 @@ async function generateThumb(folder, filename) {
return promise;
}
-// GET /api/gallery/thumb/:folder/:filename — serve or generate a video thumbnail
+async function generateImageThumb(folder, filename) {
+ const imagePath = join(MEDIA_PATH, folder, filename);
+ const { thumbDir, thumbPath } = getThumbPath(folder, filename);
+
+ if (existsSync(thumbPath)) return thumbPath;
+
+ const key = `img:${folder}/${filename}`;
+ if (thumbInFlight.has(key)) return thumbInFlight.get(key);
+
+ const promise = (async () => {
+ try {
+ // Skip corrupt/empty files
+ try {
+ const st = statSync(imagePath);
+ if (st.size < 100) return null;
+ } catch { return null; }
+
+ if (!existsSync(thumbDir)) mkdirSync(thumbDir, { recursive: true });
+ await execFileAsync('ffmpeg', [
+ '-i', imagePath,
+ '-vf', 'scale=480:-1',
+ '-q:v', '4',
+ '-y',
+ '-update', '1',
+ thumbPath,
+ ], { timeout: 30000 });
+ return thumbPath;
+ } catch (err) {
+ console.error(`[gallery] image thumb failed for ${folder}/${filename}:`, err.message);
+ if (err.stderr) console.error(`[gallery] ffmpeg stderr:`, err.stderr.trim());
+ return null;
+ } finally {
+ thumbInFlight.delete(key);
+ }
+ })();
+
+ thumbInFlight.set(key, promise);
+ return promise;
+}
+
+function serveFile(filePath, res) {
+ try {
+ const st = statSync(filePath);
+ res.writeHead(200, {
+ 'Content-Type': 'image/jpeg',
+ 'Content-Length': st.size,
+ 'Cache-Control': 'public, max-age=86400',
+ });
+ createReadStream(filePath).pipe(res);
+ } catch {
+ if (!res.headersSent) {
+ res.set('Cache-Control', 'no-cache');
+ res.status(404).json({ error: 'Not found' });
+ }
+ }
+}
+
+// GET /api/gallery/thumb/:folder/:filename — serve or generate a thumbnail (video or image)
router.get('/api/gallery/thumb/:folder/:filename', async (req, res) => {
const { folder, filename } = req.params;
if (folder.includes('..') || filename.includes('..')) {
return res.status(400).json({ error: 'Invalid path' });
}
+ if (!checkFolderAccess(req, folder)) {
+ return res.status(403).json({ error: 'Access denied' });
+ }
+
const { thumbPath } = getThumbPath(folder, filename);
// Serve cached thumb immediately
if (existsSync(thumbPath)) {
- return res.sendFile(thumbPath, { root: '/' }, (err) => {
- if (err && !res.headersSent) res.status(404).json({ error: 'Not found' });
- });
+ return serveFile(thumbPath, res);
+ }
+
+ // Determine type by extension
+ const ext = filename.toLowerCase().split('.').pop();
+ const isVideo = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv', 'm4v'].includes(ext);
+ const isImage = ['jpg', 'jpeg', 'png', 'webp', 'gif', 'bmp', 'tiff'].includes(ext);
+
+ let result;
+ if (isVideo) {
+ result = await generateThumb(folder, filename);
+ } else if (isImage) {
+ result = await generateImageThumb(folder, filename);
}
- // Generate on-demand
- const result = await generateThumb(folder, filename);
if (result && existsSync(result)) {
- res.sendFile(result, { root: '/' }, (err) => {
- if (err && !res.headersSent) res.status(500).json({ error: 'Failed to serve thumbnail' });
- });
+ serveFile(result, res);
+ } else if (isImage) {
+ // Fallback: serve original image
+ const origPath = join(MEDIA_PATH, folder, filename);
+ serveFile(origPath, res);
} else {
+ res.set('Cache-Control', 'no-cache');
res.status(500).json({ error: 'Thumbnail generation failed' });
}
});
// Bulk thumbnail generation state
-let thumbGenState = { running: false, total: 0, done: 0, errors: 0 };
+let thumbGenState = { running: false, total: 0, done: 0, errors: 0, skipped: 0 };
-// POST /api/gallery/generate-thumbs — bulk generate all video thumbnails
+// POST /api/gallery/generate-thumbs — bulk generate all thumbnails (videos + images)
router.post('/api/gallery/generate-thumbs', (req, res) => {
if (thumbGenState.running) {
return res.json({ status: 'already_running', ...thumbGenState });
}
- // Collect all videos
- const videos = [];
+ // Collect all media needing thumbs
+ const mediaItems = [];
const dirs = readdirSync(MEDIA_PATH, { withFileTypes: true })
.filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.startsWith('_'));
@@ -296,36 +469,39 @@ router.post('/api/gallery/generate-thumbs', (req, res) => {
for (const file of files) {
if (file.startsWith('.')) continue;
const ext = extname(file).toLowerCase();
- if (VIDEO_EXTS.has(ext)) {
+ if (VIDEO_EXTS.has(ext) || IMAGE_EXTS.has(ext)) {
const { thumbPath } = getThumbPath(dir.name, file);
if (!existsSync(thumbPath)) {
- videos.push({ folder: dir.name, filename: file });
+ mediaItems.push({ folder: dir.name, filename: file, type: VIDEO_EXTS.has(ext) ? 'video' : 'image' });
}
}
}
} catch { continue; }
}
- if (videos.length === 0) {
+ if (mediaItems.length === 0) {
return res.json({ status: 'done', total: 0, done: 0, errors: 0, message: 'All thumbnails already exist' });
}
- thumbGenState = { running: true, total: videos.length, done: 0, errors: 0 };
- res.json({ status: 'started', total: videos.length });
+ thumbGenState = { running: true, total: mediaItems.length, done: 0, errors: 0, skipped: 0 };
+ res.json({ status: 'started', total: mediaItems.length });
// Run in background with concurrency limit
(async () => {
- const CONCURRENCY = 3;
+ const CONCURRENCY = 2;
let i = 0;
const next = async () => {
- while (i < videos.length) {
- const { folder, filename } = videos[i++];
- const result = await generateThumb(folder, filename);
- if (result) thumbGenState.done++;
+ while (i < mediaItems.length) {
+ const { folder, filename, type } = mediaItems[i++];
+ const result = type === 'video'
+ ? await generateThumb(folder, filename)
+ : await generateImageThumb(folder, filename);
+ if (result === 'audio-only') thumbGenState.skipped++;
+ else if (result) thumbGenState.done++;
else thumbGenState.errors++;
}
};
- await Promise.all(Array.from({ length: Math.min(CONCURRENCY, videos.length) }, () => next()));
+ await Promise.all(Array.from({ length: Math.min(CONCURRENCY, mediaItems.length) }, () => next()));
thumbGenState.running = false;
})();
});
@@ -351,12 +527,15 @@ function hashFilePartial(filePath, bytes = 65536) {
}
// POST /api/gallery/scan-duplicates — start background duplicate scan
+// Query param: mode=everywhere (default) or mode=same-folder
router.post('/api/gallery/scan-duplicates', (req, res) => {
if (duplicateScanState.running) {
return res.json({ status: 'already_running', ...duplicateScanState });
}
- // Phase 1: group all files by size
+ const mode = req.query.mode || req.body?.mode || 'everywhere';
+
+ // Phase 1: group all files by size (optionally scoped per folder)
const bySize = new Map();
const dirs = readdirSync(MEDIA_PATH, { withFileTypes: true })
.filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.startsWith('_'));
@@ -372,7 +551,8 @@ router.post('/api/gallery/scan-duplicates', (req, res) => {
const filePath = join(dirPath, file);
try {
const stat = statSync(filePath);
- const key = stat.size;
+ // For same-folder mode, scope the size key by folder name
+ const key = mode === 'same-folder' ? `${dir.name}:${stat.size}` : stat.size;
if (!bySize.has(key)) bySize.set(key, []);
bySize.get(key).push({ folder: dir.name, filename: file, type: mediaType, size: stat.size, modified: stat.mtimeMs, filePath });
} catch { continue; }
@@ -440,6 +620,10 @@ router.delete('/api/gallery/media/:folder/:filename', (req, res) => {
return res.status(400).json({ error: 'Invalid path' });
}
+ if (!checkFolderAccess(req, folder)) {
+ return res.status(403).json({ error: 'Access denied' });
+ }
+
const filePath = join(MEDIA_PATH, folder, filename);
if (!existsSync(filePath)) {
return res.status(404).json({ error: 'File not found' });
@@ -484,6 +668,7 @@ router.post('/api/gallery/duplicates/clean', (req, res) => {
try {
if (existsSync(filePath)) {
unlinkSync(filePath);
+ removeMediaFile(file.folder, file.filename);
freed += file.size;
deleted++;
}
diff --git a/server/health.js b/server/health.js
new file mode 100644
index 0000000..2334dd0
--- /dev/null
+++ b/server/health.js
@@ -0,0 +1,74 @@
+import { Router } from 'express';
+import { existsSync, accessSync, constants, statfsSync } from 'fs';
+import { execFileSync } from 'child_process';
+import { getAuthConfig } from './db.js';
+import { getActiveDownloadCount } from './download.js';
+import { getActiveScrapeCount } from './scrape.js';
+
+const router = Router();
+const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
+const WVD_PATH = process.env.WVD_PATH || '/data/cdm/device.wvd';
+
+router.get('/api/health', (req, res) => {
+ const result = {
+ uptime: Math.floor(process.uptime()),
+ sqlite: false,
+ authConfigured: false,
+ mediaPathWritable: false,
+ ffmpegAvailable: false,
+ pythonAvailable: false,
+ wvdPresent: false,
+ diskSpace: null,
+ activeDownloads: 0,
+ activeScrapes: 0,
+ };
+
+ // SQLite check
+ try {
+ getAuthConfig(); // simple query to verify DB works
+ result.sqlite = true;
+ } catch { /* ignore */ }
+
+ // Auth check
+ try {
+ result.authConfigured = !!getAuthConfig();
+ } catch { /* ignore */ }
+
+ // Media path writable
+ try {
+ accessSync(MEDIA_PATH, constants.W_OK);
+ result.mediaPathWritable = true;
+ } catch { /* ignore */ }
+
+ // FFmpeg
+ try {
+ execFileSync('ffmpeg', ['-version'], { timeout: 5000, stdio: 'pipe' });
+ result.ffmpegAvailable = true;
+ } catch { /* ignore */ }
+
+ // Python
+ try {
+ execFileSync('python3', ['--version'], { timeout: 5000, stdio: 'pipe' });
+ result.pythonAvailable = true;
+ } catch { /* ignore */ }
+
+ // WVD file
+ result.wvdPresent = existsSync(WVD_PATH);
+
+ // Disk space
+ try {
+ const stats = statfsSync(MEDIA_PATH);
+ result.diskSpace = {
+ free: stats.bfree * stats.bsize,
+ total: stats.blocks * stats.bsize,
+ };
+ } catch { /* ignore */ }
+
+ // Active jobs
+ result.activeDownloads = getActiveDownloadCount();
+ result.activeScrapes = getActiveScrapeCount();
+
+ res.json(result);
+});
+
+export default router;
diff --git a/server/hls.js b/server/hls.js
index f0ae0cb..de17ead 100644
--- a/server/hls.js
+++ b/server/hls.js
@@ -1,6 +1,6 @@
import { Router } from 'express';
import { join } from 'path';
-import { existsSync } from 'fs';
+import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
import { execFile, spawn } from 'child_process';
import { promisify } from 'util';
import { getSetting } from './db.js';
@@ -8,7 +8,129 @@ import { getSetting } from './db.js';
const execFileAsync = promisify(execFile);
const router = Router();
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
+const CACHE_DIR = join(MEDIA_PATH, '.hls-cache');
const SEGMENT_DURATION = 10;
+const MAX_CONCURRENT_TRANSCODES = 2;
+const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+if (!existsSync(CACHE_DIR)) {
+ mkdirSync(CACHE_DIR, { recursive: true });
+}
+
+// Quality tiers — always transcode for reliable HLS
+const QUALITY_TIERS = {
+ '480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
+ '720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
+ '1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
+};
+
+// Compute output dimensions preserving source aspect ratio
+function fitDimensions(srcW, srcH, maxW, maxH) {
+ if (srcW <= maxW && srcH <= maxH) {
+ let w = srcW, h = srcH;
+ w += w % 2; h += h % 2;
+ return { w, h };
+ }
+ const scale = Math.min(maxW / srcW, maxH / srcH);
+ let w = Math.round(srcW * scale);
+ let h = Math.round(srcH * scale);
+ w += w % 2; h += h % 2;
+ return { w, h };
+}
+
+// --- Hardware acceleration detection (shared state with video-hls.js via same detection) ---
+let hwAccel = null; // 'vaapi' | 'qsv' | null
+let hwDetected = false;
+
+async function detectHwAccel() {
+ if (hwDetected) return hwAccel;
+
+ try {
+ await execFileAsync('ffmpeg', [
+ '-hide_banner',
+ '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+ '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+ '-vf', 'format=nv12,hwupload',
+ '-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
+ ], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
+ hwAccel = 'vaapi';
+ console.log('[hls] Intel VAAPI hardware acceleration available');
+ } catch {
+ try {
+ await execFileAsync('ffmpeg', [
+ '-hide_banner', '-init_hw_device', 'qsv=hw',
+ '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+ '-vf', 'hwupload=extra_hw_frames=64,format=qsv',
+ '-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
+ ], { timeout: 10000 });
+ hwAccel = 'qsv';
+ console.log('[hls] Intel QSV hardware acceleration available');
+ } catch {
+ hwAccel = null;
+ console.log('[hls] No hardware acceleration, using libx264');
+ }
+ }
+
+ hwDetected = true;
+ return hwAccel;
+}
+
+detectHwAccel();
+
+// --- Transcode semaphore ---
+let activeTranscodes = 0;
+const transcodeQueue = [];
+
+function acquireSlot() {
+ return new Promise((resolve) => {
+ if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
+ activeTranscodes++;
+ resolve();
+ } else {
+ transcodeQueue.push(resolve);
+ }
+ });
+}
+
+function releaseSlot() {
+ activeTranscodes--;
+ if (transcodeQueue.length > 0) {
+ activeTranscodes++;
+ transcodeQueue.shift()();
+ }
+}
+
+// --- Cache cleanup (hourly) ---
+function cleanupCache() {
+ try {
+ if (!existsSync(CACHE_DIR)) return;
+ const now = Date.now();
+ const walk = (dir) => {
+ let entries;
+ try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of entries) {
+ const full = join(dir, e.name);
+ if (e.isDirectory()) {
+ walk(full);
+ // Remove empty dirs
+ try { if (readdirSync(full).length === 0) rmSync(full); } catch { /* ignore */ }
+ } else if (e.name.endsWith('.ts')) {
+ try {
+ const st = statSync(full);
+ if (now - st.mtimeMs > CACHE_MAX_AGE_MS) rmSync(full);
+ } catch { /* ignore */ }
+ }
+ }
+ };
+ walk(CACHE_DIR);
+ } catch (err) {
+ console.error('[hls] Cache cleanup error:', err.message);
+ }
+}
+
+setInterval(cleanupCache, 60 * 60 * 1000);
+
+// --- Helpers ---
function isHlsEnabled() {
return (getSetting('hls_enabled') || process.env.HLS_ENABLED) === 'true';
@@ -23,32 +145,89 @@ function validatePath(folder, filename) {
return filePath;
}
-// GET /api/hls/:folder/:filename/master.m3u8
+// Sanitize folder+filename into a cache-safe directory name
+function cacheKey(folder, filename) {
+ return join(folder, filename.replace(/[^a-zA-Z0-9._-]/g, '_'));
+}
+
+async function probeVideo(filePath) {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error',
+ '-show_entries', 'stream=codec_type,width,height',
+ '-show_entries', 'format=duration',
+ '-of', 'json',
+ filePath,
+ ], { timeout: 15000 });
+ const info = JSON.parse(stdout);
+ const videoStream = info.streams?.find(s => s.codec_type === 'video');
+ return {
+ duration: parseFloat(info.format?.duration || '0'),
+ width: videoStream?.width || 0,
+ height: videoStream?.height || 0,
+ };
+}
+
+// --- Master playlist ---
+
router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
- if (!isHlsEnabled()) {
- return res.status(404).json({ error: 'HLS not enabled' });
- }
+ if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
const { folder, filename } = req.params;
const filePath = validatePath(folder, filename);
- if (!filePath) {
- return res.status(400).json({ error: 'Invalid path' });
- }
+ if (!filePath) return res.status(400).json({ error: 'Invalid path' });
try {
- const { stdout } = await execFileAsync('ffprobe', [
- '-v', 'error',
- '-show_entries', 'format=duration',
- '-of', 'csv=p=0',
- filePath,
- ]);
+ const info = await probeVideo(filePath);
+ const srcW = info.width || 1920;
+ const srcH = info.height || 1080;
- const duration = parseFloat(stdout.trim());
- if (isNaN(duration) || duration <= 0) {
+ let playlist = '#EXTM3U\n';
+
+ for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
+ if (tier.maxH <= srcH) {
+ const { w, h } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
+ const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
+ playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
+ playlist += `${name}/playlist.m3u8\n`;
+ }
+ }
+
+ // If source is too small for any tier, add a single tier at source resolution
+ if (!playlist.includes('EXT-X-STREAM-INF')) {
+ const { w, h } = fitDimensions(srcW, srcH, srcW, srcH);
+ playlist += `#EXT-X-STREAM-INF:BANDWIDTH=1596000,RESOLUTION=${w}x${h},NAME="480p"\n`;
+ playlist += `480p/playlist.m3u8\n`;
+ }
+
+ res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.send(playlist);
+ } catch (err) {
+ console.error('[hls] Master playlist error:', err.message);
+ res.status(500).json({ error: 'Failed to generate master playlist' });
+ }
+});
+
+// --- Variant playlist ---
+
+router.get('/api/hls/:folder/:filename/:quality/playlist.m3u8', async (req, res) => {
+ if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
+
+ const { folder, filename, quality } = req.params;
+ if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
+
+ const filePath = validatePath(folder, filename);
+ if (!filePath) return res.status(400).json({ error: 'Invalid path' });
+
+ try {
+ const info = await probeVideo(filePath);
+ const duration = info.duration;
+ if (!duration || duration <= 0) {
return res.status(500).json({ error: 'Could not determine video duration' });
}
const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
+
let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
@@ -63,54 +242,177 @@ router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
playlist += '#EXT-X-ENDLIST\n';
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+ res.setHeader('Cache-Control', 'no-cache');
res.send(playlist);
} catch (err) {
- console.error('[hls] ffprobe error:', err.message);
- res.status(500).json({ error: 'Failed to probe video' });
+ console.error('[hls] Variant playlist error:', err.message);
+ res.status(500).json({ error: 'Failed to generate variant playlist' });
}
});
-// GET /api/hls/:folder/:filename/segment-:index.ts
-router.get('/api/hls/:folder/:filename/segment-:index.ts', (req, res) => {
- if (!isHlsEnabled()) {
- return res.status(404).json({ error: 'HLS not enabled' });
- }
+// --- Segment transcoding ---
+
+router.get('/api/hls/:folder/:filename/:quality/segment-:index.ts', async (req, res) => {
+ if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
+
+ const { folder, filename, quality } = req.params;
+ const segIndex = parseInt(req.params.index, 10);
+
+ if (isNaN(segIndex) || segIndex < 0) return res.status(400).json({ error: 'Invalid segment index' });
+ if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
- const { folder, filename, index } = req.params;
const filePath = validatePath(folder, filename);
- if (!filePath) {
- return res.status(400).json({ error: 'Invalid path' });
+ if (!filePath) return res.status(400).json({ error: 'Invalid path' });
+
+ // Check cache
+ const key = cacheKey(folder, filename);
+ const segCacheDir = join(CACHE_DIR, key, quality);
+ const segCachePath = join(segCacheDir, `segment-${segIndex}.ts`);
+
+ if (existsSync(segCachePath)) {
+ const stat = statSync(segCachePath);
+ res.writeHead(200, {
+ 'Content-Type': 'video/MP2T',
+ 'Content-Length': stat.size,
+ 'Cache-Control': 'public, max-age=3600',
+ });
+ createReadStream(segCachePath).pipe(res);
+ return;
}
- const segIndex = parseInt(index, 10);
- if (isNaN(segIndex) || segIndex < 0) {
- return res.status(400).json({ error: 'Invalid segment index' });
+ await acquireSlot();
+
+ // Double-check cache after acquiring slot
+ if (existsSync(segCachePath)) {
+ releaseSlot();
+ const stat = statSync(segCachePath);
+ res.writeHead(200, {
+ 'Content-Type': 'video/MP2T',
+ 'Content-Length': stat.size,
+ 'Cache-Control': 'public, max-age=3600',
+ });
+ createReadStream(segCachePath).pipe(res);
+ return;
}
- const offset = segIndex * SEGMENT_DURATION;
+ try {
+ const offset = segIndex * SEGMENT_DURATION;
+ const accel = await detectHwAccel();
+ const tier = QUALITY_TIERS[quality];
- const ffmpeg = spawn('ffmpeg', [
- '-ss', String(offset),
- '-i', filePath,
- '-t', String(SEGMENT_DURATION),
- '-c', 'copy',
- '-f', 'mpegts',
- 'pipe:1',
- ], { stdio: ['ignore', 'pipe', 'ignore'] });
+ // Probe source dimensions for aspect-ratio-aware scaling
+ let srcW = 1920, srcH = 1080;
+ try {
+ const info = await probeVideo(filePath);
+ srcW = info.width || 1920;
+ srcH = info.height || 1080;
+ } catch { /* use defaults */ }
- res.setHeader('Content-Type', 'video/MP2T');
- ffmpeg.stdout.pipe(res);
+ const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
- req.on('close', () => {
- ffmpeg.kill('SIGKILL');
- });
-
- ffmpeg.on('error', (err) => {
- console.error('[hls] ffmpeg error:', err.message);
- if (!res.headersSent) {
- res.status(500).json({ error: 'Transcoding failed' });
+ let ffmpegArgs;
+ if (accel === 'vaapi') {
+ ffmpegArgs = [
+ '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+ '-filter_hw_device', 'va',
+ '-ss', String(offset),
+ '-i', filePath,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
+ '-c:v', 'h264_vaapi',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
+ } else if (accel === 'qsv') {
+ ffmpegArgs = [
+ '-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
+ '-ss', String(offset),
+ '-i', filePath,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `scale_qsv=w=${outW}:h=${outH}`,
+ '-c:v', 'h264_qsv',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
+ } else {
+ ffmpegArgs = [
+ '-ss', String(offset),
+ '-i', filePath,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `scale=${outW}:${outH}`,
+ '-c:v', 'libx264', '-preset', 'veryfast',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
}
- });
+
+ const spawnEnv = accel === 'vaapi'
+ ? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
+ : undefined;
+
+ const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ ...(spawnEnv && { env: spawnEnv }),
+ });
+
+ res.setHeader('Content-Type', 'video/MP2T');
+ res.setHeader('Cache-Control', 'public, max-age=3600');
+
+ const cacheChunks = [];
+ let aborted = false;
+
+ ffmpeg.stdout.on('data', (chunk) => {
+ cacheChunks.push(chunk);
+ if (!res.destroyed) res.write(chunk);
+ });
+
+ req.on('close', () => {
+ if (!ffmpeg.killed) {
+ aborted = true;
+ ffmpeg.kill('SIGKILL');
+ releaseSlot();
+ }
+ });
+
+ ffmpeg.on('close', (code) => {
+ if (aborted) return;
+ releaseSlot();
+
+ if (code === 0 && cacheChunks.length > 0) {
+ try {
+ if (!existsSync(segCacheDir)) mkdirSync(segCacheDir, { recursive: true });
+ const ws = createWriteStream(segCachePath);
+ for (const c of cacheChunks) ws.write(c);
+ ws.end();
+ } catch { /* ignore */ }
+ }
+
+ if (!res.destroyed) res.end();
+ });
+
+ ffmpeg.on('error', (err) => {
+ if (!aborted) releaseSlot();
+ console.error('[hls] ffmpeg error:', err.message);
+ if (!res.headersSent) {
+ res.status(500).json({ error: 'Transcoding failed' });
+ }
+ });
+ } catch (err) {
+ releaseSlot();
+ console.error('[hls] Segment error:', err.message);
+ if (!res.headersSent) {
+ res.status(500).json({ error: err.message });
+ }
+ }
});
export default router;
diff --git a/server/index.js b/server/index.js
index f29b885..8437497 100644
--- a/server/index.js
+++ b/server/index.js
@@ -1,19 +1,28 @@
import express from 'express';
import https from 'https';
import cors from 'cors';
+import cookieParser from 'cookie-parser';
import { existsSync, readFileSync, mkdirSync, writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import { initRules } from './signing.js';
+import authRouter, { requireAuth, checkRoutePermission } from './auth.js';
import proxyRouter from './proxy.js';
import downloadRouter from './download.js';
import galleryRouter from './gallery.js';
import hlsRouter from './hls.js';
import settingsRouter from './settings.js';
import scrapeRouter from './scrape.js';
+import flareSolverrRouter from './flaresolverr.js';
import drmStreamRouter from './drm-stream.js';
+import healthRouter from './health.js';
+import dashboardRouter from './dashboard.js';
+import videosRouter from './videos.js';
+import videoHlsRouter from './video-hls.js';
+import mediaApiRouter from './media-api.js';
import { scanMediaFiles } from './gallery.js';
+import { startScheduler } from './scheduler.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -23,11 +32,28 @@ const PORT = process.env.PORT || 3001;
const HTTPS_PORT = process.env.HTTPS_PORT || 3443;
app.use(cors());
+app.use(cookieParser());
// Parse DRM license request bodies as raw binary BEFORE global JSON parser
// (express.json can interfere with reading the raw body stream)
app.use('/api/drm-license', express.raw({ type: '*/*', limit: '1mb' }));
app.use(express.json());
+// Auth routes (public endpoints like login/setup must be before requireAuth)
+app.use(authRouter);
+
+// Apply auth middleware globally (after auth routes)
+app.use('/api', (req, res, next) => {
+ // Skip auth for app-auth public endpoints
+ if (req.path.startsWith('/app-auth/')) return next();
+ // Skip auth for internal DRM license requests from pywidevine subprocess
+ if (req.path.startsWith('/drm-license') && ['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(req.ip)) return next();
+ requireAuth(req, res, next);
+});
+app.use('/api', (req, res, next) => {
+ if (req.path.startsWith('/app-auth/')) return next();
+ checkRoutePermission(req, res, next);
+});
+
// API routes
app.use(proxyRouter);
app.use(downloadRouter);
@@ -35,7 +61,13 @@ app.use(galleryRouter);
app.use(hlsRouter);
app.use(settingsRouter);
app.use(scrapeRouter);
+app.use(flareSolverrRouter);
app.use(drmStreamRouter);
+app.use(healthRouter);
+app.use(dashboardRouter);
+app.use(videosRouter);
+app.use(videoHlsRouter);
+app.use(mediaApiRouter);
// Serve static client build in production
const clientDist = join(__dirname, '..', 'client', 'dist');
@@ -70,6 +102,8 @@ async function start() {
console.error('[server] Media scan failed:', err.message);
}
});
+ // Start auto-download/scrape scheduler
+ startScheduler();
});
// Start HTTPS server for DRM/EME support (requires secure context)
diff --git a/server/login_helper.py b/server/login_helper.py
new file mode 100644
index 0000000..3a09996
--- /dev/null
+++ b/server/login_helper.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+Login helper using undetected_chromedriver to bypass Cloudflare Turnstile.
+Runs Chrome in headed mode with Xvfb (virtual display) so Turnstile sees a real browser.
+
+Usage:
+ xvfb-run python3 login_helper.py
+
+Outputs JSON to stdout:
+ {"ok": true, "cookies": "name=val; name2=val2", "url": ""}
+ {"ok": false, "error": "reason"}
+"""
+import sys
+import json
+import time
+import os
+import shutil
+
+
+def main():
+ if len(sys.argv) < 4:
+ print(json.dumps({"ok": False, "error": "Usage: login_helper.py "}))
+ sys.exit(1)
+
+ login_url = sys.argv[1]
+ username = sys.argv[2]
+ password = sys.argv[3]
+
+ try:
+ import undetected_chromedriver as uc
+ from selenium.webdriver.common.by import By
+ from selenium.webdriver.support.ui import WebDriverWait
+ from selenium.webdriver.support import expected_conditions as EC
+ except ImportError as e:
+ print(json.dumps({"ok": False, "error": f"Missing dependency: {e}"}))
+ sys.exit(1)
+
+ driver = None
+ try:
+ # Find chromium binary
+ chromium_path = os.environ.get('CHROMIUM_PATH', '')
+ if not chromium_path or not os.path.exists(chromium_path):
+ for p in ['/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/lib/chromium/chromium']:
+ if os.path.exists(p):
+ chromium_path = p
+ break
+
+ # Find system chromedriver (Alpine: chromium-chromedriver package)
+ chromedriver_path = None
+ for p in ['/usr/bin/chromedriver', '/usr/lib/chromium/chromedriver']:
+ if os.path.exists(p):
+ chromedriver_path = p
+ break
+
+ # Get chromium version for undetected_chromedriver
+ version_main = None
+ try:
+ import subprocess
+ result = subprocess.run([chromium_path, '--version'], capture_output=True, text=True, timeout=5)
+ # e.g. "Chromium 131.0.6778.139" or "Chromium 131.0.6778.139 Alpine Linux"
+ parts = result.stdout.strip().split()
+ ver_str = None
+ for part in parts:
+ if '.' in part and part[0].isdigit():
+ ver_str = part
+ break
+ if ver_str:
+ version_main = int(ver_str.split('.')[0])
+ log(f"Chromium version: {ver_str} (major: {version_main})")
+ except Exception as e:
+ log(f"Could not detect chromium version: {e}")
+
+ options = uc.ChromeOptions()
+ options.binary_location = chromium_path
+ options.add_argument('--no-sandbox')
+ options.add_argument('--disable-dev-shm-usage')
+ options.add_argument('--disable-gpu')
+ options.add_argument('--window-size=1920,1080')
+
+ log(f"Chromium: {chromium_path}")
+ log(f"Chromedriver: {chromedriver_path}")
+
+ # Create the driver
+ # Use system chromedriver to avoid downloading (fails on Alpine/musl)
+ driver = uc.Chrome(
+ options=options,
+ driver_executable_path=chromedriver_path,
+ headless=False,
+ version_main=version_main,
+ )
+ driver.set_window_size(1920, 1080)
+
+ log(f"Navigating to {login_url}...")
+ driver.get(login_url)
+
+ # Wait for DDoS-Guard to solve and login form to appear
+ log("Waiting for login form (DDoS-Guard solving)...")
+ WebDriverWait(driver, 60).until(
+ EC.presence_of_element_located((By.CSS_SELECTOR, 'input[name="login"]'))
+ )
+ log("Login form found")
+
+ # Wait for Turnstile to auto-solve (should work in undetected headed mode)
+ log("Waiting for Turnstile to solve...")
+ turnstile_token = ""
+ for i in range(45):
+ try:
+ el = driver.find_element(By.CSS_SELECTOR, 'input[name="cf-turnstile-response"]')
+ val = el.get_attribute("value")
+ if val:
+ turnstile_token = val
+ break
+ except Exception:
+ pass
+ time.sleep(1)
+ if i % 10 == 9:
+ log(f"Still waiting for Turnstile... ({i+1}s)")
+
+ if turnstile_token:
+ log(f"Turnstile solved (token: {turnstile_token[:20]}...)")
+ else:
+ log("Warning: Turnstile token not found after 45s — attempting login anyway")
+
+ # Fill the login form
+ log("Filling login form...")
+ login_input = driver.find_element(By.CSS_SELECTOR, 'input[name="login"]')
+ login_input.clear()
+ # Type slowly to appear human
+ for ch in username:
+ login_input.send_keys(ch)
+ time.sleep(0.03)
+
+ pass_input = driver.find_element(By.CSS_SELECTOR, 'input[name="password"]')
+ pass_input.clear()
+ for ch in password:
+ pass_input.send_keys(ch)
+ time.sleep(0.03)
+
+ # Check remember checkbox
+ try:
+ remember = driver.find_element(By.CSS_SELECTOR, 'input[name="remember"]')
+ if not remember.is_selected():
+ driver.execute_script("arguments[0].checked = true;", remember)
+ except Exception:
+ pass
+
+ # Submit form
+ log("Submitting login form...")
+ try:
+ submit_btn = driver.find_element(By.CSS_SELECTOR,
+ 'button[type="submit"], input[type="submit"], .button--primary')
+ submit_btn.click()
+ except Exception:
+ driver.execute_script("""
+ var form = document.querySelector('form.block-body') ||
+ document.querySelector('form[action*="login"]');
+ if (form) form.submit();
+ """)
+
+ # Wait for navigation after submit
+ log("Waiting for redirect...")
+ time.sleep(5)
+ try:
+ WebDriverWait(driver, 15).until(
+ lambda d: d.execute_script("return document.readyState") == "complete"
+ )
+ except Exception:
+ pass
+
+ final_url = driver.current_url
+ log(f"After submit: {final_url}")
+
+ # Extract cookies
+ cookies = driver.get_cookies()
+ cookie_str = "; ".join(f"{c['name']}={c['value']}" for c in cookies)
+
+ # Check for login success
+ has_user_cookie = any(c['name'] in ('xf_user', 'ogaddgmetaprof_user') for c in cookies)
+
+ if not has_user_cookie:
+ # Check for error message
+ error_msg = "Login failed — no user cookie returned"
+ try:
+ error_el = driver.find_element(By.CSS_SELECTOR, '.blockMessage--error')
+ error_msg = error_el.text.strip()
+ except Exception:
+ pass
+ # Also dump all cookie names for debugging
+ cookie_names = [c['name'] for c in cookies]
+ log(f"Cookie names: {cookie_names}")
+ log(f"Error: {error_msg}")
+ print(json.dumps({"ok": False, "error": error_msg, "url": final_url}))
+ sys.exit(1)
+
+ log(f"Login successful — {len(cookies)} cookies")
+ print(json.dumps({"ok": True, "cookies": cookie_str, "url": final_url}))
+
+ except Exception as e:
+ log(f"Fatal error: {e}")
+ print(json.dumps({"ok": False, "error": str(e)}))
+ sys.exit(1)
+ finally:
+ if driver:
+ try:
+ driver.quit()
+ except Exception:
+ pass
+
+
+def log(msg):
+ """Log to stderr so it doesn't interfere with JSON stdout."""
+ print(f"[login_helper] {msg}", file=sys.stderr, flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/media-api.js b/server/media-api.js
new file mode 100644
index 0000000..9be955a
--- /dev/null
+++ b/server/media-api.js
@@ -0,0 +1,66 @@
+import { Router } from 'express';
+import { getMediaFiles, getSetting, getUserFolderAccess } from './db.js';
+
+const router = Router();
+const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
+
+// GET /api/media?users=folder1,folder2&type=video,image
+router.get('/api/media', (req, res) => {
+ const { users, type } = req.query;
+
+ if (!users) return res.status(400).json({ error: 'users parameter is required' });
+ if (!type) return res.status(400).json({ error: 'type parameter is required' });
+
+ const folders = users.split(',').map(u => u.trim()).filter(Boolean);
+ const types = type.split(',').map(t => t.trim().toLowerCase()).filter(Boolean);
+
+ if (folders.length === 0) return res.status(400).json({ error: 'at least one user/folder is required' });
+
+ const validTypes = ['video', 'image'];
+ const invalid = types.find(t => !validTypes.includes(t));
+ if (invalid) return res.status(400).json({ error: `invalid type: ${invalid}. Must be video, image, or both` });
+
+ // Enforce folder access for non-admin users
+ if (req.user && req.user.role !== 'admin') {
+ const allowed = getUserFolderAccess(req.user.id);
+ if (allowed.length > 0) {
+ const denied = folders.filter(f => !allowed.includes(f));
+ if (denied.length > 0) {
+ return res.status(403).json({ error: `access denied to folders: ${denied.join(', ')}` });
+ }
+ }
+ }
+
+ // If both types requested, query all; otherwise query the single type
+ const typeFilter = types.length === 2 ? 'all' : types[0];
+
+ const { rows } = getMediaFiles({
+ folders,
+ type: typeFilter,
+ offset: 0,
+ limit: 999999999,
+ });
+
+ const hlsEnabled = getSetting('hls_enabled') === 'true' || process.env.HLS_ENABLED === 'true';
+
+ const results = rows.map(r => {
+ const item = {
+ folder: r.folder,
+ path: `${MEDIA_PATH}/${r.folder}/${r.filename}`,
+ type: r.type,
+ };
+ if (r.type === 'video' && hlsEnabled) {
+ item.hlsUrl = `/api/hls/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}/master.m3u8`;
+ } else if (r.type === 'video') {
+ item.mediaUrl = `/api/gallery/media/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}`;
+ }
+ if (r.type === 'image') {
+ item.mediaUrl = `/api/gallery/media/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}`;
+ }
+ return item;
+ });
+
+ res.json(results);
+});
+
+export default router;
diff --git a/server/package-lock.json b/server/package-lock.json
index a77babb..d118ac8 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -8,10 +8,15 @@
"name": "ofapp-server",
"version": "1.0.0",
"dependencies": {
+ "bcryptjs": "^3.0.3",
"better-sqlite3": "^11.0.0",
"cheerio": "^1.2.0",
+ "cookie-parser": "^1.4.7",
"cors": "^2.8.5",
"express": "^4.21.0",
+ "jsonwebtoken": "^9.0.3",
+ "megajs": "^1.3.9",
+ "multer": "^2.0.2",
"node-fetch": "^3.3.2"
}
},
@@ -28,6 +33,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/append-field": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
+ "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
+ "license": "MIT"
+ },
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -54,6 +65,15 @@
],
"license": "MIT"
},
+ "node_modules/bcryptjs": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz",
+ "integrity": "sha512-GlF5wPWnSa/X5LKM1o0wz0suXIINz1iHRLvTS+sLyi7XPbe5ycmYI3DlZqVGZZtDgl4DmasFg7gOB3JYbphV5g==",
+ "license": "BSD-3-Clause",
+ "bin": {
+ "bcrypt": "bin/bcrypt"
+ }
+ },
"node_modules/better-sqlite3": {
"version": "11.10.0",
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",
@@ -139,6 +159,29 @@
"ieee754": "^1.1.13"
}
},
+ "node_modules/buffer-equal-constant-time": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
+ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
+ "license": "BSD-3-Clause"
+ },
+ "node_modules/buffer-from": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+ "license": "MIT"
+ },
+ "node_modules/busboy": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+ "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+ "dependencies": {
+ "streamsearch": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=10.16.0"
+ }
+ },
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -225,6 +268,21 @@
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"license": "ISC"
},
+ "node_modules/concat-stream": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+ "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+ "engines": [
+ "node >= 6.0"
+ ],
+ "license": "MIT",
+ "dependencies": {
+ "buffer-from": "^1.0.0",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.0.2",
+ "typedarray": "^0.0.6"
+ }
+ },
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -255,6 +313,25 @@
"node": ">= 0.6"
}
},
+ "node_modules/cookie-parser": {
+ "version": "1.4.7",
+ "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
+ "integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
+ "license": "MIT",
+ "dependencies": {
+ "cookie": "0.7.2",
+ "cookie-signature": "1.0.6"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/cookie-parser/node_modules/cookie-signature": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==",
+ "license": "MIT"
+ },
"node_modules/cookie-signature": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz",
@@ -445,6 +522,27 @@
"node": ">= 0.4"
}
},
+ "node_modules/duplexify": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",
+ "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==",
+ "license": "MIT",
+ "dependencies": {
+ "end-of-stream": "^1.4.1",
+ "inherits": "^2.0.3",
+ "readable-stream": "^3.1.1",
+ "stream-shift": "^1.0.2"
+ }
+ },
+ "node_modules/ecdsa-sig-formatter": {
+ "version": "1.0.11",
+ "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
+ "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "safe-buffer": "^5.0.1"
+ }
+ },
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -881,6 +979,97 @@
"node": ">= 0.10"
}
},
+ "node_modules/jsonwebtoken": {
+ "version": "9.0.3",
+ "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz",
+ "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==",
+ "license": "MIT",
+ "dependencies": {
+ "jws": "^4.0.1",
+ "lodash.includes": "^4.3.0",
+ "lodash.isboolean": "^3.0.3",
+ "lodash.isinteger": "^4.0.4",
+ "lodash.isnumber": "^3.0.3",
+ "lodash.isplainobject": "^4.0.6",
+ "lodash.isstring": "^4.0.1",
+ "lodash.once": "^4.0.0",
+ "ms": "^2.1.1",
+ "semver": "^7.5.4"
+ },
+ "engines": {
+ "node": ">=12",
+ "npm": ">=6"
+ }
+ },
+ "node_modules/jsonwebtoken/node_modules/ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+ "license": "MIT"
+ },
+ "node_modules/jwa": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",
+ "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==",
+ "license": "MIT",
+ "dependencies": {
+ "buffer-equal-constant-time": "^1.0.1",
+ "ecdsa-sig-formatter": "1.0.11",
+ "safe-buffer": "^5.0.1"
+ }
+ },
+ "node_modules/jws": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
+ "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==",
+ "license": "MIT",
+ "dependencies": {
+ "jwa": "^2.0.1",
+ "safe-buffer": "^5.0.1"
+ }
+ },
+ "node_modules/lodash.includes": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
+ "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.isboolean": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
+ "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.isinteger": {
+ "version": "4.0.4",
+ "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
+ "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.isnumber": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
+ "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.isplainobject": {
+ "version": "4.0.6",
+ "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
+ "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.isstring": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
+ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
+ "license": "MIT"
+ },
+ "node_modules/lodash.once": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
+ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
+ "license": "MIT"
+ },
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@ -899,6 +1088,16 @@
"node": ">= 0.6"
}
},
+ "node_modules/megajs": {
+ "version": "1.3.9",
+ "resolved": "https://registry.npmjs.org/megajs/-/megajs-1.3.9.tgz",
+ "integrity": "sha512-91GGJbUfUu9z/KFORHcn4bugVILWcGahaoy07Q7M5GLzT6zOsrpusxkjEvEys9XCXbxntg0v+f2JN6sITrEkPQ==",
+ "license": "MIT",
+ "dependencies": {
+ "pumpify": "^2.0.1",
+ "stream-skip": "^1.0.3"
+ }
+ },
"node_modules/merge-descriptors": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
@@ -971,6 +1170,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/mkdirp": {
+ "version": "0.5.6",
+ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
+ "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
+ "license": "MIT",
+ "dependencies": {
+ "minimist": "^1.2.6"
+ },
+ "bin": {
+ "mkdirp": "bin/cmd.js"
+ }
+ },
"node_modules/mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
@@ -983,6 +1194,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
+ "node_modules/multer": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
+ "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
+ "license": "MIT",
+ "dependencies": {
+ "append-field": "^1.0.0",
+ "busboy": "^1.6.0",
+ "concat-stream": "^2.0.0",
+ "mkdirp": "^0.5.6",
+ "object-assign": "^4.1.1",
+ "type-is": "^1.6.18",
+ "xtend": "^4.0.2"
+ },
+ "engines": {
+ "node": ">= 10.16.0"
+ }
+ },
"node_modules/napi-build-utils": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
@@ -1215,6 +1444,17 @@
"once": "^1.3.1"
}
},
+ "node_modules/pumpify": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz",
+ "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==",
+ "license": "MIT",
+ "dependencies": {
+ "duplexify": "^4.1.1",
+ "inherits": "^2.0.3",
+ "pump": "^3.0.0"
+ }
+ },
"node_modules/qs": {
"version": "6.14.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz",
@@ -1498,6 +1738,26 @@
"node": ">= 0.8"
}
},
+ "node_modules/stream-shift": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",
+ "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==",
+ "license": "MIT"
+ },
+ "node_modules/stream-skip": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/stream-skip/-/stream-skip-1.0.3.tgz",
+ "integrity": "sha512-2rB0uBiOnYSQwJxJ3wZLher+fz0yyXQxKuKnVTsidHmkqvC8rWZ2AbX50ZVdz7fsL6zkYkqaN/pPD0RldKIbpQ==",
+ "license": "MIT"
+ },
+ "node_modules/streamsearch": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+ "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+ "engines": {
+ "node": ">=10.0.0"
+ }
+ },
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -1578,6 +1838,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/typedarray": {
+ "version": "0.0.6",
+ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+ "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
+ "license": "MIT"
+ },
"node_modules/undici": {
"version": "7.22.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz",
@@ -1668,6 +1934,15 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
+ },
+ "node_modules/xtend": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4"
+ }
}
}
}
diff --git a/server/package.json b/server/package.json
index 8454292..e516918 100644
--- a/server/package.json
+++ b/server/package.json
@@ -7,10 +7,15 @@
"dev": "node --watch index.js"
},
"dependencies": {
+ "bcryptjs": "^3.0.3",
"better-sqlite3": "^11.0.0",
"cheerio": "^1.2.0",
+ "cookie-parser": "^1.4.7",
"cors": "^2.8.5",
"express": "^4.21.0",
+ "jsonwebtoken": "^9.0.3",
+ "megajs": "^1.3.9",
+ "multer": "^2.0.2",
"node-fetch": "^3.3.2"
}
}
diff --git a/server/proxy.js b/server/proxy.js
index 3fa79d1..c4e81fc 100644
--- a/server/proxy.js
+++ b/server/proxy.js
@@ -55,6 +55,23 @@ async function proxyGet(ofPath, authConfig) {
return { status: res.status, data };
}
+// GET /api/auth/check — validate auth by calling OF API
+router.get('/api/auth/check', async (req, res) => {
+ try {
+ const authConfig = getAuthConfig();
+ if (!authConfig) {
+ return res.json({ valid: false, error: 'No auth configured' });
+ }
+ const { status, data } = await proxyGet('/api2/v2/users/me', authConfig);
+ if (status === 200 && data && data.id) {
+ return res.json({ valid: true, user: { id: data.id, name: data.name, username: data.username } });
+ }
+ return res.json({ valid: false, error: data?.error?.message || data?.message || `HTTP ${status}` });
+ } catch (err) {
+ return res.json({ valid: false, error: err.message });
+ }
+});
+
// GET /api/auth
router.get('/api/auth', (req, res) => {
const config = getAuthConfig();
diff --git a/server/scheduler.js b/server/scheduler.js
new file mode 100644
index 0000000..c39e807
--- /dev/null
+++ b/server/scheduler.js
@@ -0,0 +1,95 @@
+import { getAuthConfig, getAutoDownloadUsers, updateAutoDownloadLastRun, getAutoScrapeJobs, updateAutoScrapeLastRun } from './db.js';
+import { runDownload } from './download.js';
+import { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, createJob } from './scrape.js';
+
+const INTERVAL = 12 * 60 * 60 * 1000; // 12 hours
+const STARTUP_DELAY = 30 * 1000; // 30 seconds
+
+async function runAutoDownloads() {
+ const users = getAutoDownloadUsers();
+ if (users.length === 0) return;
+
+ const authConfig = getAuthConfig();
+ if (!authConfig) {
+ console.log('[scheduler] Skipping auto-downloads: no auth config');
+ return;
+ }
+
+ console.log(`[scheduler] Starting auto-downloads for ${users.length} user(s)`);
+
+ for (const user of users) {
+ try {
+ console.log(`[scheduler] Downloading ${user.username} (${user.user_id})`);
+ await runDownload(user.user_id, authConfig, null, true, user.username);
+ updateAutoDownloadLastRun(user.user_id);
+ console.log(`[scheduler] Completed download for ${user.username}`);
+ } catch (err) {
+ console.error(`[scheduler] Error downloading ${user.username}:`, err.message);
+ }
+ }
+
+ console.log('[scheduler] Auto-downloads complete');
+}
+
+async function runAutoScrapes() {
+ const jobs = getAutoScrapeJobs();
+ if (jobs.length === 0) return;
+
+ console.log(`[scheduler] Starting auto-scrapes for ${jobs.length} job(s)`);
+
+ for (const savedJob of jobs) {
+ try {
+ const config = JSON.parse(savedJob.config);
+ config.folderName = savedJob.folder_name;
+ const job = createJob(savedJob.type, config);
+
+ console.log(`[scheduler] Running ${savedJob.type} scrape for ${savedJob.folder_name}`);
+
+ if (savedJob.type === 'forum') {
+ await runForumScrape(job);
+ } else if (savedJob.type === 'coomer') {
+ await runCoomerScrape(job);
+ } else if (savedJob.type === 'medialink') {
+ await runMediaLinkScrape(job);
+ } else if (savedJob.type === 'mega') {
+ await runMegaScrape(job);
+ }
+
+ updateAutoScrapeLastRun(savedJob.id);
+ console.log(`[scheduler] Completed scrape for ${savedJob.folder_name}`);
+ } catch (err) {
+ console.error(`[scheduler] Error scraping ${savedJob.folder_name}:`, err.message);
+ }
+ }
+
+ console.log('[scheduler] Auto-scrapes complete');
+}
+
+async function runAll() {
+ try {
+ await runAutoDownloads();
+ } catch (err) {
+ console.error('[scheduler] Auto-download batch failed:', err.message);
+ }
+ try {
+ await runAutoScrapes();
+ } catch (err) {
+ console.error('[scheduler] Auto-scrape batch failed:', err.message);
+ }
+}
+
+export function startScheduler() {
+ // Run once shortly after startup
+ setTimeout(() => {
+ console.log('[scheduler] Running initial auto-download/scrape check');
+ runAll();
+ }, STARTUP_DELAY);
+
+ // Then every 12 hours
+ setInterval(() => {
+ console.log('[scheduler] Running scheduled auto-download/scrape');
+ runAll();
+ }, INTERVAL);
+
+ console.log('[scheduler] Scheduler started (interval: 12h)');
+}
diff --git a/server/scrape.js b/server/scrape.js
index 4b70bcb..c7e2c74 100644
--- a/server/scrape.js
+++ b/server/scrape.js
@@ -1,9 +1,14 @@
import { Router } from 'express';
import { mkdirSync } from 'fs';
import { join } from 'path';
-import { scrapeForumPage, getPageUrl, detectMaxPage } from './scrapers/forum.js';
-import { parseUserUrl, fetchAllPosts, downloadFiles } from './scrapers/coomer.js';
-import { parseMediaUrl, fetchAllMedia, downloadMedia } from './scrapers/medialink.js';
+import { scrapeForumPage, getPageUrl, detectMaxPage, CookieExpiredError } from './scrapers/forum.js';
+import { refreshForumCookies } from './flaresolverr.js';
+import { parseUserUrl, fetchAllPosts, fetchSearchPosts, downloadFiles } from './scrapers/coomer.js';
+import { parseMediaUrl, fetchAllMedia, fetchAllMediaFromHtml, downloadMedia } from './scrapers/medialink.js';
+import { parseMegaUrl, listAllFiles, downloadMegaFiles } from './scrapers/mega.js';
+import { runYtdlp } from './scrapers/ytdlp.js';
+import { parseLeakGalleryUrl, fetchAllMedia as fetchLeakGalleryMedia, downloadMedia as downloadLeakGalleryMedia } from './scrapers/leakgallery.js';
+import { getAutoScrapeJobs, addAutoScrapeJob, removeAutoScrapeJob, getForumSites, getForumSiteById, createForumSite, updateForumSite, deleteForumSite } from './db.js';
const router = Router();
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
@@ -56,6 +61,8 @@ function jobToJson(job) {
progress: job.progress,
running: job.running,
cancelled: job.cancelled,
+ paused: job.paused || false,
+ resumeAt: job.resumeAt || null,
folderName: job.folderName,
startedAt: job.startedAt,
completedAt: job.completedAt,
@@ -66,13 +73,38 @@ function jobToJson(job) {
// --- Forum Scrape ---
async function runForumScrape(job) {
- const { url, startPage, endPage, delay, folderName } = job.config;
+ let { url, startPage, endPage, delay, folderName, siteId, lastPageOnly } = job.config;
+ let { cookies } = job.config;
+
+ // Load cookies from forum site record if siteId provided and no cookies passed
+ if (!cookies && siteId) {
+ const site = getForumSiteById(siteId);
+ if (site && site.cookies) {
+ cookies = site.cookies;
+ job.config.cookies = cookies;
+ addLog(job, `Loaded cookies from forum site: ${site.name}`);
+ }
+ }
+
const outputDir = join(MEDIA_PATH, folderName);
mkdirSync(outputDir, { recursive: true });
const downloadedSet = new Set();
let totalImages = 0;
+ // If lastPageOnly, detect the last page and only scrape that
+ if (lastPageOnly) {
+ addLog(job, 'Detecting last page...');
+ const maxPage = await detectMaxPage(url, (msg) => addLog(job, msg), cookies);
+ if (maxPage) {
+ startPage = maxPage;
+ endPage = maxPage;
+ addLog(job, `Last page detected: ${maxPage}`);
+ } else {
+ addLog(job, 'Could not detect last page — falling back to page range');
+ }
+ }
+
addLog(job, `Starting forum scrape: pages ${startPage}-${endPage}`);
addLog(job, `Output: ${outputDir}`);
@@ -88,7 +120,31 @@ async function runForumScrape(job) {
const pageUrl = getPageUrl(url, page);
addLog(job, `--- Page ${page}/${endPage} ---`);
- const count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg));
+ let count;
+ try {
+ count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
+ } catch (err) {
+ if (err instanceof CookieExpiredError && siteId) {
+ addLog(job, `Cookie expired (HTTP ${err.statusCode}) — attempting auto-refresh via FlareSolverr...`);
+ try {
+ cookies = await refreshForumCookies(siteId);
+ job.config.cookies = cookies;
+ addLog(job, 'Cookies refreshed successfully — retrying page...');
+ count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
+ } catch (refreshErr) {
+ addLog(job, `Cookie refresh failed: ${refreshErr.message}`);
+ addLog(job, 'Stopping scrape — fix credentials or refresh cookies manually');
+ break;
+ }
+ } else if (err instanceof CookieExpiredError) {
+ addLog(job, `Cookie expired (HTTP ${err.statusCode}) — no siteId configured for auto-refresh`);
+ addLog(job, 'Stopping scrape — refresh cookies manually and try again');
+ break;
+ } else {
+ throw err;
+ }
+ }
+
totalImages += count;
job.progress.completed = page - startPage + 1;
@@ -102,7 +158,7 @@ async function runForumScrape(job) {
} finally {
job.running = false;
job.completedAt = new Date().toISOString();
- addLog(job, `Done! ${totalImages} images saved to ${folderName}/`);
+ addLog(job, `Done! ${totalImages} files saved to ${folderName}/`);
pruneCompleted();
}
}
@@ -118,15 +174,24 @@ async function runCoomerScrape(job) {
addLog(job, `Pages: ${pages}, Workers: ${workers}`);
try {
- const { base, service, userId } = parseUserUrl(url);
- addLog(job, `Site: ${base}, Service: ${service}, User: ${userId}`);
+ const parsed = parseUserUrl(url);
+ let files;
- // Phase 1: Collect files
- addLog(job, `Fetching up to ${pages} pages...`);
- const files = await fetchAllPosts(base, service, userId, pages,
- (msg) => addLog(job, msg),
- () => job.cancelled
- );
+ if (parsed.mode === 'search') {
+ addLog(job, `Site: ${parsed.base}, Search: "${parsed.query}"`);
+ addLog(job, `Fetching up to ${pages} pages...`);
+ files = await fetchSearchPosts(parsed.base, parsed.query, pages,
+ (msg) => addLog(job, msg),
+ () => job.cancelled
+ );
+ } else {
+ addLog(job, `Site: ${parsed.base}, Service: ${parsed.service}, User: ${parsed.userId}`);
+ addLog(job, `Fetching up to ${pages} pages...`);
+ files = await fetchAllPosts(parsed.base, parsed.service, parsed.userId, pages,
+ (msg) => addLog(job, msg),
+ () => job.cancelled
+ );
+ }
if (job.cancelled) {
addLog(job, 'Cancelled by user');
@@ -174,12 +239,170 @@ async function runMediaLinkScrape(job) {
addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
try {
- const { base, userId } = parseMediaUrl(url);
- addLog(job, `Site: ${base}, User ID: ${userId}`);
+ const { base, userId, mode } = parseMediaUrl(url);
+ addLog(job, `Site: ${base}, ${mode === 'html' ? 'Slug' : 'User ID'}: ${userId} (${mode} mode)`);
- // Phase 1: Collect all media via JSON API
+ // Phase 1: Collect all media
+ let items;
+ if (mode === 'html') {
+ addLog(job, `Fetching up to ${pages} pages via HTML scraping...`);
+ items = await fetchAllMediaFromHtml(base, userId, pages, delay,
+ (msg) => addLog(job, msg),
+ () => job.cancelled
+ );
+ } else {
+ addLog(job, `Fetching up to ${pages} pages from API...`);
+ items = await fetchAllMedia(base, userId, pages, delay,
+ (msg) => addLog(job, msg),
+ () => job.cancelled
+ );
+ }
+
+ if (job.cancelled) {
+ addLog(job, 'Cancelled by user');
+ return;
+ }
+
+ if (items.length === 0) {
+ addLog(job, 'No media found');
+ return;
+ }
+
+ job.progress.total = items.length;
+ addLog(job, `Found ${items.length} media items. Downloading...`);
+
+ // Phase 2: Download all media files
+ const result = await downloadMedia(items, outputDir, workers,
+ (msg) => addLog(job, msg),
+ (completed, errors, total) => {
+ job.progress.completed = completed;
+ job.progress.errors = errors;
+ job.progress.total = total;
+ },
+ () => job.cancelled,
+ base + '/'
+ );
+
+ addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
+ } catch (err) {
+ addLog(job, `Error: ${err.message}`);
+ job.progress.errors++;
+ } finally {
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ pruneCompleted();
+ }
+}
+
+// --- Mega Scrape ---
+
+async function runMegaScrape(job) {
+ const { url, workers, folderName } = job.config;
+ const outputDir = join(MEDIA_PATH, folderName);
+ mkdirSync(outputDir, { recursive: true });
+
+ addLog(job, `Starting mega.nz scrape: ${url}`);
+ addLog(job, `Workers: ${workers}`);
+
+ try {
+ parseMegaUrl(url);
+
+ // Phase 1: List all files
+ const { folderName: megaName, items } = await listAllFiles(url,
+ (msg) => addLog(job, msg)
+ );
+
+ if (job.cancelled) {
+ addLog(job, 'Cancelled by user');
+ return;
+ }
+
+ if (items.length === 0) {
+ addLog(job, 'No files found in folder');
+ return;
+ }
+
+ job.progress.total = items.length;
+ const totalSizeMb = (items.reduce((s, i) => s + i.size, 0) / (1024 * 1024)).toFixed(0);
+ addLog(job, `Found ${items.length} files (${totalSizeMb} MB). Downloading...`);
+
+ // Phase 2: Download
+ const result = await downloadMegaFiles(items, outputDir, workers,
+ (msg) => addLog(job, msg),
+ (completed, errors, total) => {
+ job.progress.completed = completed;
+ job.progress.errors = errors;
+ job.progress.total = total;
+ },
+ () => job.cancelled,
+ (status) => {
+ job.paused = status.paused;
+ job.resumeAt = status.resumeAt;
+ }
+ );
+
+ addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
+ } catch (err) {
+ addLog(job, `Error: ${err.message}`);
+ job.progress.errors++;
+ } finally {
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ pruneCompleted();
+ }
+}
+
+// --- yt-dlp Scrape ---
+
+async function runYtdlpScrape(job) {
+ const config = job.config;
+ addLog(job, `Starting yt-dlp download: ${config.url}`);
+ addLog(job, `Quality: ${config.quality || 'best'}, Playlist: ${config.playlist ? 'yes' : 'no'}`);
+
+ try {
+ const result = await runYtdlp(
+ config,
+ (msg) => addLog(job, msg),
+ (completed, errors) => {
+ job.progress.completed = completed;
+ job.progress.errors += errors;
+ if (completed > job.progress.total) job.progress.total = completed;
+ },
+ () => job.cancelled
+ );
+
+ if (result.cancelled) {
+ addLog(job, 'Cancelled by user');
+ } else {
+ addLog(job, `Done! ${result.files} file${result.files !== 1 ? 's' : ''} downloaded`);
+ }
+ } catch (err) {
+ addLog(job, `Error: ${err.message}`);
+ job.progress.errors++;
+ } finally {
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ pruneCompleted();
+ }
+}
+
+// --- LeakGallery Scrape ---
+
+async function runLeakGalleryScrape(job) {
+ const { url, pages, workers, delay, folderName } = job.config;
+ const outputDir = join(MEDIA_PATH, folderName);
+ mkdirSync(outputDir, { recursive: true });
+
+ addLog(job, `Starting leakgallery scrape: ${url}`);
+ addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
+
+ try {
+ const { username } = parseLeakGalleryUrl(url);
+ addLog(job, `Username: ${username}`);
+
+ // Phase 1: Collect all media
addLog(job, `Fetching up to ${pages} pages from API...`);
- const items = await fetchAllMedia(base, userId, pages, delay,
+ const items = await fetchLeakGalleryMedia(username, pages, delay,
(msg) => addLog(job, msg),
() => job.cancelled
);
@@ -198,7 +421,7 @@ async function runMediaLinkScrape(job) {
addLog(job, `Found ${items.length} media items. Downloading...`);
// Phase 2: Download all media files
- const result = await downloadMedia(items, outputDir, workers,
+ const result = await downloadLeakGalleryMedia(items, outputDir, workers,
(msg) => addLog(job, msg),
(completed, errors, total) => {
job.progress.completed = completed;
@@ -222,7 +445,7 @@ async function runMediaLinkScrape(job) {
// --- Endpoints ---
router.post('/api/scrape/forum', (req, res) => {
- const { url, folderName, startPage, endPage, delay } = req.body;
+ const { url, folderName, startPage, endPage, delay, cookies, siteId, lastPageOnly } = req.body;
if (!url) return res.status(400).json({ error: 'URL is required' });
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
@@ -232,6 +455,9 @@ router.post('/api/scrape/forum', (req, res) => {
startPage: parseInt(startPage) || 1,
endPage: parseInt(endPage) || 10,
delay: parseFloat(delay) || 1.0,
+ cookies: cookies || '',
+ siteId: siteId ? parseInt(siteId, 10) : null,
+ lastPageOnly: !!lastPageOnly,
};
const job = createJob('forum', config);
@@ -289,6 +515,105 @@ router.post('/api/scrape/medialink', (req, res) => {
res.json({ jobId: job.id, message: 'MediaLink scrape started' });
});
+router.post('/api/scrape/mega', (req, res) => {
+ const { url, folderName, workers } = req.body;
+ if (!url) return res.status(400).json({ error: 'URL is required' });
+ if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
+
+ try {
+ parseMegaUrl(url);
+ } catch (err) {
+ return res.status(400).json({ error: err.message });
+ }
+
+ const config = {
+ url,
+ folderName,
+ workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
+ };
+
+ const job = createJob('mega', config);
+ runMegaScrape(job).catch(err => {
+ addLog(job, `Fatal error: ${err.message}`);
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ });
+
+ res.json({ jobId: job.id, message: 'Mega scrape started' });
+});
+
+router.post('/api/scrape/ytdlp', (req, res) => {
+ const { url, quality, customFormat, embedMetadata, embedThumbnail, embedSubs,
+ writeSubs, subLangs, restrictFilenames, outputTemplate,
+ playlist, maxDownloads, concurrentFragments, rateLimit,
+ sponsorBlock, cookiesFile } = req.body;
+ if (!url) return res.status(400).json({ error: 'URL is required' });
+
+ const config = {
+ url,
+ quality: quality || 'best',
+ customFormat: customFormat || '',
+ embedMetadata: embedMetadata !== false,
+ embedThumbnail: embedThumbnail !== false,
+ embedSubs: embedSubs !== false,
+ writeSubs: writeSubs || false,
+ subLangs: subLangs || 'en',
+ restrictFilenames: restrictFilenames !== false,
+ outputTemplate: outputTemplate || '%(title)s.%(ext)s',
+ playlist: playlist || false,
+ maxDownloads: parseInt(maxDownloads) || 0,
+ concurrentFragments: Math.min(Math.max(parseInt(concurrentFragments) || 4, 1), 16),
+ rateLimit: rateLimit || '',
+ sponsorBlock: sponsorBlock || 'off',
+ cookiesFile: cookiesFile || '',
+ folderName: (() => {
+ try {
+ const u = new URL(url);
+ const path = u.pathname.replace(/^\//, '').replace(/\/$/, '');
+ return path ? `${u.hostname}/${path}`.slice(0, 60) : u.hostname;
+ } catch { return url.slice(0, 60); }
+ })(),
+ };
+
+ const job = createJob('ytdlp', config);
+ runYtdlpScrape(job).catch(err => {
+ addLog(job, `Fatal error: ${err.message}`);
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ });
+
+ res.json({ jobId: job.id, message: 'yt-dlp download started' });
+});
+
+router.post('/api/scrape/leakgallery', (req, res) => {
+ const { url, folderName, pages, workers, delay } = req.body;
+ if (!url) return res.status(400).json({ error: 'URL is required' });
+ if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
+
+ try {
+ parseLeakGalleryUrl(url);
+ } catch (err) {
+ return res.status(400).json({ error: err.message });
+ }
+
+ const config = {
+ url,
+ folderName,
+ pages: parseInt(pages) || 100,
+ workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
+ delay: parseInt(delay) || 300,
+ };
+
+ const job = createJob('leakgallery', config);
+ runLeakGalleryScrape(job).catch(err => {
+ addLog(job, `Fatal error: ${err.message}`);
+ job.running = false;
+ job.completedAt = new Date().toISOString();
+ });
+
+ res.json({ jobId: job.id, message: 'LeakGallery scrape started' });
+});
+
router.get('/api/scrape/jobs', (_req, res) => {
const jobs = [...jobsMap.values()].map(jobToJson);
jobs.sort((a, b) => new Date(b.startedAt) - new Date(a.startedAt));
@@ -310,13 +635,95 @@ router.post('/api/scrape/jobs/:jobId/cancel', (req, res) => {
res.json({ message: 'Cancel requested' });
});
+router.delete('/api/scrape/jobs/:jobId', (req, res) => {
+ const job = jobsMap.get(req.params.jobId);
+ if (!job) return res.status(404).json({ error: 'Job not found' });
+ job.cancelled = true;
+ job.running = false;
+ jobsMap.delete(req.params.jobId);
+ res.json({ message: 'Job removed' });
+});
+
// Auto-detect max page for forum URLs
router.post('/api/scrape/forum/detect-pages', async (req, res) => {
- const { url } = req.body;
+ const { url, cookies } = req.body;
if (!url) return res.status(400).json({ error: 'URL is required' });
const logs = [];
- const maxPage = await detectMaxPage(url, (msg) => logs.push(msg));
+ const maxPage = await detectMaxPage(url, (msg) => logs.push(msg), cookies);
res.json({ maxPage, logs });
});
+// --- Forum Sites CRUD ---
+
+router.get('/api/scrape/forum-sites', (_req, res) => {
+ res.json(getForumSites());
+});
+
+router.post('/api/scrape/forum-sites', (req, res) => {
+ const { name, baseUrl, cookies, username, password } = req.body;
+ if (!name) return res.status(400).json({ error: 'Name is required' });
+ const id = createForumSite(name, baseUrl, cookies, username, password);
+ res.json(getForumSiteById(id));
+});
+
+router.put('/api/scrape/forum-sites/:id', (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ const site = getForumSiteById(id);
+ if (!site) return res.status(404).json({ error: 'Forum site not found' });
+ const { name, baseUrl, cookies, username, password } = req.body;
+ const fields = {};
+ if (name !== undefined) fields.name = name;
+ if (baseUrl !== undefined) fields.base_url = baseUrl;
+ if (cookies !== undefined) fields.cookies = cookies;
+ if (username !== undefined) fields.username = username;
+ if (password !== undefined) fields.password = password;
+ updateForumSite(id, fields);
+ res.json(getForumSiteById(id));
+});
+
+router.delete('/api/scrape/forum-sites/:id', (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ deleteForumSite(id);
+ res.json({ ok: true });
+});
+
+// --- Auto-scrape CRUD ---
+
+router.get('/api/scrape/auto', (_req, res) => {
+ res.json(getAutoScrapeJobs());
+});
+
+router.post('/api/scrape/auto', (req, res) => {
+ const { type, url, folderName, config } = req.body;
+ if (!type || !url || !folderName || !config) {
+ return res.status(400).json({ error: 'type, url, folderName, and config are required' });
+ }
+ addAutoScrapeJob(type, url, folderName, config);
+ res.json({ ok: true });
+});
+
+router.delete('/api/scrape/auto/:id', (req, res) => {
+ removeAutoScrapeJob(parseInt(req.params.id));
+ res.json({ ok: true });
+});
+
+export function getActiveScrapeCount() {
+ let count = 0;
+ for (const job of jobsMap.values()) {
+ if (job.running) count++;
+ }
+ return count;
+}
+
+export function getActiveScrapesList() {
+ const list = [];
+ for (const job of jobsMap.values()) {
+ if (job.running) {
+ list.push({ type: job.type, folderName: job.folderName, progress: job.progress });
+ }
+ }
+ return list;
+}
+
+export { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, runYtdlpScrape, runLeakGalleryScrape, createJob };
export default router;
diff --git a/server/scrapers/coomer.js b/server/scrapers/coomer.js
index f004e42..eee836e 100644
--- a/server/scrapers/coomer.js
+++ b/server/scrapers/coomer.js
@@ -7,9 +7,16 @@ const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (
export function parseUserUrl(url) {
const parsed = new URL(url);
const base = `${parsed.protocol}//${parsed.hostname}`;
+
+ // Search URL: /posts?q=query
+ if (parsed.pathname === '/posts' && parsed.searchParams.get('q')) {
+ return { base, mode: 'search', query: parsed.searchParams.get('q') };
+ }
+
+ // User URL: /SERVICE/user/USER_ID
const m = parsed.pathname.match(/^\/([^/]+)\/user\/([^/?#]+)/);
- if (!m) throw new Error(`Can't parse URL. Expected: https://coomer.su/SERVICE/user/USER_ID`);
- return { base, service: m[1], userId: m[2] };
+ if (!m) throw new Error(`Can't parse URL. Expected: https://coomer.su/SERVICE/user/USER_ID or https://coomer.su/posts?q=QUERY`);
+ return { base, mode: 'user', service: m[1], userId: m[2] };
}
async function fetchApi(apiUrl, logFn, retries = 3) {
@@ -150,6 +157,45 @@ export async function fetchAllPosts(base, service, userId, maxPages, logFn, chec
return allFiles;
}
+export async function fetchSearchPosts(base, query, maxPages, logFn, checkCancelled) {
+ const allFiles = [];
+
+ for (let page = 0; page < maxPages; page++) {
+ if (checkCancelled()) break;
+
+ const offset = page * 50;
+ const apiUrl = `${base}/api/v1/posts?q=${encodeURIComponent(query)}&o=${offset}`;
+
+ let data;
+ try {
+ data = await fetchApi(apiUrl, logFn);
+ } catch (err) {
+ logFn(`API failed: ${err.message}`);
+ break;
+ }
+
+ // Search API returns { count, posts: [...] } not a plain array
+ const posts = data?.posts || data;
+ if (!posts || !Array.isArray(posts) || posts.length === 0) break;
+
+ const parsed = new URL(base);
+ const cdnHost = `n1.${parsed.hostname}`;
+ const cdnBase = `${parsed.protocol}//${cdnHost}/data`;
+
+ const files = collectFiles(posts, cdnBase);
+ allFiles.push(...files);
+
+ if (page === 0 && data?.count) {
+ logFn(`Search found ${data.count} total results`);
+ }
+ logFn(`Page ${page + 1}: ${posts.length} posts (${allFiles.length} files total)`);
+
+ if (posts.length < 50) break;
+ }
+
+ return allFiles;
+}
+
export async function downloadFiles(files, outputDir, concurrency, logFn, progressFn, checkCancelled) {
mkdirSync(outputDir, { recursive: true });
diff --git a/server/scrapers/forum.js b/server/scrapers/forum.js
index 261a63a..89debe1 100644
--- a/server/scrapers/forum.js
+++ b/server/scrapers/forum.js
@@ -1,13 +1,43 @@
import * as cheerio from 'cheerio';
-import { createWriteStream, existsSync, mkdirSync, statSync } from 'fs';
+import { createWriteStream, existsSync, mkdirSync, statSync, writeFileSync } from 'fs';
import { basename, join, extname } from 'path';
import { pipeline } from 'stream/promises';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
import { upsertMediaFile } from '../db.js';
+const execFileAsync = promisify(execFile);
+
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const SERVER_IP = '47.185.183.191';
+
+export class CookieExpiredError extends Error {
+ constructor(statusCode) {
+ super(`Cookie expired or invalid (HTTP ${statusCode})`);
+ this.name = 'CookieExpiredError';
+ this.statusCode = statusCode;
+ }
+}
+
+// Replace DDoS-Guard __ddg9_ cookie IP with server's IP so cookies work from any browser
+function fixCookieIp(cookies) {
+ if (!cookies) return cookies;
+ return cookies.replace(/__ddg9_=[^;]+/, `__ddg9_=${SERVER_IP}`);
+}
const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);
-const SKIP_PATTERNS = ['avatar', 'smilie', 'emoji', 'icon', 'logo', 'button', 'sprite', 'badge', 'rank', 'star'];
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
+const SKIP_PATTERNS = ['avatar', 'smilie', 'emoji', 'icon', 'logo', 'button', 'sprite', 'badge', 'rank', 'star', 'dc_thumbnails'];
+
+// External hosts that gallery-dl can resolve
+const GALLERY_DL_HOSTS = [
+ /saint\d*\.\w+/i,
+ /cyberdrop\.\w+/i,
+ /bunkr+\.\w+/i,
+ /pixeldrain\.com/i,
+ /gofile\.io/i,
+ /turbo\.\w+/i,
+];
function isImageUrl(url) {
try {
@@ -16,26 +46,44 @@ function isImageUrl(url) {
} catch { return false; }
}
+function isVideoUrl(url) {
+ try {
+ const path = new URL(url).pathname.toLowerCase();
+ return [...VIDEO_EXTS].some(ext => path.endsWith(ext));
+ } catch { return false; }
+}
+
+function isMediaUrl(url) {
+ return isImageUrl(url) || isVideoUrl(url);
+}
+
+function isExternalHost(url) {
+ try {
+ const hostname = new URL(url).hostname.toLowerCase();
+ return GALLERY_DL_HOSTS.some(p => p.test(hostname));
+ } catch { return false; }
+}
+
export function getPageUrl(baseUrl, pageNum) {
const url = baseUrl.replace(/page-\d+/, `page-${pageNum}`);
return url.split('#')[0];
}
-export async function detectMaxPage(baseUrl, logFn) {
+export async function detectMaxPage(baseUrl, logFn, cookies) {
try {
- const resp = await fetch(baseUrl, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(15000) });
+ const headers = { 'User-Agent': UA };
+ if (cookies) headers['Cookie'] = fixCookieIp(cookies);
+ const resp = await fetch(baseUrl, { headers, signal: AbortSignal.timeout(15000) });
if (!resp.ok) return null;
const html = await resp.text();
const $ = cheerio.load(html);
let maxPage = 1;
- // XenForo-style
$('a.pageNav-page, .pageNav a[href*="page-"], .pagination a[href*="page-"]').each((_, el) => {
const href = $(el).attr('href') || '';
const m = href.match(/page-(\d+)/);
if (m) maxPage = Math.max(maxPage, parseInt(m[1], 10));
});
- // Generic pagination text
$('a').each((_, el) => {
const text = $(el).text().trim();
if (/^\d+$/.test(text)) {
@@ -58,6 +106,7 @@ export async function detectMaxPage(baseUrl, logFn) {
function tryFullSizeUrl(thumbUrl) {
const candidates = [];
if (thumbUrl.includes('.th.')) candidates.push(thumbUrl.replace('.th.', '.'));
+ if (thumbUrl.includes('.md.')) candidates.push(thumbUrl.replace('.md.', '.'));
if (/_thumb\./i.test(thumbUrl)) candidates.push(thumbUrl.replace(/_thumb\./i, '.'));
if (thumbUrl.includes('/thumbs/')) {
candidates.push(thumbUrl.replace('/thumbs/', '/images/'));
@@ -74,7 +123,7 @@ function tryFullSizeUrl(thumbUrl) {
return candidates;
}
-async function downloadImage(url, outputDir, downloadedSet, logFn) {
+async function downloadImage(url, outputDir, downloadedSet, logFn, cookies) {
if (downloadedSet.has(url)) return false;
if (!isImageUrl(url)) return false;
const lower = url.toLowerCase();
@@ -83,47 +132,34 @@ async function downloadImage(url, outputDir, downloadedSet, logFn) {
downloadedSet.add(url);
let filename;
- try {
- filename = basename(new URL(url).pathname);
- } catch { return false; }
+ try { filename = basename(new URL(url).pathname); } catch { return false; }
if (!filename) return false;
+ filename = filename.replace('.th.', '.').replace('.md.', '.');
- filename = filename.replace('.th.', '.');
-
- let filepath = join(outputDir, filename);
+ const filepath = join(outputDir, filename);
if (existsSync(filepath)) {
- const ext = extname(filename);
- const name = filename.slice(0, -ext.length);
- let i = 1;
- while (existsSync(filepath)) {
- filepath = join(outputDir, `${name}_${i}${ext}`);
- i++;
- }
+ return false;
}
try {
- const resp = await fetch(url, {
- headers: { 'User-Agent': UA },
- signal: AbortSignal.timeout(30000),
- });
+ const dlHeaders = { 'User-Agent': UA };
+ if (cookies) dlHeaders['Cookie'] = fixCookieIp(cookies);
+ const resp = await fetch(url, { headers: dlHeaders, signal: AbortSignal.timeout(30000) });
if (!resp.ok) {
logFn(`FAILED (${resp.status}): ${url}`);
return false;
}
- // Read full body to check size
const buf = Buffer.from(await resp.arrayBuffer());
if (buf.length < 1000) {
downloadedSet.delete(url);
return false;
}
- const { writeFileSync } = await import('fs');
writeFileSync(filepath, buf);
-
const savedName = basename(filepath);
const folderName = basename(outputDir);
- try { upsertMediaFile(folderName, savedName, 'image', buf.length, Date.now(), null); } catch { /* ignore */ }
+ try { upsertMediaFile(folderName, savedName, 'image', buf.length, Date.now(), null); } catch {}
const sizeKb = (buf.length / 1024).toFixed(1);
logFn(`Downloaded: ${savedName} (${sizeKb} KB)`);
@@ -134,28 +170,101 @@ async function downloadImage(url, outputDir, downloadedSet, logFn) {
}
}
-export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn) {
+// Use gallery-dl to download from external hosts (bunkr, saint, cyberdrop, etc.)
+async function downloadFromExternalHost(url, outputDir, downloadedSet, logFn) {
+ if (downloadedSet.has(url)) return 0;
+ downloadedSet.add(url);
+
+ logFn(`Resolving via gallery-dl: ${url}`);
+
+ try {
+ const args = [
+ '-d', outputDir,
+ '--filename', '{filename}.{extension}',
+ '--no-mtime',
+ '-o', 'directory=[]',
+ url,
+ ];
+
+ const { stdout, stderr } = await execFileAsync('gallery-dl', args, {
+ timeout: 300000, // 5 min per external link
+ maxBuffer: 10 * 1024 * 1024,
+ });
+
+ let count = 0;
+ const lines = (stdout + '\n' + stderr).split('\n').filter(Boolean);
+ for (const line of lines) {
+ // gallery-dl outputs file paths for downloaded files
+ const trimmed = line.trim();
+ if (trimmed.startsWith(outputDir) || trimmed.startsWith('/')) {
+ const filePath = trimmed.replace(/^# /, '');
+ if (existsSync(filePath)) {
+ const stat = statSync(filePath);
+ const savedName = basename(filePath);
+ const folderName = basename(outputDir);
+ const ext = extname(savedName).toLowerCase();
+ const type = VIDEO_EXTS.has(ext) ? 'video' : 'image';
+ const sizeStr = type === 'video'
+ ? `${(stat.size / (1024 * 1024)).toFixed(1)} MB`
+ : `${(stat.size / 1024).toFixed(1)} KB`;
+
+ try { upsertMediaFile(folderName, savedName, type, stat.size, Date.now(), null); } catch {}
+ logFn(`Downloaded: ${savedName} (${sizeStr}) [${type}]`);
+ count++;
+ }
+ } else if (trimmed.includes('Downloading') || trimmed.includes('Skipping')) {
+ logFn(` ${trimmed}`);
+ }
+ }
+
+ if (count === 0) {
+ // gallery-dl doesn't always output paths clearly, check stderr for errors
+ const errLines = stderr ? stderr.split('\n').filter(l => l.trim()) : [];
+ for (const line of errLines) {
+ if (line.includes('ERROR') || line.includes('error')) {
+ logFn(` gallery-dl: ${line.trim()}`);
+ }
+ }
+ logFn(` gallery-dl finished but no files detected from output`);
+ }
+
+ return count;
+ } catch (err) {
+ if (err.stderr) {
+ const errMsg = err.stderr.split('\n').find(l => l.includes('ERROR') || l.includes('error')) || err.stderr.slice(0, 200);
+ logFn(`gallery-dl error: ${errMsg.trim()}`);
+ } else {
+ logFn(`gallery-dl error: ${err.message}`);
+ }
+ return 0;
+ }
+}
+
+export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn, cookies) {
logFn(`Fetching page: ${pageUrl}`);
let html;
try {
- const resp = await fetch(pageUrl, {
- headers: { 'User-Agent': UA },
- signal: AbortSignal.timeout(15000),
- });
+ const headers = { 'User-Agent': UA };
+ if (cookies) headers['Cookie'] = fixCookieIp(cookies);
+ const resp = await fetch(pageUrl, { headers, signal: AbortSignal.timeout(15000) });
if (!resp.ok) {
+ // SimpCity returns 404 for expired sessions, 403 for blocked
+ if (cookies && (resp.status === 404 || resp.status === 403)) {
+ throw new CookieExpiredError(resp.status);
+ }
logFn(`Failed to fetch page (${resp.status})`);
return 0;
}
html = await resp.text();
} catch (err) {
+ if (err instanceof CookieExpiredError) throw err;
logFn(`Failed to fetch page: ${err.message}`);
return 0;
}
const $ = cheerio.load(html);
- // Try known content selectors, fall back to whole page
const selectors = '.message-body, .post-body, .post_body, .postcontent, .messageContent, .bbWrapper, article, .entry-content, .post_message, .post-content, #posts, .threadBody';
let contentAreas = $(selectors).toArray();
if (contentAreas.length === 0) {
@@ -163,6 +272,7 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
}
const imageUrls = [];
+ const externalUrls = new Set();
for (const area of contentAreas) {
const $area = $(area);
@@ -176,7 +286,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
let absSrc;
try { absSrc = new URL(src, pageUrl).href; } catch { return; }
- // Check parent for direct image link
const $parentA = $img.closest('a');
if ($parentA.length && $parentA.attr('href')) {
try {
@@ -188,7 +297,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
} catch {}
}
- // Try to derive full-size from thumbnail URL
const fullCandidates = tryFullSizeUrl(absSrc);
if (fullCandidates.length > 0) {
imageUrls.push(...fullCandidates);
@@ -196,7 +304,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
imageUrls.push(absSrc);
}
- // Also check data attributes
for (const attr of ['data-src', 'data-url', 'data-orig', 'data-original', 'data-full-url', 'data-zoom-src']) {
const val = $img.attr(attr);
if (val && val !== src) {
@@ -205,26 +312,64 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
}
});
- // Pass 2: pointing directly to images (no child
)
+ // Pass 2: links — images + external hosts
$area.find('a[href]').each((_, el) => {
const $a = $(el);
- if ($a.find('img').length) return;
+ let href;
+ try { href = new URL($a.attr('href'), pageUrl).href; } catch { return; }
+
+ // Skip same-forum links
try {
- const href = new URL($a.attr('href'), pageUrl).href;
- if (isImageUrl(href)) imageUrls.push(href);
+ if (new URL(href).hostname === new URL(pageUrl).hostname) return;
} catch {}
+
+ // Direct image link (without child img — those are handled in Pass 1)
+ if (isImageUrl(href) && $a.find('img').length === 0) {
+ imageUrls.push(href);
+ return;
+ }
+
+ // Direct video link
+ if (isVideoUrl(href)) {
+ externalUrls.add(href);
+ return;
+ }
+
+ // External file host (bunkr, saint, cyberdrop, etc.)
+ if (isExternalHost(href)) {
+ externalUrls.add(href);
+ }
+ });
+
+ // Pass 3: iframe embeds
+ $area.find('iframe[src]').each((_, el) => {
+ const src = $(el).attr('src');
+ if (src) {
+ try {
+ const absUrl = new URL(src, pageUrl).href;
+ if (isExternalHost(absUrl)) externalUrls.add(absUrl);
+ } catch {}
+ }
});
}
- logFn(`Found ${imageUrls.length} candidate URLs`);
+ logFn(`Found ${imageUrls.length} images, ${externalUrls.size} external links`);
let count = 0;
+
+ // Download images
for (const imgUrl of imageUrls) {
- if (await downloadImage(imgUrl, outputDir, downloadedSet, logFn)) {
+ if (await downloadImage(imgUrl, outputDir, downloadedSet, logFn, cookies)) {
count++;
}
}
- logFn(`${count} images from this page`);
+ // Download from external hosts via gallery-dl
+ for (const extUrl of externalUrls) {
+ const dlCount = await downloadFromExternalHost(extUrl, outputDir, downloadedSet, logFn);
+ count += dlCount;
+ }
+
+ logFn(`${count} files from this page`);
return count;
}
diff --git a/server/scrapers/leakgallery.js b/server/scrapers/leakgallery.js
new file mode 100644
index 0000000..1d7e659
--- /dev/null
+++ b/server/scrapers/leakgallery.js
@@ -0,0 +1,191 @@
+import { existsSync, writeFileSync, mkdirSync } from 'fs';
+import { basename, join, extname } from 'path';
+import { upsertMediaFile } from '../db.js';
+
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const API_BASE = 'https://api.leakgallery.com';
+const CDN_BASE = 'https://cdn.leakgallery.com';
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
+
+export function parseLeakGalleryUrl(url) {
+ const parsed = new URL(url);
+ if (!parsed.hostname.includes('leakgallery.com')) {
+ throw new Error('Not a leakgallery.com URL');
+ }
+ // URL format: https://leakgallery.com/{username}
+ const m = parsed.pathname.match(/^\/([a-zA-Z0-9_.-]+)\/?$/);
+ if (!m) throw new Error('Expected URL format: https://leakgallery.com/username');
+ return { username: m[1] };
+}
+
+async function fetchPage(username, page, logFn) {
+ // Page 1: /profile/{username}?type=All&sort=MostRecent
+ // Page 2+: /profile/{username}/{page}?type=All&sort=MostRecent
+ const pagePath = page <= 1 ? '' : `/${page}`;
+ const apiUrl = `${API_BASE}/profile/${username}${pagePath}?type=All&sort=MostRecent`;
+
+ try {
+ const resp = await fetch(apiUrl, {
+ headers: {
+ 'User-Agent': UA,
+ 'Accept': 'application/json',
+ 'Origin': 'https://leakgallery.com',
+ 'Referer': 'https://leakgallery.com/',
+ },
+ signal: AbortSignal.timeout(15000),
+ });
+ if (!resp.ok) {
+ if (resp.status === 404) return null;
+ logFn(`API error (${resp.status}): ${apiUrl}`);
+ return null;
+ }
+ return await resp.json();
+ } catch (err) {
+ logFn(`API fetch error: ${err.message}`);
+ return null;
+ }
+}
+
+export async function fetchAllMedia(username, maxPages, delay, logFn, checkCancelled) {
+ const allItems = [];
+ const seen = new Set();
+ let totalCount = 0;
+
+ for (let page = 1; page <= maxPages; page++) {
+ if (checkCancelled()) break;
+
+ logFn(`Fetching page ${page}...`);
+ const data = await fetchPage(username, page, logFn);
+
+ if (!data) {
+ logFn(`Page ${page}: no data — stopping`);
+ break;
+ }
+
+ // First page includes mediaCount
+ if (page === 1 && data.mediaCount) {
+ totalCount = data.mediaCount;
+ logFn(`Profile has ${totalCount} total media items`);
+ }
+
+ const medias = data.medias;
+ if (!medias || !Array.isArray(medias) || medias.length === 0) {
+ logFn(`Page ${page}: no more items — done`);
+ break;
+ }
+
+ let newCount = 0;
+ for (const item of medias) {
+ if (seen.has(item.id)) continue;
+ seen.add(item.id);
+ newCount++;
+
+ // file_path is relative, e.g. content4/username/watermark_hash__username__id_580px.webp
+ // Full-size: remove _580px.webp suffix, use .jpg (or .mp4 for videos)
+ const isVideo = !!item.is_video;
+ let fullUrl;
+ let filename;
+
+ if (isVideo) {
+ // Videos: file_path is already the video file
+ fullUrl = `${CDN_BASE}/${item.file_path}`;
+ filename = basename(item.file_path);
+ } else {
+ // Images: thumbnail has _580px.webp — convert to full-size .jpg
+ const filePath = item.file_path || item.thumbnail_path || '';
+ const fullPath = filePath
+ .replace(/_580px\.webp$/, '.jpg')
+ .replace(/_300px\.webp$/, '.jpg');
+ fullUrl = `${CDN_BASE}/${fullPath}`;
+ filename = basename(fullPath);
+ }
+
+ allItems.push({
+ id: item.id,
+ url: fullUrl,
+ filename,
+ type: isVideo ? 'video' : 'image',
+ });
+ }
+
+ if (newCount === 0) {
+ logFn(`Page ${page}: all duplicates — stopping`);
+ break;
+ }
+
+ logFn(`Page ${page}: ${medias.length} items (${newCount} new, ${allItems.length} total)`);
+
+ if (page < maxPages && !checkCancelled()) {
+ await new Promise(r => setTimeout(r, delay));
+ }
+ }
+
+ return allItems;
+}
+
+async function tryFetch(url) {
+ try {
+ const resp = await fetch(url, {
+ headers: {
+ 'User-Agent': UA,
+ 'Referer': 'https://leakgallery.com/',
+ },
+ signal: AbortSignal.timeout(60000),
+ });
+ if (!resp.ok) return null;
+ const buf = Buffer.from(await resp.arrayBuffer());
+ if (buf.length < 500) return null;
+ return buf;
+ } catch {
+ return null;
+ }
+}
+
+export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
+ mkdirSync(outputDir, { recursive: true });
+
+ let completed = 0;
+ let errors = 0;
+ let skipped = 0;
+ let index = 0;
+
+ async function processNext() {
+ while (index < items.length) {
+ if (checkCancelled()) return;
+
+ const current = index++;
+ const item = items[current];
+ const filename = item.filename || `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
+ const filepath = join(outputDir, filename);
+
+ if (existsSync(filepath)) {
+ skipped++;
+ progressFn(completed + skipped, errors, items.length);
+ continue;
+ }
+
+ const buf = await tryFetch(item.url);
+ if (buf) {
+ writeFileSync(filepath, buf);
+ const folderName = basename(outputDir);
+ const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
+ try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
+ completed++;
+ logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
+ progressFn(completed + skipped, errors, items.length);
+ } else {
+ logFn(`FAILED: ${filename}`);
+ errors++;
+ progressFn(completed + skipped, errors, items.length);
+ }
+ }
+ }
+
+ const workerPromises = [];
+ for (let i = 0; i < Math.min(workers, items.length); i++) {
+ workerPromises.push(processNext());
+ }
+ await Promise.all(workerPromises);
+
+ return { completed, errors, skipped, total: items.length };
+}
diff --git a/server/scrapers/medialink.js b/server/scrapers/medialink.js
index 53b8c4f..4c81c72 100644
--- a/server/scrapers/medialink.js
+++ b/server/scrapers/medialink.js
@@ -1,6 +1,7 @@
-import { existsSync, writeFileSync, mkdirSync } from 'fs';
+import { existsSync, writeFileSync, mkdirSync, unlinkSync } from 'fs';
import { basename, join, extname } from 'path';
-import { upsertMediaFile } from '../db.js';
+import { load as cheerioLoad } from 'cheerio';
+import { upsertMediaFile, removeMediaFile } from '../db.js';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
@@ -9,10 +10,13 @@ const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
export function parseMediaUrl(url) {
const parsed = new URL(url);
const base = `${parsed.protocol}//${parsed.hostname}`;
- // Support /model/{id} or /media/{id}
+ // Support /model/{id} or /media/{id} (fapello.to JSON API)
const m = parsed.pathname.match(/\/(?:model|media)\/(\d+)/);
- if (!m) throw new Error(`Can't parse URL. Expected: https://fapello.to/model/12345`);
- return { base, userId: m[1] };
+ if (m) return { base, userId: m[1], mode: 'api' };
+ // Support fapello.com profile slug URLs like /josie-hamming-41/
+ const slugMatch = parsed.pathname.match(/^\/([a-zA-Z0-9_-]+)\/?$/);
+ if (slugMatch) return { base, userId: slugMatch[1], mode: 'html' };
+ throw new Error(`Can't parse URL. Expected: https://fapello.to/model/12345 or https://fapello.com/username/`);
}
// Fetch JSON from the API endpoint
@@ -73,6 +77,7 @@ export async function fetchAllMedia(base, userId, maxPages, delay, logFn, checkC
allItems.push({
id: item.id,
url: fullUrl,
+ thumbUrl: item.newUrlThumb || null,
type: isVideo ? 'video' : 'image',
});
}
@@ -92,13 +97,171 @@ export async function fetchAllMedia(base, userId, maxPages, delay, logFn, checkC
return allItems;
}
+// --- HTML-based scraping (fapello.com profile pages) ---
+
+function parseMediaFromHtml(html, base) {
+ const $ = cheerioLoad(html);
+ const items = [];
+
+ // Find all image thumbnails in the grid
+ $('img[src*="_300px."]').each((_, el) => {
+ const thumbUrl = $(el).attr('src');
+ if (!thumbUrl) return;
+ // Convert thumbnail to full-size: remove _300px
+ const fullUrl = thumbUrl.replace(/_300px\./, '.');
+ const absUrl = fullUrl.startsWith('http') ? fullUrl : `${base}${fullUrl}`;
+ items.push({ url: absUrl, type: 'image' });
+ });
+
+ // Find video elements (source tags with .mp4)
+ $('video source[src*=".mp4"], video[src*=".mp4"]').each((_, el) => {
+ const src = $(el).attr('src');
+ if (!src) return;
+ const absUrl = src.startsWith('http') ? src : `${base}${src}`;
+ items.push({ url: absUrl, type: 'video' });
+ });
+
+ return items;
+}
+
+export async function fetchAllMediaFromHtml(base, slug, maxPages, delay, logFn, checkCancelled) {
+ const allItems = [];
+ const seen = new Set();
+ let totalPages = maxPages;
+
+ // Phase 1: Fetch initial profile page to get data-max
+ logFn(`Fetching profile page: ${base}/${slug}/`);
+ try {
+ const resp = await fetch(`${base}/${slug}/`, {
+ headers: { 'User-Agent': UA },
+ signal: AbortSignal.timeout(15000),
+ });
+ if (!resp.ok) {
+ logFn(`Profile page error (${resp.status})`);
+ return allItems;
+ }
+ const html = await resp.text();
+ const $ = cheerioLoad(html);
+
+ // Get max pages from data-max attribute
+ const dataMax = $('#showmore').attr('data-max');
+ if (dataMax) {
+ totalPages = Math.min(parseInt(dataMax, 10) || maxPages, maxPages);
+ logFn(`Detected ${totalPages} pages`);
+ }
+
+ // Parse initial page content
+ const initialItems = parseMediaFromHtml(html, base);
+ for (const item of initialItems) {
+ if (!seen.has(item.url)) {
+ seen.add(item.url);
+ allItems.push({ ...item, id: seen.size });
+ }
+ }
+ logFn(`Page 1: ${initialItems.length} items (${allItems.length} total)`);
+ } catch (err) {
+ logFn(`Error fetching profile: ${err.message}`);
+ return allItems;
+ }
+
+ // Phase 2: Paginate through AJAX pages
+ for (let page = 2; page <= totalPages; page++) {
+ if (checkCancelled()) break;
+
+ const ajaxUrl = `${base}/ajax/model/${slug}/page-${page}/`;
+ try {
+ const resp = await fetch(ajaxUrl, {
+ headers: {
+ 'User-Agent': UA,
+ 'X-Requested-With': 'XMLHttpRequest',
+ 'Referer': `${base}/${slug}/`,
+ },
+ signal: AbortSignal.timeout(15000),
+ });
+ if (!resp.ok) {
+ if (resp.status === 404) {
+ logFn(`Page ${page}: 404 — done`);
+ break;
+ }
+ logFn(`Page ${page}: error (${resp.status})`);
+ continue;
+ }
+ const html = await resp.text();
+ if (!html || html.trim().length === 0) {
+ logFn(`Page ${page}: empty — done`);
+ break;
+ }
+
+ const pageItems = parseMediaFromHtml(html, base);
+ let newCount = 0;
+ for (const item of pageItems) {
+ if (!seen.has(item.url)) {
+ seen.add(item.url);
+ allItems.push({ ...item, id: seen.size });
+ newCount++;
+ }
+ }
+
+ if (newCount === 0) {
+ logFn(`Page ${page}: all duplicates — stopping`);
+ break;
+ }
+
+ logFn(`Page ${page}: ${pageItems.length} items (${newCount} new, ${allItems.length} total)`);
+ } catch (err) {
+ logFn(`Page ${page}: error — ${err.message}`);
+ }
+
+ if (page < totalPages && !checkCancelled()) {
+ await new Promise(r => setTimeout(r, delay));
+ }
+ }
+
+ return allItems;
+}
+
+// Helper: derive filename from URL, with fallback
+function filenameFromUrl(url, item) {
+ try {
+ const name = basename(new URL(url).pathname);
+ if (name && name !== '/') return name;
+ } catch {}
+ return `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
+}
+
+// Helper: add _md suffix before extension
+function mdFilename(filename) {
+ const ext = extname(filename);
+ return filename.slice(0, -ext.length) + '_md' + ext;
+}
+
+// Helper: try fetching a URL, return buffer or null
+async function tryFetch(url, referer) {
+ if (!url) return null;
+ try {
+ const resp = await fetch(url, {
+ headers: { 'User-Agent': UA, 'Referer': referer || 'https://fapello.to/' },
+ signal: AbortSignal.timeout(60000),
+ });
+ if (!resp.ok) return null;
+ const buf = Buffer.from(await resp.arrayBuffer());
+ if (buf.length < 500) return null;
+ return buf;
+ } catch {
+ return null;
+ }
+}
+
// Download all collected media items with concurrency
-export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
+// Fallback: if full-res URL fails, download medium (thumbUrl) with _md suffix.
+// Upgrade: if _md file exists, try full-res again; replace _md on success.
+export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled, referer) {
mkdirSync(outputDir, { recursive: true });
let completed = 0;
let errors = 0;
let skipped = 0;
+ let upgraded = 0;
let index = 0;
async function processNext() {
@@ -108,72 +271,71 @@ export async function downloadMedia(items, outputDir, workers, logFn, progressFn
const current = index++;
const item = items[current];
- let filename;
- try {
- filename = basename(new URL(item.url).pathname);
- if (!filename || filename === '/') {
- filename = `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
- }
- } catch {
- filename = `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
- }
+ const filename = filenameFromUrl(item.url, item);
+ const filepath = join(outputDir, filename);
+ const mdName = mdFilename(filename);
+ const mdPath = join(outputDir, mdName);
- let filepath = join(outputDir, filename);
+ // Full-res already exists — skip
if (existsSync(filepath)) {
skipped++;
progressFn(completed + skipped, errors, items.length);
continue;
}
- try {
- const resp = await fetch(item.url, {
- headers: {
- 'User-Agent': UA,
- 'Referer': 'https://fapello.to/',
- },
- signal: AbortSignal.timeout(60000),
- });
- if (!resp.ok) {
- logFn(`FAILED (${resp.status}): ${filename}`);
- errors++;
+ // Medium version exists — try to upgrade to full-res
+ if (existsSync(mdPath)) {
+ const buf = await tryFetch(item.url, referer);
+ if (buf) {
+ writeFileSync(filepath, buf);
+ try { unlinkSync(mdPath); } catch {}
+ const folderName = basename(outputDir);
+ const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
+ try { removeMediaFile(folderName, mdName); } catch {}
+ try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
+ upgraded++;
+ completed++;
+ logFn(`[${completed}/${items.length}] ${filename} (upgraded from _md, ${(buf.length / 1024).toFixed(1)} KB)`);
progressFn(completed + skipped, errors, items.length);
- continue;
- }
-
- const buf = Buffer.from(await resp.arrayBuffer());
- if (buf.length < 500) {
+ } else {
skipped++;
progressFn(completed + skipped, errors, items.length);
+ }
+ continue;
+ }
+
+ // Neither exists — try full-res, then fallback to medium
+ const buf = await tryFetch(item.url, referer);
+ if (buf) {
+ writeFileSync(filepath, buf);
+ const folderName = basename(outputDir);
+ const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
+ try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
+ completed++;
+ logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
+ progressFn(completed + skipped, errors, items.length);
+ continue;
+ }
+
+ // Full-res failed — try medium (thumbUrl)
+ if (item.thumbUrl) {
+ const mdBuf = await tryFetch(item.thumbUrl, referer);
+ if (mdBuf) {
+ writeFileSync(mdPath, mdBuf);
+ const folderName = basename(outputDir);
+ const fileType = VIDEO_EXTS.has(extname(mdName).toLowerCase()) ? 'video' : 'image';
+ try { upsertMediaFile(folderName, mdName, fileType, mdBuf.length, Date.now(), null); } catch {}
+ completed++;
+ logFn(`[${completed}/${items.length}] ${mdName} (medium fallback, ${(mdBuf.length / 1024).toFixed(1)} KB)`);
+ progressFn(completed + skipped, errors, items.length);
continue;
}
-
- // Handle filename collision
- if (existsSync(filepath)) {
- const ext = extname(filename);
- const name = filename.slice(0, -ext.length);
- let i = 1;
- while (existsSync(filepath)) {
- filepath = join(outputDir, `${name}_${i}${ext}`);
- i++;
- }
- }
-
- writeFileSync(filepath, buf);
- const savedName = basename(filepath);
- const folderName = basename(outputDir);
- const fileExt = extname(savedName).toLowerCase();
- const fileType = VIDEO_EXTS.has(fileExt) ? 'video' : 'image';
- try { upsertMediaFile(folderName, savedName, fileType, buf.length, Date.now(), null); } catch {}
-
- completed++;
- const sizeKb = (buf.length / 1024).toFixed(1);
- logFn(`[${completed}/${items.length}] ${savedName} (${sizeKb} KB)`);
- progressFn(completed + skipped, errors, items.length);
- } catch (err) {
- logFn(`FAILED: ${filename} - ${err.message}`);
- errors++;
- progressFn(completed + skipped, errors, items.length);
}
+
+ // Both failed
+ logFn(`FAILED: ${filename} — full-res and medium both unavailable`);
+ errors++;
+ progressFn(completed + skipped, errors, items.length);
}
}
@@ -183,5 +345,6 @@ export async function downloadMedia(items, outputDir, workers, logFn, progressFn
}
await Promise.all(workerPromises);
+ if (upgraded > 0) logFn(`Upgraded ${upgraded} files from medium to full resolution`);
return { completed, errors, skipped, total: items.length };
}
diff --git a/server/scrapers/mega.js b/server/scrapers/mega.js
new file mode 100644
index 0000000..c1867e7
--- /dev/null
+++ b/server/scrapers/mega.js
@@ -0,0 +1,219 @@
+import { File } from 'megajs';
+import { existsSync, mkdirSync, statSync, unlinkSync } from 'fs';
+import { createWriteStream } from 'fs';
+import { basename, join, extname } from 'path';
+import { pipeline } from 'stream/promises';
+import { upsertMediaFile } from '../db.js';
+
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
+const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);
+
+export function parseMegaUrl(url) {
+ // Validate it's a mega.nz folder URL
+ const parsed = new URL(url);
+ if (!parsed.hostname.includes('mega.nz') && !parsed.hostname.includes('mega.co.nz')) {
+ throw new Error('Not a mega.nz URL');
+ }
+ if (!parsed.pathname.includes('/folder/')) {
+ throw new Error('Expected a mega.nz folder URL (e.g. https://mega.nz/folder/ABC#key)');
+ }
+ return url;
+}
+
+// Load shared folder and list all files recursively
+export async function listAllFiles(url, logFn) {
+ logFn('Loading shared folder...');
+ const folder = File.fromURL(url);
+ await folder.loadAttributes();
+
+ const folderName = folder.name || 'mega_folder';
+ logFn(`Folder: ${folderName}`);
+
+ // Recursively get all non-directory files
+ const allFiles = folder.filter(f => !f.directory, true);
+ logFn(`Found ${allFiles.length} files across all subfolders`);
+
+ // Build items with subfolder paths
+ const items = [];
+ for (const file of allFiles) {
+ const ext = extname(file.name).toLowerCase();
+ let type = 'other';
+ if (IMAGE_EXTS.has(ext)) type = 'image';
+ else if (VIDEO_EXTS.has(ext)) type = 'video';
+
+ // Build relative path from parent folders
+ let subfolder = '';
+ let parent = file.parent;
+ const parts = [];
+ while (parent && parent !== folder) {
+ parts.unshift(parent.name);
+ parent = parent.parent;
+ }
+ subfolder = parts.join('/');
+
+ items.push({
+ file,
+ name: file.name,
+ size: file.size,
+ type,
+ subfolder,
+ });
+ }
+
+ return { folderName, items };
+}
+
+// Parse bandwidth limit wait time from error message
+function parseBandwidthWait(errMsg) {
+ const m = errMsg.match(/(\d+)\s*seconds?\s*until/i);
+ if (m) return parseInt(m[1], 10);
+ if (/bandwidth/i.test(errMsg)) return 3600; // default 1hr if can't parse
+ return 0;
+}
+
+// Download all files with concurrency + bandwidth limit auto-retry
+export async function downloadMegaFiles(items, outputDir, workers, logFn, progressFn, checkCancelled, statusFn) {
+ mkdirSync(outputDir, { recursive: true });
+
+ let completed = 0;
+ let errors = 0;
+ let skipped = 0;
+ let index = 0;
+ let bandwidthPaused = false;
+
+ async function processNext() {
+ while (index < items.length) {
+ if (checkCancelled()) return;
+
+ // If another worker hit the bandwidth limit, wait for it to clear
+ if (bandwidthPaused) return;
+
+ const current = index++;
+ const item = items[current];
+
+ // All files go to root output dir (flatten subfolders)
+ const filepath = join(outputDir, item.name);
+
+ // Skip if file exists AND is non-empty (0-byte = failed partial download)
+ if (existsSync(filepath)) {
+ try {
+ const st = statSync(filepath);
+ if (st.size > 0) {
+ skipped++;
+ progressFn(completed + skipped, errors, items.length);
+ continue;
+ }
+ // Remove 0-byte leftover from previous failed download
+ unlinkSync(filepath);
+ } catch {}
+ }
+
+ try {
+ const stream = item.file.download();
+ await pipeline(stream, createWriteStream(filepath));
+
+ // Verify the file was actually written
+ let actualSize = item.size;
+ try { actualSize = statSync(filepath).size; } catch {}
+
+ const folderName = basename(outputDir);
+ const ext = extname(item.name).toLowerCase();
+ const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
+ try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
+
+ completed++;
+ const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
+ logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
+ progressFn(completed + skipped, errors, items.length);
+ } catch (err) {
+ // Clean up partial/empty file on any error
+ try { unlinkSync(filepath); } catch {}
+
+ const waitSecs = parseBandwidthWait(err.message);
+ if (waitSecs > 0) {
+ // Bandwidth limit — put this item back and pause all workers
+ index = current; // rewind so this file gets retried
+ bandwidthPaused = true;
+ const waitMins = Math.ceil(waitSecs / 60);
+ const resumeAt = Date.now() + waitSecs * 1000;
+ logFn(`Bandwidth limit reached — waiting ${waitMins} minutes for quota reset...`);
+ if (statusFn) statusFn({ paused: true, resumeAt });
+ await new Promise(r => setTimeout(r, waitSecs * 1000));
+ if (checkCancelled()) return;
+ if (statusFn) statusFn({ paused: false, resumeAt: null });
+ logFn('Quota reset — resuming downloads...');
+ bandwidthPaused = false;
+ continue;
+ }
+
+ logFn(`FAILED: ${item.name} — ${err.message}`);
+ errors++;
+ progressFn(completed + skipped, errors, items.length);
+ }
+ }
+ }
+
+ const workerPromises = [];
+ for (let i = 0; i < Math.min(workers, items.length); i++) {
+ workerPromises.push(processNext());
+ }
+ await Promise.all(workerPromises);
+
+ // If we paused for bandwidth and there are remaining files, run single-threaded to finish
+ while (index < items.length && !checkCancelled()) {
+ const current = index++;
+ const item = items[current];
+ const filepath = join(outputDir, item.name);
+
+ if (existsSync(filepath)) {
+ try {
+ const st = statSync(filepath);
+ if (st.size > 0) {
+ skipped++;
+ progressFn(completed + skipped, errors, items.length);
+ continue;
+ }
+ unlinkSync(filepath);
+ } catch {}
+ }
+
+ try {
+ const stream = item.file.download();
+ await pipeline(stream, createWriteStream(filepath));
+
+ let actualSize = item.size;
+ try { actualSize = statSync(filepath).size; } catch {}
+
+ const folderName = basename(outputDir);
+ const ext = extname(item.name).toLowerCase();
+ const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
+ try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
+
+ completed++;
+ const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
+ logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
+ progressFn(completed + skipped, errors, items.length);
+ } catch (err) {
+ try { unlinkSync(filepath); } catch {}
+
+ const waitSecs = parseBandwidthWait(err.message);
+ if (waitSecs > 0) {
+ index = current;
+ const waitMins = Math.ceil(waitSecs / 60);
+ const resumeAt = Date.now() + waitSecs * 1000;
+ logFn(`Bandwidth limit reached — waiting ${waitMins} minutes...`);
+ if (statusFn) statusFn({ paused: true, resumeAt });
+ await new Promise(r => setTimeout(r, waitSecs * 1000));
+ if (checkCancelled()) break;
+ if (statusFn) statusFn({ paused: false, resumeAt: null });
+ logFn('Quota reset — resuming...');
+ continue;
+ }
+ logFn(`FAILED: ${item.name} — ${err.message}`);
+ errors++;
+ progressFn(completed + skipped, errors, items.length);
+ }
+ }
+
+ return { completed, errors, skipped, total: items.length };
+}
diff --git a/server/scrapers/ytdlp.js b/server/scrapers/ytdlp.js
new file mode 100644
index 0000000..1badcb2
--- /dev/null
+++ b/server/scrapers/ytdlp.js
@@ -0,0 +1,300 @@
+import { spawn } from 'child_process';
+import { basename, extname, join } from 'path';
+import { existsSync, statSync, readdirSync } from 'fs';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { insertVideo, getVideoByPath } from '../db.js';
+
+const execFileAsync = promisify(execFile);
+const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
+
+// Quality presets mapped to yt-dlp format strings
+const QUALITY_PRESETS = {
+ best: 'bestvideo+bestaudio/best',
+ '2160p': 'bestvideo[height<=2160]+bestaudio/best[height<=2160]',
+ '1080p': 'bestvideo[height<=1080]+bestaudio/best[height<=1080]',
+ '720p': 'bestvideo[height<=720]+bestaudio/best[height<=720]',
+ '480p': 'bestvideo[height<=480]+bestaudio/best[height<=480]',
+ audio: 'bestaudio/best',
+};
+
+async function probeVideo(filePath) {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error',
+ '-show_entries', 'format=duration,bit_rate',
+ '-show_entries', 'stream=codec_name,width,height,r_frame_rate,codec_type',
+ '-of', 'json',
+ filePath,
+ ], { timeout: 60000 });
+
+ const info = JSON.parse(stdout);
+ const videoStream = info.streams?.find(s => s.codec_type === 'video');
+ const audioStream = info.streams?.find(s => s.codec_type === 'audio');
+ const duration = parseFloat(info.format?.duration || '0');
+ const bitrate = parseInt(info.format?.bit_rate || '0', 10);
+
+ let fps = null;
+ if (videoStream?.r_frame_rate) {
+ const [num, den] = videoStream.r_frame_rate.split('/');
+ if (den && parseInt(den, 10) > 0) {
+ fps = Math.round((parseInt(num, 10) / parseInt(den, 10)) * 100) / 100;
+ }
+ }
+
+ return {
+ duration: duration || null,
+ width: videoStream?.width || null,
+ height: videoStream?.height || null,
+ fps,
+ codec: videoStream?.codec_name || null,
+ bitrate: bitrate || null,
+ has_audio: audioStream ? 1 : 0,
+ };
+}
+
+async function generateThumbnail(filePath) {
+ const thumbDir = join(VIDEOS_PATH, '.thumbnails');
+ const filename = basename(filePath);
+ const thumbName = `${Date.now()}_${filename.replace(/\.[^.]+$/, '.jpg')}`;
+ const thumbPath = join(thumbDir, thumbName);
+
+ let duration = 0;
+ try {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', filePath,
+ ], { timeout: 15000 });
+ duration = parseFloat(stdout.trim()) || 0;
+ } catch { /* ignore */ }
+
+ const seekTime = duration > 2 ? '1' : '0';
+
+ await execFileAsync('ffmpeg', [
+ '-ss', seekTime, '-i', filePath,
+ '-frames:v', '1', '-vf', 'scale=480:-1', '-q:v', '4', '-y', '-update', '1',
+ thumbPath,
+ ], { timeout: 30000 });
+
+ return thumbPath;
+}
+
+// Register a downloaded video file into the videos DB table
+async function registerVideo(filePath, log) {
+ try {
+ if (getVideoByPath(filePath)) {
+ log(`Already indexed: ${basename(filePath)}`);
+ return;
+ }
+
+ const stat = statSync(filePath);
+ const filename = basename(filePath);
+
+ let probe;
+ try {
+ probe = await probeVideo(filePath);
+ } catch (err) {
+ log(`Probe failed for ${filename}: ${err.message}`);
+ return;
+ }
+
+ let thumbPath = null;
+ try {
+ thumbPath = await generateThumbnail(filePath);
+ } catch { /* ignore */ }
+
+ const title = basename(filename, extname(filename))
+ .replace(/[_.-]/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+
+ insertVideo({
+ title,
+ filename,
+ file_path: filePath,
+ file_size: stat.size,
+ ...probe,
+ thumbnail_path: thumbPath,
+ status: 'ready',
+ });
+
+ log(`Registered in library: ${title}`);
+ } catch (err) {
+ log(`Failed to register ${basename(filePath)}: ${err.message}`);
+ }
+}
+
+// Build yt-dlp arguments from config
+function buildArgs(config) {
+ const { url, quality, customFormat, embedMetadata, embedThumbnail, embedSubs,
+ writeSubs, subLangs, restrictFilenames, outputTemplate,
+ playlist, maxDownloads, concurrentFragments, rateLimit,
+ sponsorBlock, cookiesFile } = config;
+
+ const args = [];
+
+ // Format
+ if (customFormat) {
+ args.push('-f', customFormat);
+ } else {
+ args.push('-f', QUALITY_PRESETS[quality] || QUALITY_PRESETS.best);
+ }
+
+ // Merge to mp4 when possible
+ if (quality !== 'audio') {
+ args.push('--merge-output-format', 'mp4');
+ } else {
+ args.push('-x', '--audio-format', 'mp3');
+ }
+
+ // Embed options
+ if (embedMetadata) args.push('--embed-metadata');
+ if (embedThumbnail) args.push('--embed-thumbnail');
+ if (embedSubs) args.push('--embed-subs');
+ if (writeSubs) args.push('--write-subs');
+ if (subLangs) args.push('--sub-langs', subLangs);
+
+ // Filename
+ if (restrictFilenames) args.push('--restrict-filenames');
+ args.push('-o', join(VIDEOS_PATH, outputTemplate || '%(title)s.%(ext)s'));
+
+ // Playlist
+ if (playlist) {
+ args.push('--yes-playlist');
+ if (maxDownloads) args.push('--max-downloads', String(maxDownloads));
+ } else {
+ args.push('--no-playlist');
+ }
+
+ // Performance
+ if (concurrentFragments && concurrentFragments > 1) {
+ args.push('--concurrent-fragments', String(concurrentFragments));
+ }
+ if (rateLimit) args.push('--rate-limit', rateLimit);
+
+ // SponsorBlock
+ if (sponsorBlock === 'remove') args.push('--sponsorblock-remove', 'all');
+ else if (sponsorBlock === 'mark') args.push('--sponsorblock-mark', 'all');
+
+ // Cookies
+ if (cookiesFile) args.push('--cookies', cookiesFile);
+
+ // Progress & output
+ args.push('--newline', '--no-colors', '--no-overwrites');
+ // Print downloaded file paths
+ args.push('--print', 'after_move:filepath');
+
+ args.push(url);
+ return args;
+}
+
+// Run yt-dlp download. Returns a promise. Progress/logs via callbacks.
+export function runYtdlp(config, log, onProgress, isCancelled) {
+ return new Promise((resolve, reject) => {
+ const args = buildArgs(config);
+ log(`yt-dlp ${args.join(' ')}`);
+
+ const proc = spawn('yt-dlp', args, {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+
+ const downloadedFiles = [];
+ let currentFile = '';
+ let fileCount = 0;
+
+ proc.stdout.on('data', (data) => {
+ const lines = data.toString().split('\n').filter(Boolean);
+ for (const line of lines) {
+ // yt-dlp --print after_move:filepath outputs the final file path on its own line
+ // These lines don't start with [ and are absolute paths
+ if (line.startsWith('/') && existsSync(line.trim())) {
+ const filePath = line.trim();
+ if (!downloadedFiles.includes(filePath)) {
+ downloadedFiles.push(filePath);
+ fileCount++;
+ onProgress(fileCount, 0);
+ log(`Downloaded: ${basename(filePath)}`);
+ }
+ continue;
+ }
+
+ // Parse progress lines: [download] 45.2% of 250.00MiB at 5.00MiB/s ETA 00:25
+ const progressMatch = line.match(/\[download\]\s+([\d.]+)%\s+of\s+~?([\d.]+\w+)\s+at\s+([\d.]+\w+\/s|Unknown)\s+ETA\s+(\S+)/);
+ if (progressMatch) {
+ const pct = parseFloat(progressMatch[1]);
+ const size = progressMatch[2];
+ const speed = progressMatch[3];
+ const eta = progressMatch[4];
+ log(`[download] ${pct.toFixed(1)}% of ${size} at ${speed} ETA ${eta}`);
+ continue;
+ }
+
+ // Destination line: [download] Destination: filename.mp4
+ const destMatch = line.match(/\[download\] Destination:\s+(.+)/);
+ if (destMatch) {
+ currentFile = basename(destMatch[1]);
+ log(`Downloading: ${currentFile}`);
+ continue;
+ }
+
+ // Already downloaded
+ if (line.includes('has already been downloaded')) {
+ log(line.trim());
+ onProgress(fileCount, 0);
+ continue;
+ }
+
+ // Log other yt-dlp output
+ if (line.trim()) {
+ log(line.trim());
+ }
+ }
+ });
+
+ proc.stderr.on('data', (data) => {
+ const lines = data.toString().split('\n').filter(Boolean);
+ for (const line of lines) {
+ if (line.includes('WARNING:')) {
+ log(`Warning: ${line.replace(/WARNING:\s*/, '')}`);
+ } else if (line.includes('ERROR:')) {
+ log(`ERROR: ${line.replace(/ERROR:\s*/, '')}`);
+ onProgress(fileCount, 1);
+ } else if (line.trim()) {
+ log(line.trim());
+ }
+ }
+ });
+
+ // Check for cancellation
+ const cancelCheck = setInterval(() => {
+ if (isCancelled()) {
+ proc.kill('SIGTERM');
+ clearInterval(cancelCheck);
+ }
+ }, 500);
+
+ proc.on('close', async (code) => {
+ clearInterval(cancelCheck);
+
+ // Register downloaded video files in the library
+ for (const filePath of downloadedFiles) {
+ const ext = extname(filePath).toLowerCase();
+ if (VIDEO_EXTS.has(ext)) {
+ await registerVideo(filePath, log);
+ }
+ }
+
+ if (code === 0) {
+ resolve({ files: downloadedFiles.length, errors: 0 });
+ } else if (isCancelled()) {
+ resolve({ files: downloadedFiles.length, errors: 0, cancelled: true });
+ } else {
+ resolve({ files: downloadedFiles.length, errors: 1 });
+ }
+ });
+
+ proc.on('error', (err) => {
+ clearInterval(cancelCheck);
+ reject(err);
+ });
+ });
+}
diff --git a/server/video-hls.js b/server/video-hls.js
new file mode 100644
index 0000000..8a83824
--- /dev/null
+++ b/server/video-hls.js
@@ -0,0 +1,434 @@
+import { Router } from 'express';
+import { join, extname } from 'path';
+import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
+import { execFile, spawn } from 'child_process';
+import { promisify } from 'util';
+import { getVideoById } from './db.js';
+
+const execFileAsync = promisify(execFile);
+const router = Router();
+
+const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
+const CACHE_DIR = join(VIDEOS_PATH, '.hls-cache');
+const SEGMENT_DURATION = 10;
+const MAX_CONCURRENT_TRANSCODES = 2;
+const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+// Ensure cache dir exists
+if (!existsSync(CACHE_DIR)) {
+ mkdirSync(CACHE_DIR, { recursive: true });
+}
+
+// Quality tiers (no "original" copy mode — always transcode for reliable HLS)
+const QUALITY_TIERS = {
+ '480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
+ '720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
+ '1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
+};
+
+// Compute output dimensions preserving source aspect ratio, clamped to maxW x maxH
+function fitDimensions(srcW, srcH, maxW, maxH) {
+ if (srcW <= maxW && srcH <= maxH) {
+ // Already fits — round to even
+ let w = srcW, h = srcH;
+ w += w % 2; h += h % 2;
+ return { w, h };
+ }
+ const scale = Math.min(maxW / srcW, maxH / srcH);
+ let w = Math.round(srcW * scale);
+ let h = Math.round(srcH * scale);
+ w += w % 2; h += h % 2; // ensure even (required for encoding)
+ return { w, h };
+}
+
+// Hardware acceleration detection: VAAPI > QSV > libx264
+// Alpine ships intel-media-driver (VA-API) but not oneVPL GPU runtime (QSV),
+// so VAAPI is the preferred HW accel path for Intel iGPUs.
+let hwAccel = null; // 'vaapi' | 'qsv' | null
+
+async function detectHwAccel() {
+ if (hwAccel !== null) return hwAccel;
+
+ // Try VAAPI first (works on Alpine with intel-media-driver)
+ try {
+ await execFileAsync('ffmpeg', [
+ '-hide_banner',
+ '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+ '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+ '-vf', 'format=nv12,hwupload',
+ '-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
+ ], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
+ hwAccel = 'vaapi';
+ console.log('[video-hls] Intel VAAPI hardware acceleration available');
+ return hwAccel;
+ } catch {
+ // VAAPI failed, try QSV
+ }
+
+ try {
+ await execFileAsync('ffmpeg', [
+ '-hide_banner', '-init_hw_device', 'qsv=hw',
+ '-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
+ '-vf', 'hwupload=extra_hw_frames=64,format=qsv',
+ '-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
+ ], { timeout: 10000 });
+ hwAccel = 'qsv';
+ console.log('[video-hls] Intel QSV hardware acceleration available');
+ return hwAccel;
+ } catch {
+ // QSV also failed
+ }
+
+ hwAccel = null;
+ console.log('[video-hls] No hardware acceleration available, using libx264 fallback');
+ return hwAccel;
+}
+
+// Detect on startup
+detectHwAccel();
+
+// --- Transcode semaphore ---
+
+let activeTranscodes = 0;
+const transcodeQueue = [];
+
+function acquireTranscodeSlot() {
+ return new Promise((resolve) => {
+ if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
+ activeTranscodes++;
+ resolve();
+ } else {
+ transcodeQueue.push(resolve);
+ }
+ });
+}
+
+function releaseTranscodeSlot() {
+ activeTranscodes--;
+ if (transcodeQueue.length > 0) {
+ activeTranscodes++;
+ transcodeQueue.shift()();
+ }
+}
+
+// --- Cache cleanup (hourly) ---
+
+function cleanupCache() {
+ try {
+ if (!existsSync(CACHE_DIR)) return;
+ const videoDirs = readdirSync(CACHE_DIR, { withFileTypes: true });
+ const now = Date.now();
+
+ for (const dir of videoDirs) {
+ if (!dir.isDirectory()) continue;
+ const videoCacheDir = join(CACHE_DIR, dir.name);
+
+ // Find newest segment mtime across all quality dirs
+ let newestMtime = 0;
+ try {
+ const qualityDirs = readdirSync(videoCacheDir, { withFileTypes: true });
+ for (const qDir of qualityDirs) {
+ if (!qDir.isDirectory()) continue;
+ const qPath = join(videoCacheDir, qDir.name);
+ const files = readdirSync(qPath);
+ for (const f of files) {
+ try {
+ const st = statSync(join(qPath, f));
+ if (st.mtimeMs > newestMtime) newestMtime = st.mtimeMs;
+ } catch { /* ignore */ }
+ }
+ }
+ } catch { continue; }
+
+ if (newestMtime > 0 && (now - newestMtime) > CACHE_MAX_AGE_MS) {
+ try {
+ rmSync(videoCacheDir, { recursive: true });
+ console.log(`[video-hls] Cleaned cache for video ${dir.name}`);
+ } catch { /* ignore */ }
+ }
+ }
+ } catch (err) {
+ console.error('[video-hls] Cache cleanup error:', err.message);
+ }
+}
+
+// Run cleanup every hour
+setInterval(cleanupCache, 60 * 60 * 1000);
+
+// --- Probe video duration ---
+
+async function getVideoDuration(filePath) {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error', '-show_entries', 'format=duration',
+ '-of', 'csv=p=0', filePath,
+ ], { timeout: 15000 });
+ return parseFloat(stdout.trim()) || 0;
+}
+
+async function getVideoInfo(filePath) {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error',
+ '-show_entries', 'stream=codec_type,width,height',
+ '-show_entries', 'format=duration',
+ '-of', 'json',
+ filePath,
+ ], { timeout: 15000 });
+ const info = JSON.parse(stdout);
+ const videoStream = info.streams?.find(s => s.codec_type === 'video');
+ return {
+ duration: parseFloat(info.format?.duration || '0'),
+ width: videoStream?.width || 0,
+ height: videoStream?.height || 0,
+ hasAudio: !!info.streams?.find(s => s.codec_type === 'audio'),
+ };
+}
+
+// --- Master playlist ---
+
+// GET /api/video-hls/:id/master.m3u8
+router.get('/api/video-hls/:id/master.m3u8', async (req, res) => {
+ try {
+ const id = parseInt(req.params.id, 10);
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+ if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+ const info = await getVideoInfo(video.file_path);
+ const sourceWidth = info.width || video.width || 1920;
+ const sourceHeight = info.height || video.height || 1080;
+
+ let playlist = '#EXTM3U\n';
+
+ // Add quality tiers at or below source resolution (all transcoded, aspect-ratio preserved)
+ for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
+ if (tier.maxH <= sourceHeight) {
+ const { w, h } = fitDimensions(sourceWidth, sourceHeight, tier.maxW, tier.maxH);
+ const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
+ playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
+ playlist += `${name}/playlist.m3u8\n`;
+ }
+ }
+
+ res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.send(playlist);
+ } catch (err) {
+ console.error('[video-hls] Master playlist error:', err.message);
+ res.status(500).json({ error: 'Failed to generate master playlist' });
+ }
+});
+
+// --- Variant playlist ---
+
+// GET /api/video-hls/:id/:quality/playlist.m3u8
+router.get('/api/video-hls/:id/:quality/playlist.m3u8', async (req, res) => {
+ try {
+ const id = parseInt(req.params.id, 10);
+ const { quality } = req.params;
+
+ if (!QUALITY_TIERS[quality]) {
+ return res.status(400).json({ error: 'Invalid quality' });
+ }
+
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+ if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+ const duration = await getVideoDuration(video.file_path);
+ if (!duration || duration <= 0) {
+ return res.status(500).json({ error: 'Could not determine video duration' });
+ }
+
+ const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
+
+ let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
+ playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
+ playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
+
+ for (let i = 0; i < segmentCount; i++) {
+ const remaining = duration - i * SEGMENT_DURATION;
+ const segDuration = Math.min(SEGMENT_DURATION, remaining);
+ playlist += `#EXTINF:${segDuration.toFixed(3)},\n`;
+ playlist += `segment-${i}.ts\n`;
+ }
+
+ playlist += '#EXT-X-ENDLIST\n';
+
+ res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.send(playlist);
+ } catch (err) {
+ console.error('[video-hls] Variant playlist error:', err.message);
+ res.status(500).json({ error: 'Failed to generate variant playlist' });
+ }
+});
+
+// --- Segment transcoding ---
+
+// GET /api/video-hls/:id/:quality/segment-:index.ts
+router.get('/api/video-hls/:id/:quality/segment-:index.ts', async (req, res) => {
+ try {
+ const id = parseInt(req.params.id, 10);
+ const { quality } = req.params;
+ const segIndex = parseInt(req.params.index, 10);
+
+ if (isNaN(segIndex) || segIndex < 0) {
+ return res.status(400).json({ error: 'Invalid segment index' });
+ }
+ if (!QUALITY_TIERS[quality]) {
+ return res.status(400).json({ error: 'Invalid quality' });
+ }
+
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+ if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
+
+ // Check cache first
+ const segmentCacheDir = join(CACHE_DIR, String(id), quality);
+ const segmentCachePath = join(segmentCacheDir, `segment-${segIndex}.ts`);
+
+ if (existsSync(segmentCachePath)) {
+ const stat = statSync(segmentCachePath);
+ res.writeHead(200, {
+ 'Content-Type': 'video/MP2T',
+ 'Content-Length': stat.size,
+ 'Cache-Control': 'public, max-age=3600',
+ });
+ createReadStream(segmentCachePath).pipe(res);
+ return;
+ }
+
+ // Transcode on-demand
+ await acquireTranscodeSlot();
+
+ // Check cache again after acquiring slot (another request may have cached it)
+ if (existsSync(segmentCachePath)) {
+ releaseTranscodeSlot();
+ const stat = statSync(segmentCachePath);
+ res.writeHead(200, {
+ 'Content-Type': 'video/MP2T',
+ 'Content-Length': stat.size,
+ 'Cache-Control': 'public, max-age=3600',
+ });
+ createReadStream(segmentCachePath).pipe(res);
+ return;
+ }
+
+ const offset = segIndex * SEGMENT_DURATION;
+ const accel = await detectHwAccel();
+ const tier = QUALITY_TIERS[quality];
+
+ // Compute output dimensions preserving source aspect ratio
+ const srcW = video.width || 1920;
+ const srcH = video.height || 1080;
+ const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
+
+ // -output_ts_offset ensures PTS continuity across segments (each segment's PTS
+ // starts where the previous one ended, required for smooth HLS playback)
+ let ffmpegArgs;
+ if (accel === 'vaapi') {
+ ffmpegArgs = [
+ '-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
+ '-filter_hw_device', 'va',
+ '-ss', String(offset),
+ '-i', video.file_path,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
+ '-c:v', 'h264_vaapi',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
+ } else if (accel === 'qsv') {
+ ffmpegArgs = [
+ '-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
+ '-ss', String(offset),
+ '-i', video.file_path,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `scale_qsv=w=${outW}:h=${outH}`,
+ '-c:v', 'h264_qsv',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
+ } else {
+ ffmpegArgs = [
+ '-ss', String(offset),
+ '-i', video.file_path,
+ '-t', String(SEGMENT_DURATION),
+ '-output_ts_offset', String(offset),
+ '-vf', `scale=${outW}:${outH}`,
+ '-c:v', 'libx264', '-preset', 'veryfast',
+ '-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
+ '-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
+ '-f', 'mpegts',
+ 'pipe:1',
+ ];
+ }
+
+ const spawnEnv = accel === 'vaapi'
+ ? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
+ : undefined;
+
+ const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
+ stdio: ['ignore', 'pipe', 'pipe'],
+ ...(spawnEnv && { env: spawnEnv }),
+ });
+
+ // Stream directly to client while also collecting chunks for cache
+ res.setHeader('Content-Type', 'video/MP2T');
+ res.setHeader('Cache-Control', 'public, max-age=3600');
+
+ const cacheChunks = [];
+ let aborted = false;
+
+ ffmpeg.stdout.on('data', (chunk) => {
+ cacheChunks.push(chunk);
+ if (!res.destroyed) res.write(chunk);
+ });
+
+ req.on('close', () => {
+ if (!ffmpeg.killed) {
+ aborted = true;
+ ffmpeg.kill('SIGKILL');
+ releaseTranscodeSlot();
+ }
+ });
+
+ ffmpeg.on('close', (code) => {
+ if (aborted) return;
+ releaseTranscodeSlot();
+
+ // Write cache file on success
+ if (code === 0 && cacheChunks.length > 0) {
+ try {
+ if (!existsSync(segmentCacheDir)) mkdirSync(segmentCacheDir, { recursive: true });
+ const cacheStream = createWriteStream(segmentCachePath);
+ for (const chunk of cacheChunks) cacheStream.write(chunk);
+ cacheStream.end();
+ } catch { /* ignore cache write failure */ }
+ }
+
+ if (!res.destroyed) res.end();
+ });
+
+ ffmpeg.on('error', (err) => {
+ if (!aborted) releaseTranscodeSlot();
+ console.error('[video-hls] ffmpeg error:', err.message);
+ if (!res.headersSent) {
+ res.status(500).json({ error: 'Transcoding failed' });
+ }
+ });
+ } catch (err) {
+ console.error('[video-hls] Segment error:', err.message);
+ if (!res.headersSent) {
+ res.status(500).json({ error: err.message });
+ }
+ }
+});
+
+export default router;
diff --git a/server/videos.js b/server/videos.js
new file mode 100644
index 0000000..60287b4
--- /dev/null
+++ b/server/videos.js
@@ -0,0 +1,445 @@
+import { Router } from 'express';
+import multer from 'multer';
+import { join, extname, basename } from 'path';
+import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync, createReadStream, rmSync } from 'fs';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import {
+ insertVideo, getVideoById, getVideoByPath, updateVideo, deleteVideoById,
+ searchVideos, getOrCreateTag, getAllTags, setVideoTags, getVideoTags,
+} from './db.js';
+
+const execFileAsync = promisify(execFile);
+const router = Router();
+
+const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
+const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
+
+// Ensure videos dir exists
+if (!existsSync(VIDEOS_PATH)) {
+ mkdirSync(VIDEOS_PATH, { recursive: true });
+}
+
+// Multer config for uploads
+const storage = multer.diskStorage({
+ destination: (req, file, cb) => cb(null, VIDEOS_PATH),
+ filename: (req, file, cb) => {
+ // Preserve original name, avoid collisions
+ let name = file.originalname;
+ const filePath = join(VIDEOS_PATH, name);
+ if (existsSync(filePath)) {
+ const ext = extname(name);
+ const base = basename(name, ext);
+ name = `${base}_${Date.now()}${ext}`;
+ }
+ cb(null, name);
+ },
+});
+
+const upload = multer({
+ storage,
+ fileFilter: (req, file, cb) => {
+ const ext = extname(file.originalname).toLowerCase();
+ if (VIDEO_EXTS.has(ext)) {
+ cb(null, true);
+ } else {
+ cb(new Error(`Unsupported file type: ${ext}`));
+ }
+ },
+ limits: { fileSize: 50 * 1024 * 1024 * 1024 }, // 50 GB
+});
+
+// --- ffprobe helper ---
+
+async function probeVideo(filePath) {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error',
+ '-show_entries', 'format=duration,bit_rate',
+ '-show_entries', 'stream=codec_name,width,height,r_frame_rate,codec_type',
+ '-of', 'json',
+ filePath,
+ ], { timeout: 60000 });
+
+ const info = JSON.parse(stdout);
+ const videoStream = info.streams?.find(s => s.codec_type === 'video');
+ const audioStream = info.streams?.find(s => s.codec_type === 'audio');
+ const duration = parseFloat(info.format?.duration || '0');
+ const bitrate = parseInt(info.format?.bit_rate || '0', 10);
+
+ let fps = null;
+ if (videoStream?.r_frame_rate) {
+ const [num, den] = videoStream.r_frame_rate.split('/');
+ if (den && parseInt(den, 10) > 0) {
+ fps = Math.round((parseInt(num, 10) / parseInt(den, 10)) * 100) / 100;
+ }
+ }
+
+ return {
+ duration: duration || null,
+ width: videoStream?.width || null,
+ height: videoStream?.height || null,
+ fps,
+ codec: videoStream?.codec_name || null,
+ bitrate: bitrate || null,
+ has_audio: audioStream ? 1 : 0,
+ };
+}
+
+// --- thumbnail generation ---
+
+async function generateVideoThumbnail(filePath, outputPath) {
+ const dir = join(VIDEOS_PATH, '.thumbnails');
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+ // Seek 1s in for a better frame
+ let duration = 0;
+ try {
+ const { stdout } = await execFileAsync('ffprobe', [
+ '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', filePath,
+ ], { timeout: 15000 });
+ duration = parseFloat(stdout.trim()) || 0;
+ } catch { /* ignore */ }
+
+ const seekTime = duration > 2 ? '1' : '0';
+
+ await execFileAsync('ffmpeg', [
+ '-ss', seekTime,
+ '-i', filePath,
+ '-frames:v', '1',
+ '-vf', 'scale=480:-1',
+ '-q:v', '4',
+ '-y',
+ '-update', '1',
+ outputPath,
+ ], { timeout: 30000 });
+
+ return outputPath;
+}
+
+// --- Scan state ---
+
+let scanState = { running: false, total: 0, done: 0, added: 0, skipped: 0, errors: 0 };
+
+// GET /api/videos — browse/search
+router.get('/api/videos', (req, res, next) => {
+ try {
+ const { search, sort, offset, limit, minDuration, maxDuration, minWidth } = req.query;
+ const tagsParam = req.query.tags;
+ const tagsArr = tagsParam
+ ? tagsParam.split(',').map(t => t.trim()).filter(Boolean)
+ : undefined;
+
+ const result = searchVideos({
+ search: search || undefined,
+ tags: tagsArr,
+ minDuration: minDuration || undefined,
+ maxDuration: maxDuration || undefined,
+ minWidth: minWidth || undefined,
+ sort: sort || 'latest',
+ offset: parseInt(offset || '0', 10),
+ limit: parseInt(limit || '48', 10),
+ });
+
+ // Attach tags to each video
+ const videos = result.rows.map(v => ({
+ ...v,
+ tags: getVideoTags(v.id),
+ }));
+
+ res.json({ total: result.total, offset: parseInt(offset || '0', 10), videos });
+ } catch (err) {
+ next(err);
+ }
+});
+
+// GET /api/videos/tags — all tags with counts
+router.get('/api/videos/tags', (req, res, next) => {
+ try {
+ const { search } = req.query;
+ const tags = getAllTags(search || undefined);
+ res.json(tags);
+ } catch (err) {
+ next(err);
+ }
+});
+
+// GET /api/videos/scan/status
+router.get('/api/videos/scan/status', (req, res) => {
+ res.json(scanState);
+});
+
+// GET /api/videos/:id
+router.get('/api/videos/:id', (req, res, next) => {
+ try {
+ const video = getVideoById(parseInt(req.params.id, 10));
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+ video.tags = getVideoTags(video.id);
+ res.json(video);
+ } catch (err) {
+ next(err);
+ }
+});
+
+// PUT /api/videos/:id — update title, description, tags
+router.put('/api/videos/:id', (req, res, next) => {
+ try {
+ const id = parseInt(req.params.id, 10);
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+
+ const { title, description, tags } = req.body;
+ const updates = {};
+ if (title !== undefined) updates.title = title;
+ if (description !== undefined) updates.description = description;
+
+ if (Object.keys(updates).length > 0) {
+ updateVideo(id, updates);
+ }
+ if (Array.isArray(tags)) {
+ setVideoTags(id, tags);
+ }
+
+ const updated = getVideoById(id);
+ updated.tags = getVideoTags(id);
+ res.json(updated);
+ } catch (err) {
+ next(err);
+ }
+});
+
+// DELETE /api/videos/:id
+router.delete('/api/videos/:id', (req, res, next) => {
+ try {
+ const id = parseInt(req.params.id, 10);
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+
+ // Delete file
+ if (existsSync(video.file_path)) {
+ try { unlinkSync(video.file_path); } catch { /* ignore */ }
+ }
+
+ // Delete thumbnail
+ if (video.thumbnail_path && existsSync(video.thumbnail_path)) {
+ try { unlinkSync(video.thumbnail_path); } catch { /* ignore */ }
+ }
+
+ // Delete HLS cache
+ const hlsCacheDir = join(VIDEOS_PATH, '.hls-cache', String(id));
+ if (existsSync(hlsCacheDir)) {
+ try { rmSync(hlsCacheDir, { recursive: true }); } catch { /* ignore */ }
+ }
+
+ deleteVideoById(id);
+ res.json({ ok: true });
+ } catch (err) {
+ next(err);
+ }
+});
+
+// POST /api/videos/upload — multipart file upload
+router.post('/api/videos/upload', upload.single('video'), async (req, res) => {
+ if (!req.file) {
+ return res.status(400).json({ error: 'No video file provided' });
+ }
+
+ const filePath = req.file.path;
+ const filename = req.file.filename;
+
+ try {
+ // Check for dupe
+ const existing = getVideoByPath(filePath);
+ if (existing) {
+ return res.json({ video: existing, duplicate: true });
+ }
+
+ const stat = statSync(filePath);
+ const probe = await probeVideo(filePath);
+
+ // Generate thumbnail
+ const thumbName = filename.replace(/\.[^.]+$/, '.jpg');
+ const thumbPath = join(VIDEOS_PATH, '.thumbnails', thumbName);
+ let thumbResult = null;
+ try {
+ thumbResult = await generateVideoThumbnail(filePath, thumbPath);
+ } catch { /* ignore */ }
+
+ const title = basename(filename, extname(filename))
+ .replace(/[_.-]/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+
+ const videoId = insertVideo({
+ title,
+ filename,
+ file_path: filePath,
+ file_size: stat.size,
+ ...probe,
+ thumbnail_path: thumbResult || null,
+ status: 'ready',
+ });
+
+ const video = getVideoById(videoId);
+ video.tags = [];
+ res.json({ video });
+ } catch (err) {
+ console.error('[videos] Upload processing failed:', err.message);
+ res.status(500).json({ error: err.message });
+ }
+});
+
+// POST /api/videos/scan — scan VIDEOS_PATH for new files
+router.post('/api/videos/scan', (req, res) => {
+ if (scanState.running) {
+ return res.json({ status: 'already_running', ...scanState });
+ }
+
+ scanState = { running: true, total: 0, done: 0, added: 0, skipped: 0, errors: 0 };
+ res.json({ status: 'started' });
+
+ setImmediate(async () => {
+ try {
+ // Collect all video files
+ const videoFiles = [];
+ const collectFiles = (dir) => {
+ let entries;
+ try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const entry of entries) {
+ if (entry.name.startsWith('.')) continue;
+ const fullPath = join(dir, entry.name);
+ if (entry.isDirectory()) {
+ collectFiles(fullPath);
+ } else {
+ const ext = extname(entry.name).toLowerCase();
+ if (VIDEO_EXTS.has(ext)) {
+ videoFiles.push(fullPath);
+ }
+ }
+ }
+ };
+ collectFiles(VIDEOS_PATH);
+
+ scanState.total = videoFiles.length;
+ console.log(`[videos] Scan found ${videoFiles.length} video files`);
+
+ for (const filePath of videoFiles) {
+ try {
+ // Skip if already indexed
+ const existing = getVideoByPath(filePath);
+ if (existing) {
+ scanState.skipped++;
+ scanState.done++;
+ continue;
+ }
+
+ const stat = statSync(filePath);
+ const filename = basename(filePath);
+
+ // Probe metadata
+ let probe;
+ try {
+ probe = await probeVideo(filePath);
+ } catch (err) {
+ console.error(`[videos] Probe failed for ${filename}:`, err.message);
+ scanState.errors++;
+ scanState.done++;
+ continue;
+ }
+
+ // Generate thumbnail
+ const thumbName = `${Date.now()}_${filename.replace(/\.[^.]+$/, '.jpg')}`;
+ const thumbPath = join(VIDEOS_PATH, '.thumbnails', thumbName);
+ let thumbResult = null;
+ try {
+ thumbResult = await generateVideoThumbnail(filePath, thumbPath);
+ } catch { /* ignore */ }
+
+ const title = basename(filename, extname(filename))
+ .replace(/[_.-]/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+
+ insertVideo({
+ title,
+ filename,
+ file_path: filePath,
+ file_size: stat.size,
+ ...probe,
+ thumbnail_path: thumbResult || null,
+ status: 'ready',
+ });
+
+ scanState.added++;
+ scanState.done++;
+ } catch (err) {
+ console.error(`[videos] Scan error for ${filePath}:`, err.message);
+ scanState.errors++;
+ scanState.done++;
+ }
+ }
+
+ console.log(`[videos] Scan complete: ${scanState.added} added, ${scanState.skipped} skipped, ${scanState.errors} errors`);
+ } catch (err) {
+ console.error('[videos] Scan failed:', err.message);
+ } finally {
+ scanState.running = false;
+ }
+ });
+});
+
+// GET /api/videos/:id/thumbnail
+router.get('/api/videos/:id/thumbnail', (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+
+ if (video.thumbnail_path && existsSync(video.thumbnail_path)) {
+ const stat = statSync(video.thumbnail_path);
+ res.writeHead(200, {
+ 'Content-Type': 'image/jpeg',
+ 'Content-Length': stat.size,
+ 'Cache-Control': 'public, max-age=86400',
+ });
+ createReadStream(video.thumbnail_path).pipe(res);
+ } else {
+ // Return a placeholder
+ res.status(404).json({ error: 'No thumbnail' });
+ }
+});
+
+// GET /api/videos/:id/stream — direct file serve for grid wall playback
+router.get('/api/videos/:id/stream', (req, res) => {
+ const id = parseInt(req.params.id, 10);
+ const video = getVideoById(id);
+ if (!video) return res.status(404).json({ error: 'Video not found' });
+ if (!existsSync(video.file_path)) return res.status(404).json({ error: 'File not found' });
+
+ const stat = statSync(video.file_path);
+ const ext = extname(video.file_path).toLowerCase();
+ const mimeTypes = { '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime', '.mkv': 'video/x-matroska', '.m4v': 'video/mp4' };
+ const contentType = mimeTypes[ext] || 'video/mp4';
+
+ // Support range requests
+ const range = req.headers.range;
+ if (range) {
+ const parts = range.replace(/bytes=/, '').split('-');
+ const start = parseInt(parts[0], 10);
+ const end = parts[1] ? parseInt(parts[1], 10) : stat.size - 1;
+ res.writeHead(206, {
+ 'Content-Range': `bytes ${start}-${end}/${stat.size}`,
+ 'Accept-Ranges': 'bytes',
+ 'Content-Length': end - start + 1,
+ 'Content-Type': contentType,
+ });
+ createReadStream(video.file_path, { start, end }).pipe(res);
+ } else {
+ res.writeHead(200, {
+ 'Content-Length': stat.size,
+ 'Content-Type': contentType,
+ 'Accept-Ranges': 'bytes',
+ });
+ createReadStream(video.file_path).pipe(res);
+ }
+});
+
+export default router;