Add app auth, dashboard, scheduler, video management, and new scrapers
- JWT-based app authentication with user roles, folder/route access control - Dashboard with storage stats, health checks, and recent activity - Auto-download/scrape scheduler (12h interval) with per-user and per-job configs - Video upload, tagging, HLS transcoding, and detail pages - New scrapers: LeakGallery, Mega (megajs), yt-dlp - FlareSolverr integration for Cloudflare-protected sites - Gallery: advanced filtering (date, size, search), sort modes, equal-mix shuffle - Forum sites management with stored cookies/auth - GridWall/GridCell components for responsive media grid - Media API with folder-access permissions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Trey T
+355
@@ -0,0 +1,355 @@
|
||||
import { Router } from 'express';
|
||||
import bcrypt from 'bcryptjs';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import crypto from 'crypto';
|
||||
import {
|
||||
getSetting, setSetting,
|
||||
getAppUserCount, createAppUser, getAppUserByUsername, getAppUserById,
|
||||
getAllAppUsers, updateAppUser, deleteAppUser,
|
||||
getUserFolderAccess, setUserFolderAccess,
|
||||
getUserRouteAccess, setUserRouteAccess,
|
||||
getAllIndexedFolders,
|
||||
} from './db.js';
|
||||
|
||||
const router = Router();
|
||||
const TOKEN_COOKIE = 'ofapp_token';
|
||||
const TOKEN_EXPIRY = undefined; // no expiration
|
||||
|
||||
function getJwtSecret() {
|
||||
let secret = getSetting('jwt_secret');
|
||||
if (!secret) {
|
||||
secret = crypto.randomBytes(48).toString('hex');
|
||||
setSetting('jwt_secret', secret);
|
||||
console.log('[auth] Generated new JWT secret');
|
||||
}
|
||||
return secret;
|
||||
}
|
||||
|
||||
function signToken(userId) {
|
||||
return jwt.sign({ userId }, getJwtSecret(), { expiresIn: TOKEN_EXPIRY });
|
||||
}
|
||||
|
||||
function setTokenCookie(res, token) {
|
||||
res.cookie(TOKEN_COOKIE, token, {
|
||||
httpOnly: true,
|
||||
sameSite: 'lax',
|
||||
maxAge: 10 * 365 * 24 * 60 * 60 * 1000, // ~10 years
|
||||
secure: false, // allow HTTP (self-signed HTTPS + local network)
|
||||
});
|
||||
}
|
||||
|
||||
function userPayload(user, routes, folders) {
|
||||
return {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
display_name: user.display_name,
|
||||
role: user.role,
|
||||
enabled: user.enabled,
|
||||
routes,
|
||||
folders: user.role === 'admin' ? null : folders, // null = all access
|
||||
};
|
||||
}
|
||||
|
||||
// --- Middleware ---
|
||||
|
||||
export function requireAuth(req, res, next) {
|
||||
// Setup mode: if no users exist, allow all requests as synthetic admin
|
||||
const userCount = getAppUserCount();
|
||||
if (userCount === 0) {
|
||||
req.user = { id: 0, username: 'setup', role: 'admin', enabled: 1 };
|
||||
return next();
|
||||
}
|
||||
|
||||
const token = req.cookies?.[TOKEN_COOKIE];
|
||||
if (!token) {
|
||||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = jwt.verify(token, getJwtSecret());
|
||||
const user = getAppUserById(decoded.userId);
|
||||
if (!user || !user.enabled) {
|
||||
return res.status(401).json({ error: 'Account disabled or not found' });
|
||||
}
|
||||
|
||||
req.user = user;
|
||||
|
||||
next();
|
||||
} catch (err) {
|
||||
return res.status(401).json({ error: 'Invalid or expired token' });
|
||||
}
|
||||
}
|
||||
|
||||
export function requireAdmin(req, res, next) {
|
||||
if (req.user?.role !== 'admin') {
|
||||
return res.status(403).json({ error: 'Admin access required' });
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
// Route permission map: API path prefix -> route key
|
||||
const ROUTE_PERMISSION_MAP = {
|
||||
'/api/feed': 'feed',
|
||||
'/api/subscriptions': 'users',
|
||||
'/api/users': 'users',
|
||||
'/api/download': 'downloads',
|
||||
'/api/gallery': 'gallery',
|
||||
'/api/hls': 'gallery',
|
||||
'/api/scrape': 'scrape',
|
||||
'/api/settings': 'settings',
|
||||
'/api/auth': 'settings',
|
||||
'/api/dashboard': 'dashboard',
|
||||
'/api/health': 'dashboard',
|
||||
'/api/videos': 'videos',
|
||||
'/api/video-hls': 'videos',
|
||||
'/api/drm': null,
|
||||
'/api/media-proxy': null,
|
||||
'/api/me': null,
|
||||
'/api/admin': null, // handled by requireAdmin
|
||||
'/api/app-auth': null, // public
|
||||
};
|
||||
|
||||
export function checkRoutePermission(req, res, next) {
|
||||
// Admins bypass all route checks
|
||||
if (req.user?.role === 'admin') return next();
|
||||
|
||||
// Find matching route key
|
||||
const path = req.path;
|
||||
let routeKey = undefined;
|
||||
for (const [prefix, key] of Object.entries(ROUTE_PERMISSION_MAP)) {
|
||||
if (path.startsWith(prefix)) {
|
||||
routeKey = key;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// null = always allowed for authenticated users
|
||||
if (routeKey === null || routeKey === undefined) return next();
|
||||
|
||||
// Check user's route access
|
||||
const userRoutes = getUserRouteAccess(req.user.id);
|
||||
if (!userRoutes.includes(routeKey)) {
|
||||
return res.status(403).json({ error: 'Access denied' });
|
||||
}
|
||||
|
||||
next();
|
||||
}
|
||||
|
||||
// --- Public endpoints (no auth required) ---
|
||||
|
||||
// GET /api/app-auth/status — check if setup is needed
|
||||
router.get('/api/app-auth/status', (req, res) => {
|
||||
const count = getAppUserCount();
|
||||
res.json({ setupRequired: count === 0 });
|
||||
});
|
||||
|
||||
// POST /api/app-auth/setup — create initial admin (only when 0 users exist)
|
||||
router.post('/api/app-auth/setup', (req, res) => {
|
||||
const count = getAppUserCount();
|
||||
if (count > 0) {
|
||||
return res.status(400).json({ error: 'Setup already completed' });
|
||||
}
|
||||
|
||||
const { username, password } = req.body;
|
||||
if (!username || !password) {
|
||||
return res.status(400).json({ error: 'Username and password required' });
|
||||
}
|
||||
if (password.length < 4) {
|
||||
return res.status(400).json({ error: 'Password must be at least 4 characters' });
|
||||
}
|
||||
|
||||
const hash = bcrypt.hashSync(password, 10);
|
||||
const userId = createAppUser(username.trim(), hash, 'admin', username.trim());
|
||||
const user = getAppUserById(userId);
|
||||
const token = signToken(userId);
|
||||
setTokenCookie(res, token);
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
user: userPayload(user, [], null),
|
||||
});
|
||||
});
|
||||
|
||||
// POST /api/app-auth/login — validate creds, set JWT cookie
|
||||
router.post('/api/app-auth/login', (req, res) => {
|
||||
const { username, password } = req.body;
|
||||
if (!username || !password) {
|
||||
return res.status(400).json({ error: 'Username and password required' });
|
||||
}
|
||||
|
||||
const user = getAppUserByUsername(username.trim());
|
||||
if (!user) {
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
if (!user.enabled) {
|
||||
return res.status(401).json({ error: 'Account is disabled' });
|
||||
}
|
||||
if (!bcrypt.compareSync(password, user.password_hash)) {
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
|
||||
const routes = getUserRouteAccess(user.id);
|
||||
const folders = getUserFolderAccess(user.id);
|
||||
const token = signToken(user.id);
|
||||
setTokenCookie(res, token);
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
user: userPayload(user, routes, folders),
|
||||
});
|
||||
});
|
||||
|
||||
// POST /api/app-auth/logout — clear JWT cookie
|
||||
router.post('/api/app-auth/logout', (req, res) => {
|
||||
res.clearCookie(TOKEN_COOKIE);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// --- Authenticated endpoints ---
|
||||
|
||||
// GET /api/app-auth/me — current user + permissions
|
||||
router.get('/api/app-auth/me', requireAuth, (req, res) => {
|
||||
if (!req.user || req.user.id === 0) {
|
||||
return res.json({ setupRequired: true });
|
||||
}
|
||||
|
||||
const routes = getUserRouteAccess(req.user.id);
|
||||
const folders = getUserFolderAccess(req.user.id);
|
||||
res.json({ user: userPayload(req.user, routes, folders) });
|
||||
});
|
||||
|
||||
// --- Admin-only endpoints ---
|
||||
|
||||
// GET /api/admin/users — list all users with permissions
|
||||
router.get('/api/admin/users', requireAuth, requireAdmin, (req, res) => {
|
||||
const users = getAllAppUsers();
|
||||
const result = users.map((u) => ({
|
||||
...u,
|
||||
routes: getUserRouteAccess(u.id),
|
||||
folders: getUserFolderAccess(u.id),
|
||||
}));
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
// POST /api/admin/users — create user
|
||||
router.post('/api/admin/users', requireAuth, requireAdmin, (req, res) => {
|
||||
const { username, password, role, display_name, routes, folders } = req.body;
|
||||
if (!username || !password) {
|
||||
return res.status(400).json({ error: 'Username and password required' });
|
||||
}
|
||||
if (password.length < 4) {
|
||||
return res.status(400).json({ error: 'Password must be at least 4 characters' });
|
||||
}
|
||||
|
||||
const existing = getAppUserByUsername(username.trim());
|
||||
if (existing) {
|
||||
return res.status(400).json({ error: 'Username already exists' });
|
||||
}
|
||||
|
||||
const hash = bcrypt.hashSync(password, 10);
|
||||
const userId = createAppUser(username.trim(), hash, role || 'user', display_name || username.trim());
|
||||
|
||||
if (routes && Array.isArray(routes)) {
|
||||
setUserRouteAccess(userId, routes);
|
||||
}
|
||||
if (folders && Array.isArray(folders)) {
|
||||
setUserFolderAccess(userId, folders);
|
||||
}
|
||||
|
||||
const user = getAppUserById(userId);
|
||||
res.json({
|
||||
...user,
|
||||
routes: getUserRouteAccess(userId),
|
||||
folders: getUserFolderAccess(userId),
|
||||
});
|
||||
});
|
||||
|
||||
// PUT /api/admin/users/:id — update user
|
||||
router.put('/api/admin/users/:id', requireAuth, requireAdmin, (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const target = getAppUserById(id);
|
||||
if (!target) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
|
||||
const { username, password, role, display_name, enabled, routes, folders } = req.body;
|
||||
|
||||
// Prevent demoting/disabling last admin
|
||||
if (target.role === 'admin' && (role === 'user' || enabled === 0)) {
|
||||
const allUsers = getAllAppUsers();
|
||||
const adminCount = allUsers.filter(u => u.role === 'admin' && u.enabled).length;
|
||||
if (adminCount <= 1) {
|
||||
return res.status(400).json({ error: 'Cannot demote or disable the last admin' });
|
||||
}
|
||||
}
|
||||
|
||||
const fields = {};
|
||||
if (username !== undefined) fields.username = username.trim();
|
||||
if (display_name !== undefined) fields.display_name = display_name;
|
||||
if (role !== undefined) fields.role = role;
|
||||
if (enabled !== undefined) fields.enabled = enabled;
|
||||
if (password) {
|
||||
if (password.length < 4) {
|
||||
return res.status(400).json({ error: 'Password must be at least 4 characters' });
|
||||
}
|
||||
fields.password_hash = bcrypt.hashSync(password, 10);
|
||||
}
|
||||
|
||||
if (Object.keys(fields).length > 0) {
|
||||
updateAppUser(id, fields);
|
||||
}
|
||||
|
||||
if (routes !== undefined && Array.isArray(routes)) {
|
||||
setUserRouteAccess(id, routes);
|
||||
}
|
||||
if (folders !== undefined && Array.isArray(folders)) {
|
||||
setUserFolderAccess(id, folders);
|
||||
}
|
||||
|
||||
const updated = getAppUserById(id);
|
||||
res.json({
|
||||
id: updated.id,
|
||||
username: updated.username,
|
||||
display_name: updated.display_name,
|
||||
role: updated.role,
|
||||
enabled: updated.enabled,
|
||||
created_at: updated.created_at,
|
||||
updated_at: updated.updated_at,
|
||||
routes: getUserRouteAccess(id),
|
||||
folders: getUserFolderAccess(id),
|
||||
});
|
||||
});
|
||||
|
||||
// DELETE /api/admin/users/:id — delete user
|
||||
router.delete('/api/admin/users/:id', requireAuth, requireAdmin, (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const target = getAppUserById(id);
|
||||
if (!target) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
|
||||
// Cannot delete self
|
||||
if (req.user.id === id) {
|
||||
return res.status(400).json({ error: 'Cannot delete your own account' });
|
||||
}
|
||||
|
||||
// Cannot delete last admin
|
||||
if (target.role === 'admin') {
|
||||
const allUsers = getAllAppUsers();
|
||||
const adminCount = allUsers.filter(u => u.role === 'admin').length;
|
||||
if (adminCount <= 1) {
|
||||
return res.status(400).json({ error: 'Cannot delete the last admin' });
|
||||
}
|
||||
}
|
||||
|
||||
deleteAppUser(id);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// GET /api/admin/available-folders — all gallery folders
|
||||
router.get('/api/admin/available-folders', requireAuth, requireAdmin, (req, res) => {
|
||||
const folders = getAllIndexedFolders();
|
||||
res.json(folders);
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Router } from 'express';
|
||||
import {
|
||||
getStorageStats, getTotalStorageSize, getDownloadsToday,
|
||||
getRecentDownloads, getMediaFileCount, getAutoDownloadUsers, getAutoScrapeJobs,
|
||||
getAuthConfig,
|
||||
} from './db.js';
|
||||
import { getActiveDownloadCount, getActiveDownloadsList } from './download.js';
|
||||
import { getActiveScrapeCount, getActiveScrapesList } from './scrape.js';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/api/dashboard', (req, res) => {
|
||||
try {
|
||||
const storageStats = getStorageStats();
|
||||
const totalStorage = getTotalStorageSize();
|
||||
const totalFiles = getMediaFileCount();
|
||||
const downloadsToday = getDownloadsToday();
|
||||
const recentDownloads = getRecentDownloads(10);
|
||||
const autoDownloads = getAutoDownloadUsers();
|
||||
const autoScrapes = getAutoScrapeJobs();
|
||||
|
||||
res.json({
|
||||
stats: {
|
||||
totalFiles,
|
||||
totalStorage,
|
||||
totalFolders: storageStats.length,
|
||||
downloadsToday,
|
||||
},
|
||||
topFolders: storageStats.slice(0, 10),
|
||||
activeJobs: {
|
||||
downloads: getActiveDownloadCount(),
|
||||
scrapes: getActiveScrapeCount(),
|
||||
downloadList: getActiveDownloadsList(),
|
||||
scrapeList: getActiveScrapesList(),
|
||||
},
|
||||
auth: {
|
||||
configured: !!getAuthConfig(),
|
||||
},
|
||||
scheduler: {
|
||||
autoDownloadCount: autoDownloads.length,
|
||||
autoScrapeCount: autoScrapes.length,
|
||||
},
|
||||
recentDownloads,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[dashboard] Error:', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
+549
-7
@@ -60,6 +60,64 @@ db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_media_type ON media_files(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_modified ON media_files(modified);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_posted_at ON media_files(posted_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auto_download_users (
|
||||
user_id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
last_run TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auto_scrape_jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
type TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
folder_name TEXT NOT NULL,
|
||||
config TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
last_run TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
||||
password_hash TEXT NOT NULL,
|
||||
display_name TEXT DEFAULT '',
|
||||
role TEXT NOT NULL DEFAULT 'user',
|
||||
enabled INTEGER DEFAULT 1,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_folder_access (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL,
|
||||
folder TEXT NOT NULL,
|
||||
UNIQUE(user_id, folder),
|
||||
FOREIGN KEY (user_id) REFERENCES app_users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_route_access (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL,
|
||||
route_key TEXT NOT NULL,
|
||||
UNIQUE(user_id, route_key),
|
||||
FOREIGN KEY (user_id) REFERENCES app_users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_folder_user ON user_folder_access(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_route_user ON user_route_access(user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS forum_sites (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
base_url TEXT DEFAULT '',
|
||||
cookies TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
`);
|
||||
|
||||
// Migration: add posted_at column if missing
|
||||
@@ -68,6 +126,18 @@ if (!cols.includes('posted_at')) {
|
||||
db.exec('ALTER TABLE download_history ADD COLUMN posted_at TEXT');
|
||||
}
|
||||
|
||||
// Migration: add username, password, cookie_expires_at columns to forum_sites
|
||||
const forumCols = db.prepare("PRAGMA table_info(forum_sites)").all().map((c) => c.name);
|
||||
if (!forumCols.includes('username')) {
|
||||
db.exec("ALTER TABLE forum_sites ADD COLUMN username TEXT DEFAULT ''");
|
||||
}
|
||||
if (!forumCols.includes('password')) {
|
||||
db.exec("ALTER TABLE forum_sites ADD COLUMN password TEXT DEFAULT ''");
|
||||
}
|
||||
if (!forumCols.includes('cookie_expires_at')) {
|
||||
db.exec('ALTER TABLE forum_sites ADD COLUMN cookie_expires_at TEXT');
|
||||
}
|
||||
|
||||
export function getAuthConfig() {
|
||||
const row = db.prepare('SELECT * FROM auth_config LIMIT 1').get();
|
||||
return row || null;
|
||||
@@ -177,15 +247,19 @@ export function getMediaFolders() {
|
||||
SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) AS images,
|
||||
SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) AS videos
|
||||
FROM media_files
|
||||
WHERE folder NOT LIKE '\\_%' ESCAPE '\\'
|
||||
GROUP BY folder
|
||||
ORDER BY folder
|
||||
`).all();
|
||||
}
|
||||
|
||||
export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
|
||||
export function getMediaFiles({ folder, folders, type, sort, offset, limit, dateFrom, dateTo, minSize, maxSize, search }) {
|
||||
const conditions = [];
|
||||
const params = [];
|
||||
|
||||
// Always exclude folders starting with _
|
||||
conditions.push("folder NOT LIKE '\\_%' ESCAPE '\\'");
|
||||
|
||||
if (folder) {
|
||||
conditions.push('folder = ?');
|
||||
params.push(folder);
|
||||
@@ -199,17 +273,89 @@ export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
|
||||
params.push(type);
|
||||
}
|
||||
|
||||
if (dateFrom) {
|
||||
conditions.push("COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) >= ?");
|
||||
params.push(dateFrom);
|
||||
}
|
||||
if (dateTo) {
|
||||
conditions.push("COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) <= ?");
|
||||
params.push(dateTo + 'T23:59:59');
|
||||
}
|
||||
if (minSize) {
|
||||
conditions.push('size >= ?');
|
||||
params.push(parseInt(minSize, 10));
|
||||
}
|
||||
if (maxSize) {
|
||||
conditions.push('size <= ?');
|
||||
params.push(parseInt(maxSize, 10));
|
||||
}
|
||||
if (search) {
|
||||
conditions.push('(filename LIKE ? OR folder LIKE ?)');
|
||||
params.push(`%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
|
||||
const countRow = db.prepare(`SELECT COUNT(*) AS total FROM media_files ${where}`).get(...params);
|
||||
const total = countRow.total;
|
||||
|
||||
let orderBy;
|
||||
const effectiveLimit = limit || 50;
|
||||
const effectiveOffset = offset || 0;
|
||||
|
||||
// Equal-mix shuffle: when shuffling with multiple folders, sample equally from each
|
||||
if (sort === 'shuffle') {
|
||||
orderBy = 'ORDER BY RANDOM()';
|
||||
} else {
|
||||
// 'latest' — prefer posted_at, fall back to modified
|
||||
orderBy = 'ORDER BY COALESCE(posted_at, datetime(modified / 1000, \'unixepoch\')) DESC';
|
||||
// Count distinct folders in the result set
|
||||
const folderCountRow = db.prepare(
|
||||
`SELECT COUNT(DISTINCT folder) AS cnt FROM media_files ${where}`
|
||||
).get(...params);
|
||||
const numFolders = folderCountRow?.cnt || 1;
|
||||
|
||||
if (numFolders > 1) {
|
||||
// Use ROW_NUMBER to pick equal random samples per folder
|
||||
const perFolder = Math.ceil(effectiveLimit / numFolders);
|
||||
const rows = db.prepare(`
|
||||
WITH ranked AS (
|
||||
SELECT folder, filename, type, size, modified, posted_at,
|
||||
ROW_NUMBER() OVER (PARTITION BY folder ORDER BY RANDOM()) AS rn
|
||||
FROM media_files
|
||||
${where}
|
||||
)
|
||||
SELECT folder, filename, type, size, modified, posted_at
|
||||
FROM ranked
|
||||
WHERE rn <= ?
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ? OFFSET ?
|
||||
`).all(...params, perFolder, effectiveLimit, effectiveOffset);
|
||||
return { total, rows };
|
||||
}
|
||||
|
||||
// Single folder or no folder filter — plain random
|
||||
const rows = db.prepare(`
|
||||
SELECT folder, filename, type, size, modified, posted_at
|
||||
FROM media_files
|
||||
${where}
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ? OFFSET ?
|
||||
`).all(...params, effectiveLimit, effectiveOffset);
|
||||
return { total, rows };
|
||||
}
|
||||
|
||||
let orderBy;
|
||||
switch (sort) {
|
||||
case 'oldest':
|
||||
orderBy = "ORDER BY COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) ASC";
|
||||
break;
|
||||
case 'largest':
|
||||
orderBy = 'ORDER BY size DESC';
|
||||
break;
|
||||
case 'smallest':
|
||||
orderBy = 'ORDER BY size ASC';
|
||||
break;
|
||||
case 'name':
|
||||
orderBy = 'ORDER BY filename ASC';
|
||||
break;
|
||||
default:
|
||||
orderBy = "ORDER BY COALESCE(posted_at, datetime(modified / 1000, 'unixepoch')) DESC";
|
||||
}
|
||||
|
||||
const rows = db.prepare(`
|
||||
@@ -218,7 +364,7 @@ export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
|
||||
${where}
|
||||
${orderBy}
|
||||
LIMIT ? OFFSET ?
|
||||
`).all(...params, limit || 50, offset || 0);
|
||||
`).all(...params, effectiveLimit, effectiveOffset);
|
||||
|
||||
return { total, rows };
|
||||
}
|
||||
@@ -244,3 +390,399 @@ export function removeStaleFiles(folder, existingFilenames) {
|
||||
export function getMediaFileCount() {
|
||||
return db.prepare('SELECT COUNT(*) AS count FROM media_files').get().count;
|
||||
}
|
||||
|
||||
export function getNewMediaCount(since) {
|
||||
return db.prepare('SELECT COUNT(*) AS count FROM media_files WHERE created_at > ?').get(since).count;
|
||||
}
|
||||
|
||||
// --- auto_download_users helpers ---
|
||||
|
||||
export function getAutoDownloadUsers() {
|
||||
return db.prepare('SELECT * FROM auto_download_users WHERE enabled = 1').all();
|
||||
}
|
||||
|
||||
export function addAutoDownloadUser(userId, username) {
|
||||
db.prepare(
|
||||
'INSERT INTO auto_download_users (user_id, username) VALUES (?, ?) ON CONFLICT(user_id) DO UPDATE SET username = excluded.username, enabled = 1'
|
||||
).run(String(userId), username);
|
||||
}
|
||||
|
||||
export function removeAutoDownloadUser(userId) {
|
||||
db.prepare('DELETE FROM auto_download_users WHERE user_id = ?').run(String(userId));
|
||||
}
|
||||
|
||||
export function isAutoDownloadUser(userId) {
|
||||
return !!db.prepare('SELECT 1 FROM auto_download_users WHERE user_id = ? AND enabled = 1').get(String(userId));
|
||||
}
|
||||
|
||||
export function updateAutoDownloadLastRun(userId) {
|
||||
db.prepare('UPDATE auto_download_users SET last_run = datetime(\'now\') WHERE user_id = ?').run(String(userId));
|
||||
}
|
||||
|
||||
// --- auto_scrape_jobs helpers ---
|
||||
|
||||
export function getAutoScrapeJobs() {
|
||||
return db.prepare('SELECT * FROM auto_scrape_jobs WHERE enabled = 1').all();
|
||||
}
|
||||
|
||||
export function addAutoScrapeJob(type, url, folderName, config) {
|
||||
db.prepare(
|
||||
'INSERT INTO auto_scrape_jobs (type, url, folder_name, config) VALUES (?, ?, ?, ?)'
|
||||
).run(type, url, folderName, JSON.stringify(config));
|
||||
}
|
||||
|
||||
export function removeAutoScrapeJob(id) {
|
||||
db.prepare('DELETE FROM auto_scrape_jobs WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
export function updateAutoScrapeLastRun(id) {
|
||||
db.prepare('UPDATE auto_scrape_jobs SET last_run = datetime(\'now\') WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
// --- Dashboard / stats helpers ---
|
||||
|
||||
export function getStorageStats() {
|
||||
return db.prepare(`
|
||||
SELECT folder,
|
||||
COUNT(*) AS file_count,
|
||||
SUM(size) AS total_size,
|
||||
SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) AS images,
|
||||
SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) AS videos
|
||||
FROM media_files
|
||||
GROUP BY folder
|
||||
ORDER BY SUM(size) DESC
|
||||
`).all();
|
||||
}
|
||||
|
||||
export function getTotalStorageSize() {
|
||||
const row = db.prepare('SELECT SUM(size) AS total FROM media_files').get();
|
||||
return row?.total || 0;
|
||||
}
|
||||
|
||||
// --- videos tables ---
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS videos (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
filename TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
file_size INTEGER NOT NULL DEFAULT 0,
|
||||
duration REAL,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
fps REAL,
|
||||
codec TEXT,
|
||||
bitrate INTEGER,
|
||||
has_audio INTEGER DEFAULT 1,
|
||||
thumbnail_path TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
error_message TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS video_tags (
|
||||
video_id INTEGER NOT NULL,
|
||||
tag_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (video_id, tag_id),
|
||||
FOREIGN KEY (video_id) REFERENCES videos(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
// --- video helpers ---
|
||||
|
||||
export function insertVideo(data) {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO videos (title, description, filename, file_path, file_size, duration, width, height, fps, codec, bitrate, has_audio, thumbnail_path, status, error_message)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
const result = stmt.run(
|
||||
data.title, data.description || '', data.filename, data.file_path, data.file_size || 0,
|
||||
data.duration || null, data.width || null, data.height || null, data.fps || null,
|
||||
data.codec || null, data.bitrate || null, data.has_audio ?? 1,
|
||||
data.thumbnail_path || null, data.status || 'pending', data.error_message || null
|
||||
);
|
||||
return result.lastInsertRowid;
|
||||
}
|
||||
|
||||
export function getVideoById(id) {
|
||||
return db.prepare('SELECT * FROM videos WHERE id = ?').get(id) || null;
|
||||
}
|
||||
|
||||
export function getVideoByPath(filePath) {
|
||||
return db.prepare('SELECT * FROM videos WHERE file_path = ?').get(filePath) || null;
|
||||
}
|
||||
|
||||
export function updateVideo(id, data) {
|
||||
const fields = [];
|
||||
const values = [];
|
||||
for (const [key, val] of Object.entries(data)) {
|
||||
if (key === 'id') continue;
|
||||
fields.push(`${key} = ?`);
|
||||
values.push(val);
|
||||
}
|
||||
if (fields.length === 0) return;
|
||||
fields.push("updated_at = datetime('now')");
|
||||
values.push(id);
|
||||
db.prepare(`UPDATE videos SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
export function deleteVideoById(id) {
|
||||
db.prepare('DELETE FROM video_tags WHERE video_id = ?').run(id);
|
||||
db.prepare('DELETE FROM videos WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
export function searchVideos({ search, tags, minDuration, maxDuration, minWidth, sort, offset, limit }) {
|
||||
const conditions = [];
|
||||
const params = [];
|
||||
|
||||
if (search) {
|
||||
conditions.push('(v.title LIKE ? OR v.description LIKE ? OR v.filename LIKE ?)');
|
||||
params.push(`%${search}%`, `%${search}%`, `%${search}%`);
|
||||
}
|
||||
if (minDuration) {
|
||||
conditions.push('v.duration >= ?');
|
||||
params.push(parseFloat(minDuration));
|
||||
}
|
||||
if (maxDuration) {
|
||||
conditions.push('v.duration <= ?');
|
||||
params.push(parseFloat(maxDuration));
|
||||
}
|
||||
if (minWidth) {
|
||||
conditions.push('v.width >= ?');
|
||||
params.push(parseInt(minWidth, 10));
|
||||
}
|
||||
if (tags && tags.length > 0) {
|
||||
conditions.push(`v.id IN (
|
||||
SELECT vt.video_id FROM video_tags vt
|
||||
JOIN tags t ON t.id = vt.tag_id
|
||||
WHERE t.name IN (${tags.map(() => '?').join(',')})
|
||||
)`);
|
||||
params.push(...tags);
|
||||
}
|
||||
|
||||
conditions.push("v.status = 'ready'");
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
|
||||
const countRow = db.prepare(`SELECT COUNT(*) AS total FROM videos v ${where}`).get(...params);
|
||||
const total = countRow.total;
|
||||
|
||||
let orderBy;
|
||||
switch (sort) {
|
||||
case 'oldest': orderBy = 'ORDER BY v.created_at ASC'; break;
|
||||
case 'longest': orderBy = 'ORDER BY v.duration DESC'; break;
|
||||
case 'shortest': orderBy = 'ORDER BY v.duration ASC'; break;
|
||||
case 'largest': orderBy = 'ORDER BY v.file_size DESC'; break;
|
||||
case 'title': orderBy = 'ORDER BY v.title ASC'; break;
|
||||
case 'shuffle': orderBy = 'ORDER BY RANDOM()'; break;
|
||||
default: orderBy = 'ORDER BY v.created_at DESC';
|
||||
}
|
||||
|
||||
const effectiveLimit = limit || 48;
|
||||
const effectiveOffset = offset || 0;
|
||||
|
||||
const rows = db.prepare(`
|
||||
SELECT v.* FROM videos v ${where} ${orderBy} LIMIT ? OFFSET ?
|
||||
`).all(...params, effectiveLimit, effectiveOffset);
|
||||
|
||||
return { total, rows };
|
||||
}
|
||||
|
||||
export function getOrCreateTag(name) {
|
||||
const trimmed = name.trim();
|
||||
if (!trimmed) return null;
|
||||
let row = db.prepare('SELECT id FROM tags WHERE name = ? COLLATE NOCASE').get(trimmed);
|
||||
if (!row) {
|
||||
const result = db.prepare('INSERT INTO tags (name) VALUES (?)').run(trimmed);
|
||||
return result.lastInsertRowid;
|
||||
}
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function getAllTags(search) {
|
||||
if (search) {
|
||||
return db.prepare(`
|
||||
SELECT t.id, t.name, COUNT(vt.video_id) AS count
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
WHERE t.name LIKE ?
|
||||
GROUP BY t.id
|
||||
ORDER BY count DESC, t.name ASC
|
||||
`).all(`%${search}%`);
|
||||
}
|
||||
return db.prepare(`
|
||||
SELECT t.id, t.name, COUNT(vt.video_id) AS count
|
||||
FROM tags t
|
||||
LEFT JOIN video_tags vt ON vt.tag_id = t.id
|
||||
GROUP BY t.id
|
||||
ORDER BY count DESC, t.name ASC
|
||||
`).all();
|
||||
}
|
||||
|
||||
export function setVideoTags(videoId, tagNames) {
|
||||
const setTags = db.transaction((names) => {
|
||||
db.prepare('DELETE FROM video_tags WHERE video_id = ?').run(videoId);
|
||||
for (const name of names) {
|
||||
const tagId = getOrCreateTag(name);
|
||||
if (tagId) {
|
||||
db.prepare('INSERT OR IGNORE INTO video_tags (video_id, tag_id) VALUES (?, ?)').run(videoId, tagId);
|
||||
}
|
||||
}
|
||||
});
|
||||
setTags(tagNames);
|
||||
}
|
||||
|
||||
export function getVideoTags(videoId) {
|
||||
return db.prepare(`
|
||||
SELECT t.id, t.name FROM tags t
|
||||
JOIN video_tags vt ON vt.tag_id = t.id
|
||||
WHERE vt.video_id = ?
|
||||
ORDER BY t.name ASC
|
||||
`).all(videoId);
|
||||
}
|
||||
|
||||
export function getDownloadsToday() {
|
||||
// created_at is stored in UTC via SQLite datetime('now').
|
||||
// Compute local midnight in UTC-relative format so "today" matches the server's local day.
|
||||
const now = new Date();
|
||||
const localMidnight = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
// Format as SQLite-compatible "YYYY-MM-DD HH:MM:SS"
|
||||
const y = localMidnight.getUTCFullYear();
|
||||
const mo = String(localMidnight.getUTCMonth() + 1).padStart(2, '0');
|
||||
const d = String(localMidnight.getUTCDate()).padStart(2, '0');
|
||||
const h = String(localMidnight.getUTCHours()).padStart(2, '0');
|
||||
const mi = String(localMidnight.getUTCMinutes()).padStart(2, '0');
|
||||
const s = String(localMidnight.getUTCSeconds()).padStart(2, '0');
|
||||
const todayUtc = `${y}-${mo}-${d} ${h}:${mi}:${s}`;
|
||||
const row = db.prepare('SELECT COUNT(*) AS count FROM media_files WHERE created_at >= ?').get(todayUtc);
|
||||
return row?.count || 0;
|
||||
}
|
||||
|
||||
export function getRecentDownloads(limit = 10) {
|
||||
// Merge recent items from both download_history and media_files (for scrapes)
|
||||
return db.prepare(`
|
||||
SELECT filename, type AS media_type, folder AS user_id, created_at AS downloaded_at
|
||||
FROM media_files
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?
|
||||
`).all(limit);
|
||||
}
|
||||
|
||||
// --- app_users helpers ---
|
||||
|
||||
export function createAppUser(username, passwordHash, role = 'user', displayName = '') {
|
||||
const result = db.prepare(
|
||||
'INSERT INTO app_users (username, password_hash, role, display_name) VALUES (?, ?, ?, ?)'
|
||||
).run(username, passwordHash, role, displayName);
|
||||
return result.lastInsertRowid;
|
||||
}
|
||||
|
||||
export function getAppUserByUsername(username) {
|
||||
return db.prepare('SELECT * FROM app_users WHERE username = ?').get(username) || null;
|
||||
}
|
||||
|
||||
export function getAppUserById(id) {
|
||||
return db.prepare('SELECT * FROM app_users WHERE id = ?').get(id) || null;
|
||||
}
|
||||
|
||||
export function getAllAppUsers() {
|
||||
return db.prepare('SELECT id, username, display_name, role, enabled, created_at, updated_at FROM app_users ORDER BY id').all();
|
||||
}
|
||||
|
||||
export function updateAppUser(id, fields) {
|
||||
const allowed = ['username', 'password_hash', 'display_name', 'role', 'enabled'];
|
||||
const sets = [];
|
||||
const values = [];
|
||||
for (const [key, val] of Object.entries(fields)) {
|
||||
if (!allowed.includes(key)) continue;
|
||||
sets.push(`${key} = ?`);
|
||||
values.push(val);
|
||||
}
|
||||
if (sets.length === 0) return;
|
||||
sets.push("updated_at = datetime('now')");
|
||||
values.push(id);
|
||||
db.prepare(`UPDATE app_users SET ${sets.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
export function deleteAppUser(id) {
|
||||
db.prepare('DELETE FROM app_users WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
export function getAppUserCount() {
|
||||
return db.prepare('SELECT COUNT(*) AS count FROM app_users').get().count;
|
||||
}
|
||||
|
||||
export function getUserFolderAccess(userId) {
|
||||
return db.prepare('SELECT folder FROM user_folder_access WHERE user_id = ?').all(userId).map(r => r.folder);
|
||||
}
|
||||
|
||||
export function setUserFolderAccess(userId, folders) {
|
||||
const update = db.transaction((flds) => {
|
||||
db.prepare('DELETE FROM user_folder_access WHERE user_id = ?').run(userId);
|
||||
const ins = db.prepare('INSERT INTO user_folder_access (user_id, folder) VALUES (?, ?)');
|
||||
for (const f of flds) {
|
||||
ins.run(userId, f);
|
||||
}
|
||||
});
|
||||
update(folders);
|
||||
}
|
||||
|
||||
export function getUserRouteAccess(userId) {
|
||||
return db.prepare('SELECT route_key FROM user_route_access WHERE user_id = ?').all(userId).map(r => r.route_key);
|
||||
}
|
||||
|
||||
export function setUserRouteAccess(userId, routes) {
|
||||
const update = db.transaction((rts) => {
|
||||
db.prepare('DELETE FROM user_route_access WHERE user_id = ?').run(userId);
|
||||
const ins = db.prepare('INSERT INTO user_route_access (user_id, route_key) VALUES (?, ?)');
|
||||
for (const r of rts) {
|
||||
ins.run(userId, r);
|
||||
}
|
||||
});
|
||||
update(routes);
|
||||
}
|
||||
|
||||
// --- Forum Sites ---
|
||||
|
||||
export function getForumSites() {
|
||||
return db.prepare('SELECT * FROM forum_sites ORDER BY name').all();
|
||||
}
|
||||
|
||||
export function getForumSiteById(id) {
|
||||
return db.prepare('SELECT * FROM forum_sites WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
export function createForumSite(name, baseUrl, cookies, username, password) {
|
||||
const result = db.prepare('INSERT INTO forum_sites (name, base_url, cookies, username, password) VALUES (?, ?, ?, ?, ?)').run(name, baseUrl || '', cookies || '', username || '', password || '');
|
||||
return result.lastInsertRowid;
|
||||
}
|
||||
|
||||
export function updateForumSite(id, fields) {
|
||||
const allowed = ['name', 'base_url', 'cookies', 'username', 'password', 'cookie_expires_at'];
|
||||
const sets = [];
|
||||
const vals = [];
|
||||
for (const [k, v] of Object.entries(fields)) {
|
||||
if (allowed.includes(k)) {
|
||||
sets.push(`${k} = ?`);
|
||||
vals.push(v);
|
||||
}
|
||||
}
|
||||
if (sets.length === 0) return;
|
||||
sets.push("updated_at = datetime('now')");
|
||||
vals.push(id);
|
||||
db.prepare(`UPDATE forum_sites SET ${sets.join(', ')} WHERE id = ?`).run(...vals);
|
||||
}
|
||||
|
||||
export function deleteForumSite(id) {
|
||||
db.prepare('DELETE FROM forum_sites WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
+142
-2
@@ -3,7 +3,7 @@ import fetch from 'node-fetch';
|
||||
import { mkdirSync, createWriteStream, statSync } from 'fs';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { extname } from 'path';
|
||||
import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile } from './db.js';
|
||||
import { getAuthConfig, isMediaDownloaded, recordDownload, getDownloadStats, saveCursor, getCursor, clearCursor, upsertMediaFile, getAutoDownloadUsers, addAutoDownloadUser, removeAutoDownloadUser } from './db.js';
|
||||
import { createSignedHeaders, getRules } from './signing.js';
|
||||
import { downloadDrmMedia, hasCDM } from './drm-download.js';
|
||||
|
||||
@@ -15,6 +15,36 @@ const DOWNLOAD_DELAY = parseInt(process.env.DOWNLOAD_DELAY || '1000', 10);
|
||||
// In-memory progress: userId -> { total, completed, errors, running }
|
||||
const progressMap = new Map();
|
||||
|
||||
// In-memory download logs: userId -> last N file entries
|
||||
const downloadLogMap = new Map();
|
||||
const MAX_LOG_ENTRIES = 20;
|
||||
|
||||
function addDownloadLog(userId, entry) {
|
||||
const key = String(userId);
|
||||
if (!downloadLogMap.has(key)) downloadLogMap.set(key, []);
|
||||
const logs = downloadLogMap.get(key);
|
||||
logs.push({ ...entry, timestamp: new Date().toISOString() });
|
||||
if (logs.length > MAX_LOG_ENTRIES) logs.shift();
|
||||
}
|
||||
|
||||
export function getActiveDownloadCount() {
|
||||
let count = 0;
|
||||
for (const p of progressMap.values()) {
|
||||
if (p.running) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getActiveDownloadsList() {
|
||||
const list = [];
|
||||
for (const [userId, p] of progressMap.entries()) {
|
||||
if (p.running) {
|
||||
list.push({ userId, username: p.username, total: p.total, completed: p.completed, errors: p.errors });
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
function buildHeaders(authConfig, signedHeaders) {
|
||||
const rules = getRules();
|
||||
const headers = {
|
||||
@@ -83,8 +113,9 @@ async function downloadFile(url, dest) {
|
||||
}
|
||||
|
||||
async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const progress = { total: 0, completed: 0, errors: 0, running: true };
|
||||
const progress = { total: 0, completed: 0, errors: 0, running: true, username: username || null };
|
||||
progressMap.set(String(userId), progress);
|
||||
console.log(`[download] Starting download for user ${userId} (${username || 'unknown'})${postLimit ? ` limit=${postLimit}` : ' all posts'}${resume ? ' (resume)' : ''}`);
|
||||
|
||||
try {
|
||||
let beforePublishTime = null;
|
||||
@@ -156,6 +187,7 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
}
|
||||
|
||||
progress.total = allMedia.length;
|
||||
console.log(`[download] User ${userId}: found ${allMedia.length} media items across ${postsFetched} posts`);
|
||||
|
||||
// Phase 2: Download each media item
|
||||
for (const { postId, media, postDate } of allMedia) {
|
||||
@@ -203,9 +235,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const st = statSync(`${userDir}/${drmFilename}`);
|
||||
upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate);
|
||||
} catch { /* stat may fail if file was cleaned up */ }
|
||||
addDownloadLog(userId, { filename: drmFilename, mediaType: 'video', status: 'ok' });
|
||||
progress.completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
|
||||
addDownloadLog(userId, { filename: `${postId}_${mediaId}_video.mp4`, mediaType: 'video', status: 'error' });
|
||||
progress.errors++;
|
||||
progress.completed++;
|
||||
}
|
||||
@@ -234,9 +268,11 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
|
||||
if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
|
||||
} catch { /* ignore */ }
|
||||
addDownloadLog(userId, { filename, mediaType, status: 'ok' });
|
||||
progress.completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] Error downloading media ${media.id}:`, err.message);
|
||||
addDownloadLog(userId, { filename: `${postId}_${media.id}`, mediaType: media.type || 'unknown', status: 'error' });
|
||||
progress.errors++;
|
||||
progress.completed++;
|
||||
}
|
||||
@@ -251,6 +287,75 @@ async function runDownload(userId, authConfig, postLimit, resume, username) {
|
||||
}
|
||||
}
|
||||
|
||||
// POST /api/download/post — download media from a single post
|
||||
router.post('/api/download/post', async (req, res, next) => {
|
||||
try {
|
||||
const { userId, username, postId, postedAt, media: mediaItems } = req.body;
|
||||
if (!userId || !postId || !Array.isArray(mediaItems) || mediaItems.length === 0) {
|
||||
return res.status(400).json({ error: 'userId, postId, and media[] are required' });
|
||||
}
|
||||
const postDate = postedAt || null;
|
||||
|
||||
const userDir = `${MEDIA_PATH}/${username || userId}`;
|
||||
mkdirSync(userDir, { recursive: true });
|
||||
let completed = 0, errors = 0;
|
||||
|
||||
console.log(`[download] Post ${postId}: downloading ${mediaItems.length} media items for ${username || userId}, postedAt=${postDate}`);
|
||||
|
||||
for (const media of mediaItems) {
|
||||
try {
|
||||
const mediaId = String(media.id);
|
||||
if (isMediaDownloaded(mediaId)) { completed++; continue; }
|
||||
if (media.canView === false) { completed++; continue; }
|
||||
|
||||
// DRM video
|
||||
const drm = media.files?.drm;
|
||||
if (drm?.manifest?.dash && drm?.signature?.dash) {
|
||||
if (!hasCDM()) { completed++; continue; }
|
||||
try {
|
||||
const sig = drm.signature.dash;
|
||||
const cfCookies = { cp: sig['CloudFront-Policy'], cs: sig['CloudFront-Signature'], ck: sig['CloudFront-Key-Pair-Id'] };
|
||||
const drmFilename = `${postId}_${mediaId}_video.mp4`;
|
||||
await downloadDrmMedia({ mpdUrl: drm.manifest.dash, cfCookies, mediaId, entityType: 'post', entityId: String(postId), outputDir: userDir, outputFilename: drmFilename });
|
||||
recordDownload(userId, String(postId), mediaId, 'video', drmFilename, postDate);
|
||||
try { const st = statSync(`${userDir}/${drmFilename}`); upsertMediaFile(username || String(userId), drmFilename, 'video', st.size, st.mtimeMs, postDate); } catch {}
|
||||
completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] DRM download failed for media ${mediaId}:`, err.message);
|
||||
errors++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = getMediaUrl(media);
|
||||
if (!url) { completed++; continue; }
|
||||
|
||||
const mediaType = media.type || 'unknown';
|
||||
const ext = getExtFromUrl(url);
|
||||
const filename = `${postId}_${mediaId}_${mediaType}${ext}`;
|
||||
const dest = `${userDir}/${filename}`;
|
||||
|
||||
await downloadFile(url, dest);
|
||||
recordDownload(userId, String(postId), mediaId, mediaType, filename, postDate);
|
||||
try {
|
||||
const st = statSync(dest);
|
||||
const indexType = /^(photo|image)$/i.test(mediaType) ? 'image' : /^(video|gif)$/i.test(mediaType) ? 'video' : null;
|
||||
if (indexType) upsertMediaFile(username || String(userId), filename, indexType, st.size, st.mtimeMs, postDate);
|
||||
} catch {}
|
||||
completed++;
|
||||
} catch (err) {
|
||||
console.error(`[download] Error downloading media ${media.id}:`, err.message);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[download] Post ${postId}: done (${completed} downloaded, ${errors} errors)`);
|
||||
res.json({ status: 'done', completed, errors, total: mediaItems.length });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/download/:userId — start background download
|
||||
router.post('/api/download/:userId', (req, res, next) => {
|
||||
try {
|
||||
@@ -302,6 +407,21 @@ router.get('/api/download/active', (req, res) => {
|
||||
res.json(active);
|
||||
});
|
||||
|
||||
// GET /api/download/active/details — active downloads with recent file logs
|
||||
router.get('/api/download/active/details', (req, res) => {
|
||||
const active = [];
|
||||
for (const [userId, progress] of progressMap.entries()) {
|
||||
if (progress.running) {
|
||||
active.push({
|
||||
user_id: userId,
|
||||
...progress,
|
||||
recentFiles: downloadLogMap.get(String(userId))?.slice(-5) || [],
|
||||
});
|
||||
}
|
||||
}
|
||||
res.json(active);
|
||||
});
|
||||
|
||||
// GET /api/download/history
|
||||
router.get('/api/download/history', (req, res, next) => {
|
||||
try {
|
||||
@@ -312,4 +432,24 @@ router.get('/api/download/history', (req, res, next) => {
|
||||
}
|
||||
});
|
||||
|
||||
// --- Auto-download CRUD ---
|
||||
|
||||
router.get('/api/download/auto', (_req, res) => {
|
||||
res.json(getAutoDownloadUsers());
|
||||
});
|
||||
|
||||
router.post('/api/download/auto/:userId', (req, res) => {
|
||||
const { userId } = req.params;
|
||||
const { username } = req.body;
|
||||
if (!username) return res.status(400).json({ error: 'username is required' });
|
||||
addAutoDownloadUser(userId, username);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
router.delete('/api/download/auto/:userId', (req, res) => {
|
||||
removeAutoDownloadUser(req.params.userId);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
export { runDownload };
|
||||
export default router;
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { Router } from 'express';
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { getForumSiteById, updateForumSite } from './db.js';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
const router = Router();
|
||||
const FLARESOLVERR_URL = process.env.FLARESOLVERR_URL || 'http://localhost:8191';
|
||||
const CHROMIUM_PATH = process.env.CHROMIUM_PATH || '/usr/bin/chromium-browser';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
/**
|
||||
* Refresh forum cookies using undetected_chromedriver (Python).
|
||||
* Runs login_helper.py via xvfb-run so Chrome runs in headed mode
|
||||
* with a virtual display — this is what lets Turnstile auto-solve.
|
||||
*/
|
||||
export async function refreshForumCookies(siteId) {
|
||||
const site = getForumSiteById(siteId);
|
||||
if (!site) throw new Error(`Forum site ${siteId} not found`);
|
||||
if (!site.username || !site.password) {
|
||||
throw new Error('Forum site has no saved credentials — set username and password first');
|
||||
}
|
||||
|
||||
const baseUrl = site.base_url || 'https://simpcity.su';
|
||||
const loginUrl = `${baseUrl}/login/`;
|
||||
const helperPath = path.join(__dirname, 'login_helper.py');
|
||||
|
||||
console.log(`[flaresolverr] Refreshing cookies for site ${siteId} (${site.name})`);
|
||||
console.log(`[flaresolverr] Login URL: ${loginUrl}`);
|
||||
|
||||
// Run the Python helper with xvfb-run for virtual display
|
||||
// Escape arguments for shell safety
|
||||
const escapedUrl = loginUrl.replace(/'/g, "'\\''");
|
||||
const escapedUser = site.username.replace(/'/g, "'\\''");
|
||||
const escapedPass = site.password.replace(/'/g, "'\\''");
|
||||
|
||||
const cmd = `xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' python3 '${helperPath}' '${escapedUrl}' '${escapedUser}' '${escapedPass}'`;
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await execAsync(cmd, {
|
||||
timeout: 120000, // 2 minutes
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
env: { ...process.env, CHROMIUM_PATH },
|
||||
});
|
||||
|
||||
// Log stderr (debug output from login_helper.py)
|
||||
if (stderr) {
|
||||
for (const line of stderr.split('\n').filter(Boolean)) {
|
||||
console.log(`[flaresolverr] ${line}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON from stdout
|
||||
const result = JSON.parse(stdout.trim());
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(result.error || 'Login failed');
|
||||
}
|
||||
|
||||
// Update DB with new cookies
|
||||
const expiresAt = new Date(Date.now() + 25 * 24 * 60 * 60 * 1000).toISOString();
|
||||
updateForumSite(siteId, {
|
||||
cookies: result.cookies,
|
||||
cookie_expires_at: expiresAt,
|
||||
});
|
||||
|
||||
console.log(`[flaresolverr] Cookie refresh successful for site ${siteId}`);
|
||||
return result.cookies;
|
||||
} catch (err) {
|
||||
// If execAsync fails, the error might have stderr info
|
||||
if (err.stderr) {
|
||||
for (const line of err.stderr.split('\n').filter(Boolean)) {
|
||||
console.error(`[flaresolverr] ${line}`);
|
||||
}
|
||||
}
|
||||
// Try to parse stdout for a structured error
|
||||
if (err.stdout) {
|
||||
try {
|
||||
const result = JSON.parse(err.stdout.trim());
|
||||
if (result.error) throw new Error(result.error);
|
||||
} catch (parseErr) {
|
||||
// Not JSON, use original error
|
||||
}
|
||||
}
|
||||
throw new Error(`Cookie refresh failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- API Endpoints ---
|
||||
|
||||
// Manual cookie refresh
|
||||
router.post('/api/flaresolverr/refresh/:siteId', async (req, res) => {
|
||||
const siteId = parseInt(req.params.siteId, 10);
|
||||
try {
|
||||
const cookieStr = await refreshForumCookies(siteId);
|
||||
res.json({ ok: true, cookies: cookieStr });
|
||||
} catch (err) {
|
||||
console.error(`[flaresolverr] Refresh failed for site ${siteId}:`, err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Check if cookie refresh is available (Chromium + xvfb-run installed)
|
||||
router.get('/api/flaresolverr/status', async (_req, res) => {
|
||||
try {
|
||||
// Check for xvfb-run and chromium
|
||||
await execAsync('which xvfb-run && which chromium-browser || which chromium', { timeout: 5000 });
|
||||
// Check for undetected_chromedriver python package
|
||||
await execAsync('python3 -c "import undetected_chromedriver"', { timeout: 5000 });
|
||||
res.json({ available: true });
|
||||
} catch {
|
||||
// Fallback: check FlareSolverr service
|
||||
try {
|
||||
const resp = await fetch(`${FLARESOLVERR_URL}/health`, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
res.json({ available: resp.ok });
|
||||
} catch {
|
||||
res.json({ available: false, error: 'Neither undetected_chromedriver nor FlareSolverr available' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
+218
-33
@@ -8,6 +8,7 @@ import {
|
||||
getPostDateByFilename, getSetting,
|
||||
upsertMediaFileBatch, removeMediaFile, removeStaleFiles,
|
||||
getMediaFolders, getMediaFiles, getMediaFileCount, getAllIndexedFolders,
|
||||
getNewMediaCount, getUserFolderAccess,
|
||||
} from './db.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -107,20 +108,49 @@ export function scanMediaFiles() {
|
||||
console.log(`[gallery] Index scan complete: ${totalFiles} files in ${scannedFolders.size} folders (${elapsed}s). DB total: ${dbCount}`);
|
||||
}
|
||||
|
||||
// Helper: get allowed folders for current user (null = all access)
|
||||
function getAllowedFolders(req) {
|
||||
if (!req.user || req.user.role === 'admin') return null;
|
||||
return getUserFolderAccess(req.user.id);
|
||||
}
|
||||
|
||||
function checkFolderAccess(req, folder) {
|
||||
const allowed = getAllowedFolders(req);
|
||||
if (allowed === null) return true; // admin or no restrictions
|
||||
return allowed.includes(folder);
|
||||
}
|
||||
|
||||
// GET /api/gallery/new-count — count of media added since last gallery visit
|
||||
router.get('/api/gallery/new-count', (req, res, next) => {
|
||||
try {
|
||||
const lastSeen = getSetting('gallery_last_seen');
|
||||
if (!lastSeen) return res.json({ count: 0 });
|
||||
const count = getNewMediaCount(lastSeen);
|
||||
res.json({ count });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/gallery/folders — list all folders with file counts (from DB index)
|
||||
router.get('/api/gallery/folders', (req, res, next) => {
|
||||
try {
|
||||
const folders = getMediaFolders();
|
||||
let folders = getMediaFolders();
|
||||
const allowed = getAllowedFolders(req);
|
||||
if (allowed !== null) {
|
||||
const set = new Set(allowed);
|
||||
folders = folders.filter(f => set.has(f.name));
|
||||
}
|
||||
res.json(folders);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/gallery/files?folder=&type=&sort=&offset=&limit= (from DB index)
|
||||
// GET /api/gallery/files?folder=&type=&sort=&offset=&limit=&dateFrom=&dateTo=&minSize=&maxSize=&search= (from DB index)
|
||||
router.get('/api/gallery/files', (req, res, next) => {
|
||||
try {
|
||||
const { folder, type, sort, offset, limit } = req.query;
|
||||
const { folder, type, sort, offset, limit, dateFrom, dateTo, minSize, maxSize, search } = req.query;
|
||||
const foldersParam = req.query.folders;
|
||||
const foldersArr = foldersParam
|
||||
? foldersParam.split(',').map((f) => f.trim()).filter(Boolean)
|
||||
@@ -130,13 +160,45 @@ router.get('/api/gallery/files', (req, res, next) => {
|
||||
const limitNum = parseInt(limit || '50', 10);
|
||||
const hlsEnabled = (getSetting('hls_enabled') || process.env.HLS_ENABLED) === 'true';
|
||||
|
||||
// Enforce folder access for non-admin users
|
||||
const allowed = getAllowedFolders(req);
|
||||
let effectiveFolder = folder || undefined;
|
||||
let effectiveFolders = foldersArr;
|
||||
|
||||
if (allowed !== null) {
|
||||
const allowedSet = new Set(allowed);
|
||||
if (effectiveFolder) {
|
||||
// Requested specific folder — must be allowed
|
||||
if (!allowedSet.has(effectiveFolder)) {
|
||||
return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
|
||||
}
|
||||
} else if (effectiveFolders && effectiveFolders.length > 0) {
|
||||
// Requested multiple folders — intersect with allowed
|
||||
effectiveFolders = effectiveFolders.filter(f => allowedSet.has(f));
|
||||
if (effectiveFolders.length === 0) {
|
||||
return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
|
||||
}
|
||||
} else {
|
||||
// No folder filter — restrict to allowed folders only
|
||||
effectiveFolders = allowed;
|
||||
if (effectiveFolders.length === 0) {
|
||||
return res.json({ total: 0, offset: offsetNum, limit: limitNum, files: [] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { total, rows } = getMediaFiles({
|
||||
folder: folder || undefined,
|
||||
folders: foldersArr,
|
||||
folder: effectiveFolder,
|
||||
folders: effectiveFolders,
|
||||
type: type || 'all',
|
||||
sort: sort || 'latest',
|
||||
offset: offsetNum,
|
||||
limit: limitNum,
|
||||
dateFrom: dateFrom || undefined,
|
||||
dateTo: dateTo || undefined,
|
||||
minSize: minSize || undefined,
|
||||
maxSize: maxSize || undefined,
|
||||
search: search || undefined,
|
||||
});
|
||||
|
||||
const files = rows.map((r) => {
|
||||
@@ -197,6 +259,10 @@ router.get('/api/gallery/media/:folder/:filename', (req, res) => {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
}
|
||||
|
||||
if (!checkFolderAccess(req, folder)) {
|
||||
return res.status(403).json({ error: 'Access denied' });
|
||||
}
|
||||
|
||||
const filePath = join(MEDIA_PATH, folder, filename);
|
||||
res.sendFile(filePath, { root: '/' }, (err) => {
|
||||
if (err && !res.headersSent) {
|
||||
@@ -225,19 +291,54 @@ async function generateThumb(folder, filename) {
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
// Skip corrupt/empty files
|
||||
try {
|
||||
const st = statSync(videoPath);
|
||||
if (st.size < 1000) return null;
|
||||
} catch { return null; }
|
||||
|
||||
if (!existsSync(thumbDir)) mkdirSync(thumbDir, { recursive: true });
|
||||
|
||||
// Check if file has a video stream and get duration
|
||||
let hasVideo = false;
|
||||
let probeFailed = false;
|
||||
let duration = 0;
|
||||
try {
|
||||
const probe = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-select_streams', 'v',
|
||||
'-show_entries', 'stream=codec_type',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'json',
|
||||
videoPath,
|
||||
], { timeout: 15000 });
|
||||
const info = JSON.parse(probe.stdout);
|
||||
hasVideo = info.streams && info.streams.length > 0;
|
||||
duration = parseFloat(info.format?.duration || '0');
|
||||
} catch {
|
||||
probeFailed = true;
|
||||
}
|
||||
|
||||
if (!hasVideo && !probeFailed) return 'audio-only'; // confirmed audio-only, skip
|
||||
if (!hasVideo && probeFailed) return null; // probe failed, count as error
|
||||
|
||||
const seekTime = duration > 1.5 ? '1' : '0';
|
||||
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-ss', '1',
|
||||
'-ss', seekTime,
|
||||
'-i', videoPath,
|
||||
'-frames:v', '1',
|
||||
'-vf', 'scale=320:-1',
|
||||
'-pix_fmt', 'yuvj420p',
|
||||
'-q:v', '6',
|
||||
'-y',
|
||||
'-update', '1',
|
||||
thumbPath,
|
||||
], { timeout: 10000 });
|
||||
], { timeout: 30000 });
|
||||
return thumbPath;
|
||||
} catch (err) {
|
||||
console.error(`[gallery] thumb failed for ${key}:`, err.message);
|
||||
if (err.stderr) console.error(`[gallery] ffmpeg stderr:`, err.stderr.trim());
|
||||
return null;
|
||||
} finally {
|
||||
thumbInFlight.delete(key);
|
||||
@@ -248,44 +349,116 @@ async function generateThumb(folder, filename) {
|
||||
return promise;
|
||||
}
|
||||
|
||||
// GET /api/gallery/thumb/:folder/:filename — serve or generate a video thumbnail
|
||||
async function generateImageThumb(folder, filename) {
|
||||
const imagePath = join(MEDIA_PATH, folder, filename);
|
||||
const { thumbDir, thumbPath } = getThumbPath(folder, filename);
|
||||
|
||||
if (existsSync(thumbPath)) return thumbPath;
|
||||
|
||||
const key = `img:${folder}/${filename}`;
|
||||
if (thumbInFlight.has(key)) return thumbInFlight.get(key);
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
// Skip corrupt/empty files
|
||||
try {
|
||||
const st = statSync(imagePath);
|
||||
if (st.size < 100) return null;
|
||||
} catch { return null; }
|
||||
|
||||
if (!existsSync(thumbDir)) mkdirSync(thumbDir, { recursive: true });
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-i', imagePath,
|
||||
'-vf', 'scale=480:-1',
|
||||
'-q:v', '4',
|
||||
'-y',
|
||||
'-update', '1',
|
||||
thumbPath,
|
||||
], { timeout: 30000 });
|
||||
return thumbPath;
|
||||
} catch (err) {
|
||||
console.error(`[gallery] image thumb failed for ${folder}/${filename}:`, err.message);
|
||||
if (err.stderr) console.error(`[gallery] ffmpeg stderr:`, err.stderr.trim());
|
||||
return null;
|
||||
} finally {
|
||||
thumbInFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
|
||||
thumbInFlight.set(key, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
function serveFile(filePath, res) {
|
||||
try {
|
||||
const st = statSync(filePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Content-Length': st.size,
|
||||
'Cache-Control': 'public, max-age=86400',
|
||||
});
|
||||
createReadStream(filePath).pipe(res);
|
||||
} catch {
|
||||
if (!res.headersSent) {
|
||||
res.set('Cache-Control', 'no-cache');
|
||||
res.status(404).json({ error: 'Not found' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/gallery/thumb/:folder/:filename — serve or generate a thumbnail (video or image)
|
||||
router.get('/api/gallery/thumb/:folder/:filename', async (req, res) => {
|
||||
const { folder, filename } = req.params;
|
||||
if (folder.includes('..') || filename.includes('..')) {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
}
|
||||
|
||||
if (!checkFolderAccess(req, folder)) {
|
||||
return res.status(403).json({ error: 'Access denied' });
|
||||
}
|
||||
|
||||
const { thumbPath } = getThumbPath(folder, filename);
|
||||
|
||||
// Serve cached thumb immediately
|
||||
if (existsSync(thumbPath)) {
|
||||
return res.sendFile(thumbPath, { root: '/' }, (err) => {
|
||||
if (err && !res.headersSent) res.status(404).json({ error: 'Not found' });
|
||||
});
|
||||
return serveFile(thumbPath, res);
|
||||
}
|
||||
|
||||
// Determine type by extension
|
||||
const ext = filename.toLowerCase().split('.').pop();
|
||||
const isVideo = ['mp4', 'mov', 'avi', 'mkv', 'webm', 'wmv', 'flv', 'm4v'].includes(ext);
|
||||
const isImage = ['jpg', 'jpeg', 'png', 'webp', 'gif', 'bmp', 'tiff'].includes(ext);
|
||||
|
||||
let result;
|
||||
if (isVideo) {
|
||||
result = await generateThumb(folder, filename);
|
||||
} else if (isImage) {
|
||||
result = await generateImageThumb(folder, filename);
|
||||
}
|
||||
|
||||
// Generate on-demand
|
||||
const result = await generateThumb(folder, filename);
|
||||
if (result && existsSync(result)) {
|
||||
res.sendFile(result, { root: '/' }, (err) => {
|
||||
if (err && !res.headersSent) res.status(500).json({ error: 'Failed to serve thumbnail' });
|
||||
});
|
||||
serveFile(result, res);
|
||||
} else if (isImage) {
|
||||
// Fallback: serve original image
|
||||
const origPath = join(MEDIA_PATH, folder, filename);
|
||||
serveFile(origPath, res);
|
||||
} else {
|
||||
res.set('Cache-Control', 'no-cache');
|
||||
res.status(500).json({ error: 'Thumbnail generation failed' });
|
||||
}
|
||||
});
|
||||
|
||||
// Bulk thumbnail generation state
|
||||
let thumbGenState = { running: false, total: 0, done: 0, errors: 0 };
|
||||
let thumbGenState = { running: false, total: 0, done: 0, errors: 0, skipped: 0 };
|
||||
|
||||
// POST /api/gallery/generate-thumbs — bulk generate all video thumbnails
|
||||
// POST /api/gallery/generate-thumbs — bulk generate all thumbnails (videos + images)
|
||||
router.post('/api/gallery/generate-thumbs', (req, res) => {
|
||||
if (thumbGenState.running) {
|
||||
return res.json({ status: 'already_running', ...thumbGenState });
|
||||
}
|
||||
|
||||
// Collect all videos
|
||||
const videos = [];
|
||||
// Collect all media needing thumbs
|
||||
const mediaItems = [];
|
||||
const dirs = readdirSync(MEDIA_PATH, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.startsWith('_'));
|
||||
|
||||
@@ -296,36 +469,39 @@ router.post('/api/gallery/generate-thumbs', (req, res) => {
|
||||
for (const file of files) {
|
||||
if (file.startsWith('.')) continue;
|
||||
const ext = extname(file).toLowerCase();
|
||||
if (VIDEO_EXTS.has(ext)) {
|
||||
if (VIDEO_EXTS.has(ext) || IMAGE_EXTS.has(ext)) {
|
||||
const { thumbPath } = getThumbPath(dir.name, file);
|
||||
if (!existsSync(thumbPath)) {
|
||||
videos.push({ folder: dir.name, filename: file });
|
||||
mediaItems.push({ folder: dir.name, filename: file, type: VIDEO_EXTS.has(ext) ? 'video' : 'image' });
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { continue; }
|
||||
}
|
||||
|
||||
if (videos.length === 0) {
|
||||
if (mediaItems.length === 0) {
|
||||
return res.json({ status: 'done', total: 0, done: 0, errors: 0, message: 'All thumbnails already exist' });
|
||||
}
|
||||
|
||||
thumbGenState = { running: true, total: videos.length, done: 0, errors: 0 };
|
||||
res.json({ status: 'started', total: videos.length });
|
||||
thumbGenState = { running: true, total: mediaItems.length, done: 0, errors: 0, skipped: 0 };
|
||||
res.json({ status: 'started', total: mediaItems.length });
|
||||
|
||||
// Run in background with concurrency limit
|
||||
(async () => {
|
||||
const CONCURRENCY = 3;
|
||||
const CONCURRENCY = 2;
|
||||
let i = 0;
|
||||
const next = async () => {
|
||||
while (i < videos.length) {
|
||||
const { folder, filename } = videos[i++];
|
||||
const result = await generateThumb(folder, filename);
|
||||
if (result) thumbGenState.done++;
|
||||
while (i < mediaItems.length) {
|
||||
const { folder, filename, type } = mediaItems[i++];
|
||||
const result = type === 'video'
|
||||
? await generateThumb(folder, filename)
|
||||
: await generateImageThumb(folder, filename);
|
||||
if (result === 'audio-only') thumbGenState.skipped++;
|
||||
else if (result) thumbGenState.done++;
|
||||
else thumbGenState.errors++;
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, videos.length) }, () => next()));
|
||||
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, mediaItems.length) }, () => next()));
|
||||
thumbGenState.running = false;
|
||||
})();
|
||||
});
|
||||
@@ -351,12 +527,15 @@ function hashFilePartial(filePath, bytes = 65536) {
|
||||
}
|
||||
|
||||
// POST /api/gallery/scan-duplicates — start background duplicate scan
|
||||
// Query param: mode=everywhere (default) or mode=same-folder
|
||||
router.post('/api/gallery/scan-duplicates', (req, res) => {
|
||||
if (duplicateScanState.running) {
|
||||
return res.json({ status: 'already_running', ...duplicateScanState });
|
||||
}
|
||||
|
||||
// Phase 1: group all files by size
|
||||
const mode = req.query.mode || req.body?.mode || 'everywhere';
|
||||
|
||||
// Phase 1: group all files by size (optionally scoped per folder)
|
||||
const bySize = new Map();
|
||||
const dirs = readdirSync(MEDIA_PATH, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.startsWith('_'));
|
||||
@@ -372,7 +551,8 @@ router.post('/api/gallery/scan-duplicates', (req, res) => {
|
||||
const filePath = join(dirPath, file);
|
||||
try {
|
||||
const stat = statSync(filePath);
|
||||
const key = stat.size;
|
||||
// For same-folder mode, scope the size key by folder name
|
||||
const key = mode === 'same-folder' ? `${dir.name}:${stat.size}` : stat.size;
|
||||
if (!bySize.has(key)) bySize.set(key, []);
|
||||
bySize.get(key).push({ folder: dir.name, filename: file, type: mediaType, size: stat.size, modified: stat.mtimeMs, filePath });
|
||||
} catch { continue; }
|
||||
@@ -440,6 +620,10 @@ router.delete('/api/gallery/media/:folder/:filename', (req, res) => {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
}
|
||||
|
||||
if (!checkFolderAccess(req, folder)) {
|
||||
return res.status(403).json({ error: 'Access denied' });
|
||||
}
|
||||
|
||||
const filePath = join(MEDIA_PATH, folder, filename);
|
||||
if (!existsSync(filePath)) {
|
||||
return res.status(404).json({ error: 'File not found' });
|
||||
@@ -484,6 +668,7 @@ router.post('/api/gallery/duplicates/clean', (req, res) => {
|
||||
try {
|
||||
if (existsSync(filePath)) {
|
||||
unlinkSync(filePath);
|
||||
removeMediaFile(file.folder, file.filename);
|
||||
freed += file.size;
|
||||
deleted++;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { Router } from 'express';
|
||||
import { existsSync, accessSync, constants, statfsSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { getAuthConfig } from './db.js';
|
||||
import { getActiveDownloadCount } from './download.js';
|
||||
import { getActiveScrapeCount } from './scrape.js';
|
||||
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
const WVD_PATH = process.env.WVD_PATH || '/data/cdm/device.wvd';
|
||||
|
||||
router.get('/api/health', (req, res) => {
|
||||
const result = {
|
||||
uptime: Math.floor(process.uptime()),
|
||||
sqlite: false,
|
||||
authConfigured: false,
|
||||
mediaPathWritable: false,
|
||||
ffmpegAvailable: false,
|
||||
pythonAvailable: false,
|
||||
wvdPresent: false,
|
||||
diskSpace: null,
|
||||
activeDownloads: 0,
|
||||
activeScrapes: 0,
|
||||
};
|
||||
|
||||
// SQLite check
|
||||
try {
|
||||
getAuthConfig(); // simple query to verify DB works
|
||||
result.sqlite = true;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Auth check
|
||||
try {
|
||||
result.authConfigured = !!getAuthConfig();
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Media path writable
|
||||
try {
|
||||
accessSync(MEDIA_PATH, constants.W_OK);
|
||||
result.mediaPathWritable = true;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// FFmpeg
|
||||
try {
|
||||
execFileSync('ffmpeg', ['-version'], { timeout: 5000, stdio: 'pipe' });
|
||||
result.ffmpegAvailable = true;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Python
|
||||
try {
|
||||
execFileSync('python3', ['--version'], { timeout: 5000, stdio: 'pipe' });
|
||||
result.pythonAvailable = true;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// WVD file
|
||||
result.wvdPresent = existsSync(WVD_PATH);
|
||||
|
||||
// Disk space
|
||||
try {
|
||||
const stats = statfsSync(MEDIA_PATH);
|
||||
result.diskSpace = {
|
||||
free: stats.bfree * stats.bsize,
|
||||
total: stats.blocks * stats.bsize,
|
||||
};
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Active jobs
|
||||
result.activeDownloads = getActiveDownloadCount();
|
||||
result.activeScrapes = getActiveScrapeCount();
|
||||
|
||||
res.json(result);
|
||||
});
|
||||
|
||||
export default router;
|
||||
+351
-49
@@ -1,6 +1,6 @@
|
||||
import { Router } from 'express';
|
||||
import { join } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
|
||||
import { execFile, spawn } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { getSetting } from './db.js';
|
||||
@@ -8,7 +8,129 @@ import { getSetting } from './db.js';
|
||||
const execFileAsync = promisify(execFile);
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
const CACHE_DIR = join(MEDIA_PATH, '.hls-cache');
|
||||
const SEGMENT_DURATION = 10;
|
||||
const MAX_CONCURRENT_TRANSCODES = 2;
|
||||
const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
if (!existsSync(CACHE_DIR)) {
|
||||
mkdirSync(CACHE_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Quality tiers — always transcode for reliable HLS
|
||||
const QUALITY_TIERS = {
|
||||
'480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
|
||||
'720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
|
||||
'1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
|
||||
};
|
||||
|
||||
// Compute output dimensions preserving source aspect ratio
|
||||
function fitDimensions(srcW, srcH, maxW, maxH) {
|
||||
if (srcW <= maxW && srcH <= maxH) {
|
||||
let w = srcW, h = srcH;
|
||||
w += w % 2; h += h % 2;
|
||||
return { w, h };
|
||||
}
|
||||
const scale = Math.min(maxW / srcW, maxH / srcH);
|
||||
let w = Math.round(srcW * scale);
|
||||
let h = Math.round(srcH * scale);
|
||||
w += w % 2; h += h % 2;
|
||||
return { w, h };
|
||||
}
|
||||
|
||||
// --- Hardware acceleration detection (shared state with video-hls.js via same detection) ---
|
||||
let hwAccel = null; // 'vaapi' | 'qsv' | null
|
||||
let hwDetected = false;
|
||||
|
||||
async function detectHwAccel() {
|
||||
if (hwDetected) return hwAccel;
|
||||
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner',
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'format=nv12,hwupload',
|
||||
'-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
|
||||
hwAccel = 'vaapi';
|
||||
console.log('[hls] Intel VAAPI hardware acceleration available');
|
||||
} catch {
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner', '-init_hw_device', 'qsv=hw',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'hwupload=extra_hw_frames=64,format=qsv',
|
||||
'-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000 });
|
||||
hwAccel = 'qsv';
|
||||
console.log('[hls] Intel QSV hardware acceleration available');
|
||||
} catch {
|
||||
hwAccel = null;
|
||||
console.log('[hls] No hardware acceleration, using libx264');
|
||||
}
|
||||
}
|
||||
|
||||
hwDetected = true;
|
||||
return hwAccel;
|
||||
}
|
||||
|
||||
detectHwAccel();
|
||||
|
||||
// --- Transcode semaphore ---
|
||||
let activeTranscodes = 0;
|
||||
const transcodeQueue = [];
|
||||
|
||||
function acquireSlot() {
|
||||
return new Promise((resolve) => {
|
||||
if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
|
||||
activeTranscodes++;
|
||||
resolve();
|
||||
} else {
|
||||
transcodeQueue.push(resolve);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function releaseSlot() {
|
||||
activeTranscodes--;
|
||||
if (transcodeQueue.length > 0) {
|
||||
activeTranscodes++;
|
||||
transcodeQueue.shift()();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Cache cleanup (hourly) ---
|
||||
function cleanupCache() {
|
||||
try {
|
||||
if (!existsSync(CACHE_DIR)) return;
|
||||
const now = Date.now();
|
||||
const walk = (dir) => {
|
||||
let entries;
|
||||
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const e of entries) {
|
||||
const full = join(dir, e.name);
|
||||
if (e.isDirectory()) {
|
||||
walk(full);
|
||||
// Remove empty dirs
|
||||
try { if (readdirSync(full).length === 0) rmSync(full); } catch { /* ignore */ }
|
||||
} else if (e.name.endsWith('.ts')) {
|
||||
try {
|
||||
const st = statSync(full);
|
||||
if (now - st.mtimeMs > CACHE_MAX_AGE_MS) rmSync(full);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(CACHE_DIR);
|
||||
} catch (err) {
|
||||
console.error('[hls] Cache cleanup error:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(cleanupCache, 60 * 60 * 1000);
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function isHlsEnabled() {
|
||||
return (getSetting('hls_enabled') || process.env.HLS_ENABLED) === 'true';
|
||||
@@ -23,32 +145,89 @@ function validatePath(folder, filename) {
|
||||
return filePath;
|
||||
}
|
||||
|
||||
// GET /api/hls/:folder/:filename/master.m3u8
|
||||
// Sanitize folder+filename into a cache-safe directory name
|
||||
function cacheKey(folder, filename) {
|
||||
return join(folder, filename.replace(/[^a-zA-Z0-9._-]/g, '_'));
|
||||
}
|
||||
|
||||
async function probeVideo(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'stream=codec_type,width,height',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'json',
|
||||
filePath,
|
||||
], { timeout: 15000 });
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = info.streams?.find(s => s.codec_type === 'video');
|
||||
return {
|
||||
duration: parseFloat(info.format?.duration || '0'),
|
||||
width: videoStream?.width || 0,
|
||||
height: videoStream?.height || 0,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Master playlist ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
|
||||
if (!isHlsEnabled()) {
|
||||
return res.status(404).json({ error: 'HLS not enabled' });
|
||||
}
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename } = req.params;
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
}
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'csv=p=0',
|
||||
filePath,
|
||||
]);
|
||||
const info = await probeVideo(filePath);
|
||||
const srcW = info.width || 1920;
|
||||
const srcH = info.height || 1080;
|
||||
|
||||
const duration = parseFloat(stdout.trim());
|
||||
if (isNaN(duration) || duration <= 0) {
|
||||
let playlist = '#EXTM3U\n';
|
||||
|
||||
for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
|
||||
if (tier.maxH <= srcH) {
|
||||
const { w, h } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
|
||||
const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
|
||||
playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
|
||||
playlist += `${name}/playlist.m3u8\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// If source is too small for any tier, add a single tier at source resolution
|
||||
if (!playlist.includes('EXT-X-STREAM-INF')) {
|
||||
const { w, h } = fitDimensions(srcW, srcH, srcW, srcH);
|
||||
playlist += `#EXT-X-STREAM-INF:BANDWIDTH=1596000,RESOLUTION=${w}x${h},NAME="480p"\n`;
|
||||
playlist += `480p/playlist.m3u8\n`;
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[hls] Master playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate master playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Variant playlist ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/:quality/playlist.m3u8', async (req, res) => {
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename, quality } = req.params;
|
||||
if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
|
||||
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
try {
|
||||
const info = await probeVideo(filePath);
|
||||
const duration = info.duration;
|
||||
if (!duration || duration <= 0) {
|
||||
return res.status(500).json({ error: 'Could not determine video duration' });
|
||||
}
|
||||
|
||||
const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
|
||||
|
||||
let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
|
||||
playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
|
||||
playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
|
||||
@@ -63,54 +242,177 @@ router.get('/api/hls/:folder/:filename/master.m3u8', async (req, res) => {
|
||||
playlist += '#EXT-X-ENDLIST\n';
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[hls] ffprobe error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to probe video' });
|
||||
console.error('[hls] Variant playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate variant playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/hls/:folder/:filename/segment-:index.ts
|
||||
router.get('/api/hls/:folder/:filename/segment-:index.ts', (req, res) => {
|
||||
if (!isHlsEnabled()) {
|
||||
return res.status(404).json({ error: 'HLS not enabled' });
|
||||
}
|
||||
// --- Segment transcoding ---
|
||||
|
||||
router.get('/api/hls/:folder/:filename/:quality/segment-:index.ts', async (req, res) => {
|
||||
if (!isHlsEnabled()) return res.status(404).json({ error: 'HLS not enabled' });
|
||||
|
||||
const { folder, filename, quality } = req.params;
|
||||
const segIndex = parseInt(req.params.index, 10);
|
||||
|
||||
if (isNaN(segIndex) || segIndex < 0) return res.status(400).json({ error: 'Invalid segment index' });
|
||||
if (!QUALITY_TIERS[quality]) return res.status(400).json({ error: 'Invalid quality' });
|
||||
|
||||
const { folder, filename, index } = req.params;
|
||||
const filePath = validatePath(folder, filename);
|
||||
if (!filePath) {
|
||||
return res.status(400).json({ error: 'Invalid path' });
|
||||
if (!filePath) return res.status(400).json({ error: 'Invalid path' });
|
||||
|
||||
// Check cache
|
||||
const key = cacheKey(folder, filename);
|
||||
const segCacheDir = join(CACHE_DIR, key, quality);
|
||||
const segCachePath = join(segCacheDir, `segment-${segIndex}.ts`);
|
||||
|
||||
if (existsSync(segCachePath)) {
|
||||
const stat = statSync(segCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const segIndex = parseInt(index, 10);
|
||||
if (isNaN(segIndex) || segIndex < 0) {
|
||||
return res.status(400).json({ error: 'Invalid segment index' });
|
||||
await acquireSlot();
|
||||
|
||||
// Double-check cache after acquiring slot
|
||||
if (existsSync(segCachePath)) {
|
||||
releaseSlot();
|
||||
const stat = statSync(segCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const offset = segIndex * SEGMENT_DURATION;
|
||||
try {
|
||||
const offset = segIndex * SEGMENT_DURATION;
|
||||
const accel = await detectHwAccel();
|
||||
const tier = QUALITY_TIERS[quality];
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', [
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-c', 'copy',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
// Probe source dimensions for aspect-ratio-aware scaling
|
||||
let srcW = 1920, srcH = 1080;
|
||||
try {
|
||||
const info = await probeVideo(filePath);
|
||||
srcW = info.width || 1920;
|
||||
srcH = info.height || 1080;
|
||||
} catch { /* use defaults */ }
|
||||
|
||||
res.setHeader('Content-Type', 'video/MP2T');
|
||||
ffmpeg.stdout.pipe(res);
|
||||
const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
|
||||
|
||||
req.on('close', () => {
|
||||
ffmpeg.kill('SIGKILL');
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
console.error('[hls] ffmpeg error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Transcoding failed' });
|
||||
let ffmpegArgs;
|
||||
if (accel === 'vaapi') {
|
||||
ffmpegArgs = [
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-filter_hw_device', 'va',
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_vaapi',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else if (accel === 'qsv') {
|
||||
ffmpegArgs = [
|
||||
'-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale_qsv=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_qsv',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else {
|
||||
ffmpegArgs = [
|
||||
'-ss', String(offset),
|
||||
'-i', filePath,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale=${outW}:${outH}`,
|
||||
'-c:v', 'libx264', '-preset', 'veryfast',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
const spawnEnv = accel === 'vaapi'
|
||||
? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
|
||||
: undefined;
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
...(spawnEnv && { env: spawnEnv }),
|
||||
});
|
||||
|
||||
res.setHeader('Content-Type', 'video/MP2T');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
|
||||
const cacheChunks = [];
|
||||
let aborted = false;
|
||||
|
||||
ffmpeg.stdout.on('data', (chunk) => {
|
||||
cacheChunks.push(chunk);
|
||||
if (!res.destroyed) res.write(chunk);
|
||||
});
|
||||
|
||||
req.on('close', () => {
|
||||
if (!ffmpeg.killed) {
|
||||
aborted = true;
|
||||
ffmpeg.kill('SIGKILL');
|
||||
releaseSlot();
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (aborted) return;
|
||||
releaseSlot();
|
||||
|
||||
if (code === 0 && cacheChunks.length > 0) {
|
||||
try {
|
||||
if (!existsSync(segCacheDir)) mkdirSync(segCacheDir, { recursive: true });
|
||||
const ws = createWriteStream(segCachePath);
|
||||
for (const c of cacheChunks) ws.write(c);
|
||||
ws.end();
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
if (!res.destroyed) res.end();
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (!aborted) releaseSlot();
|
||||
console.error('[hls] ffmpeg error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Transcoding failed' });
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
releaseSlot();
|
||||
console.error('[hls] Segment error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
@@ -1,19 +1,28 @@
|
||||
import express from 'express';
|
||||
import https from 'https';
|
||||
import cors from 'cors';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import { existsSync, readFileSync, mkdirSync, writeFileSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
import { initRules } from './signing.js';
|
||||
import authRouter, { requireAuth, checkRoutePermission } from './auth.js';
|
||||
import proxyRouter from './proxy.js';
|
||||
import downloadRouter from './download.js';
|
||||
import galleryRouter from './gallery.js';
|
||||
import hlsRouter from './hls.js';
|
||||
import settingsRouter from './settings.js';
|
||||
import scrapeRouter from './scrape.js';
|
||||
import flareSolverrRouter from './flaresolverr.js';
|
||||
import drmStreamRouter from './drm-stream.js';
|
||||
import healthRouter from './health.js';
|
||||
import dashboardRouter from './dashboard.js';
|
||||
import videosRouter from './videos.js';
|
||||
import videoHlsRouter from './video-hls.js';
|
||||
import mediaApiRouter from './media-api.js';
|
||||
import { scanMediaFiles } from './gallery.js';
|
||||
import { startScheduler } from './scheduler.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -23,11 +32,28 @@ const PORT = process.env.PORT || 3001;
|
||||
const HTTPS_PORT = process.env.HTTPS_PORT || 3443;
|
||||
|
||||
app.use(cors());
|
||||
app.use(cookieParser());
|
||||
// Parse DRM license request bodies as raw binary BEFORE global JSON parser
|
||||
// (express.json can interfere with reading the raw body stream)
|
||||
app.use('/api/drm-license', express.raw({ type: '*/*', limit: '1mb' }));
|
||||
app.use(express.json());
|
||||
|
||||
// Auth routes (public endpoints like login/setup must be before requireAuth)
|
||||
app.use(authRouter);
|
||||
|
||||
// Apply auth middleware globally (after auth routes)
|
||||
app.use('/api', (req, res, next) => {
|
||||
// Skip auth for app-auth public endpoints
|
||||
if (req.path.startsWith('/app-auth/')) return next();
|
||||
// Skip auth for internal DRM license requests from pywidevine subprocess
|
||||
if (req.path.startsWith('/drm-license') && ['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(req.ip)) return next();
|
||||
requireAuth(req, res, next);
|
||||
});
|
||||
app.use('/api', (req, res, next) => {
|
||||
if (req.path.startsWith('/app-auth/')) return next();
|
||||
checkRoutePermission(req, res, next);
|
||||
});
|
||||
|
||||
// API routes
|
||||
app.use(proxyRouter);
|
||||
app.use(downloadRouter);
|
||||
@@ -35,7 +61,13 @@ app.use(galleryRouter);
|
||||
app.use(hlsRouter);
|
||||
app.use(settingsRouter);
|
||||
app.use(scrapeRouter);
|
||||
app.use(flareSolverrRouter);
|
||||
app.use(drmStreamRouter);
|
||||
app.use(healthRouter);
|
||||
app.use(dashboardRouter);
|
||||
app.use(videosRouter);
|
||||
app.use(videoHlsRouter);
|
||||
app.use(mediaApiRouter);
|
||||
|
||||
// Serve static client build in production
|
||||
const clientDist = join(__dirname, '..', 'client', 'dist');
|
||||
@@ -70,6 +102,8 @@ async function start() {
|
||||
console.error('[server] Media scan failed:', err.message);
|
||||
}
|
||||
});
|
||||
// Start auto-download/scrape scheduler
|
||||
startScheduler();
|
||||
});
|
||||
|
||||
// Start HTTPS server for DRM/EME support (requires secure context)
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Login helper using undetected_chromedriver to bypass Cloudflare Turnstile.
|
||||
Runs Chrome in headed mode with Xvfb (virtual display) so Turnstile sees a real browser.
|
||||
|
||||
Usage:
|
||||
xvfb-run python3 login_helper.py <login_url> <username> <password>
|
||||
|
||||
Outputs JSON to stdout:
|
||||
{"ok": true, "cookies": "name=val; name2=val2", "url": "<final_url>"}
|
||||
{"ok": false, "error": "reason"}
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 4:
|
||||
print(json.dumps({"ok": False, "error": "Usage: login_helper.py <login_url> <username> <password>"}))
|
||||
sys.exit(1)
|
||||
|
||||
login_url = sys.argv[1]
|
||||
username = sys.argv[2]
|
||||
password = sys.argv[3]
|
||||
|
||||
try:
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
except ImportError as e:
|
||||
print(json.dumps({"ok": False, "error": f"Missing dependency: {e}"}))
|
||||
sys.exit(1)
|
||||
|
||||
driver = None
|
||||
try:
|
||||
# Find chromium binary
|
||||
chromium_path = os.environ.get('CHROMIUM_PATH', '')
|
||||
if not chromium_path or not os.path.exists(chromium_path):
|
||||
for p in ['/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/lib/chromium/chromium']:
|
||||
if os.path.exists(p):
|
||||
chromium_path = p
|
||||
break
|
||||
|
||||
# Find system chromedriver (Alpine: chromium-chromedriver package)
|
||||
chromedriver_path = None
|
||||
for p in ['/usr/bin/chromedriver', '/usr/lib/chromium/chromedriver']:
|
||||
if os.path.exists(p):
|
||||
chromedriver_path = p
|
||||
break
|
||||
|
||||
# Get chromium version for undetected_chromedriver
|
||||
version_main = None
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run([chromium_path, '--version'], capture_output=True, text=True, timeout=5)
|
||||
# e.g. "Chromium 131.0.6778.139" or "Chromium 131.0.6778.139 Alpine Linux"
|
||||
parts = result.stdout.strip().split()
|
||||
ver_str = None
|
||||
for part in parts:
|
||||
if '.' in part and part[0].isdigit():
|
||||
ver_str = part
|
||||
break
|
||||
if ver_str:
|
||||
version_main = int(ver_str.split('.')[0])
|
||||
log(f"Chromium version: {ver_str} (major: {version_main})")
|
||||
except Exception as e:
|
||||
log(f"Could not detect chromium version: {e}")
|
||||
|
||||
options = uc.ChromeOptions()
|
||||
options.binary_location = chromium_path
|
||||
options.add_argument('--no-sandbox')
|
||||
options.add_argument('--disable-dev-shm-usage')
|
||||
options.add_argument('--disable-gpu')
|
||||
options.add_argument('--window-size=1920,1080')
|
||||
|
||||
log(f"Chromium: {chromium_path}")
|
||||
log(f"Chromedriver: {chromedriver_path}")
|
||||
|
||||
# Create the driver
|
||||
# Use system chromedriver to avoid downloading (fails on Alpine/musl)
|
||||
driver = uc.Chrome(
|
||||
options=options,
|
||||
driver_executable_path=chromedriver_path,
|
||||
headless=False,
|
||||
version_main=version_main,
|
||||
)
|
||||
driver.set_window_size(1920, 1080)
|
||||
|
||||
log(f"Navigating to {login_url}...")
|
||||
driver.get(login_url)
|
||||
|
||||
# Wait for DDoS-Guard to solve and login form to appear
|
||||
log("Waiting for login form (DDoS-Guard solving)...")
|
||||
WebDriverWait(driver, 60).until(
|
||||
EC.presence_of_element_located((By.CSS_SELECTOR, 'input[name="login"]'))
|
||||
)
|
||||
log("Login form found")
|
||||
|
||||
# Wait for Turnstile to auto-solve (should work in undetected headed mode)
|
||||
log("Waiting for Turnstile to solve...")
|
||||
turnstile_token = ""
|
||||
for i in range(45):
|
||||
try:
|
||||
el = driver.find_element(By.CSS_SELECTOR, 'input[name="cf-turnstile-response"]')
|
||||
val = el.get_attribute("value")
|
||||
if val:
|
||||
turnstile_token = val
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
if i % 10 == 9:
|
||||
log(f"Still waiting for Turnstile... ({i+1}s)")
|
||||
|
||||
if turnstile_token:
|
||||
log(f"Turnstile solved (token: {turnstile_token[:20]}...)")
|
||||
else:
|
||||
log("Warning: Turnstile token not found after 45s — attempting login anyway")
|
||||
|
||||
# Fill the login form
|
||||
log("Filling login form...")
|
||||
login_input = driver.find_element(By.CSS_SELECTOR, 'input[name="login"]')
|
||||
login_input.clear()
|
||||
# Type slowly to appear human
|
||||
for ch in username:
|
||||
login_input.send_keys(ch)
|
||||
time.sleep(0.03)
|
||||
|
||||
pass_input = driver.find_element(By.CSS_SELECTOR, 'input[name="password"]')
|
||||
pass_input.clear()
|
||||
for ch in password:
|
||||
pass_input.send_keys(ch)
|
||||
time.sleep(0.03)
|
||||
|
||||
# Check remember checkbox
|
||||
try:
|
||||
remember = driver.find_element(By.CSS_SELECTOR, 'input[name="remember"]')
|
||||
if not remember.is_selected():
|
||||
driver.execute_script("arguments[0].checked = true;", remember)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Submit form
|
||||
log("Submitting login form...")
|
||||
try:
|
||||
submit_btn = driver.find_element(By.CSS_SELECTOR,
|
||||
'button[type="submit"], input[type="submit"], .button--primary')
|
||||
submit_btn.click()
|
||||
except Exception:
|
||||
driver.execute_script("""
|
||||
var form = document.querySelector('form.block-body') ||
|
||||
document.querySelector('form[action*="login"]');
|
||||
if (form) form.submit();
|
||||
""")
|
||||
|
||||
# Wait for navigation after submit
|
||||
log("Waiting for redirect...")
|
||||
time.sleep(5)
|
||||
try:
|
||||
WebDriverWait(driver, 15).until(
|
||||
lambda d: d.execute_script("return document.readyState") == "complete"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
final_url = driver.current_url
|
||||
log(f"After submit: {final_url}")
|
||||
|
||||
# Extract cookies
|
||||
cookies = driver.get_cookies()
|
||||
cookie_str = "; ".join(f"{c['name']}={c['value']}" for c in cookies)
|
||||
|
||||
# Check for login success
|
||||
has_user_cookie = any(c['name'] in ('xf_user', 'ogaddgmetaprof_user') for c in cookies)
|
||||
|
||||
if not has_user_cookie:
|
||||
# Check for error message
|
||||
error_msg = "Login failed — no user cookie returned"
|
||||
try:
|
||||
error_el = driver.find_element(By.CSS_SELECTOR, '.blockMessage--error')
|
||||
error_msg = error_el.text.strip()
|
||||
except Exception:
|
||||
pass
|
||||
# Also dump all cookie names for debugging
|
||||
cookie_names = [c['name'] for c in cookies]
|
||||
log(f"Cookie names: {cookie_names}")
|
||||
log(f"Error: {error_msg}")
|
||||
print(json.dumps({"ok": False, "error": error_msg, "url": final_url}))
|
||||
sys.exit(1)
|
||||
|
||||
log(f"Login successful — {len(cookies)} cookies")
|
||||
print(json.dumps({"ok": True, "cookies": cookie_str, "url": final_url}))
|
||||
|
||||
except Exception as e:
|
||||
log(f"Fatal error: {e}")
|
||||
print(json.dumps({"ok": False, "error": str(e)}))
|
||||
sys.exit(1)
|
||||
finally:
|
||||
if driver:
|
||||
try:
|
||||
driver.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def log(msg):
|
||||
"""Log to stderr so it doesn't interfere with JSON stdout."""
|
||||
print(f"[login_helper] {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Router } from 'express';
|
||||
import { getMediaFiles, getSetting, getUserFolderAccess } from './db.js';
|
||||
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
|
||||
// GET /api/media?users=folder1,folder2&type=video,image
|
||||
router.get('/api/media', (req, res) => {
|
||||
const { users, type } = req.query;
|
||||
|
||||
if (!users) return res.status(400).json({ error: 'users parameter is required' });
|
||||
if (!type) return res.status(400).json({ error: 'type parameter is required' });
|
||||
|
||||
const folders = users.split(',').map(u => u.trim()).filter(Boolean);
|
||||
const types = type.split(',').map(t => t.trim().toLowerCase()).filter(Boolean);
|
||||
|
||||
if (folders.length === 0) return res.status(400).json({ error: 'at least one user/folder is required' });
|
||||
|
||||
const validTypes = ['video', 'image'];
|
||||
const invalid = types.find(t => !validTypes.includes(t));
|
||||
if (invalid) return res.status(400).json({ error: `invalid type: ${invalid}. Must be video, image, or both` });
|
||||
|
||||
// Enforce folder access for non-admin users
|
||||
if (req.user && req.user.role !== 'admin') {
|
||||
const allowed = getUserFolderAccess(req.user.id);
|
||||
if (allowed.length > 0) {
|
||||
const denied = folders.filter(f => !allowed.includes(f));
|
||||
if (denied.length > 0) {
|
||||
return res.status(403).json({ error: `access denied to folders: ${denied.join(', ')}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If both types requested, query all; otherwise query the single type
|
||||
const typeFilter = types.length === 2 ? 'all' : types[0];
|
||||
|
||||
const { rows } = getMediaFiles({
|
||||
folders,
|
||||
type: typeFilter,
|
||||
offset: 0,
|
||||
limit: 999999999,
|
||||
});
|
||||
|
||||
const hlsEnabled = getSetting('hls_enabled') === 'true' || process.env.HLS_ENABLED === 'true';
|
||||
|
||||
const results = rows.map(r => {
|
||||
const item = {
|
||||
folder: r.folder,
|
||||
path: `${MEDIA_PATH}/${r.folder}/${r.filename}`,
|
||||
type: r.type,
|
||||
};
|
||||
if (r.type === 'video' && hlsEnabled) {
|
||||
item.hlsUrl = `/api/hls/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}/master.m3u8`;
|
||||
} else if (r.type === 'video') {
|
||||
item.mediaUrl = `/api/gallery/media/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}`;
|
||||
}
|
||||
if (r.type === 'image') {
|
||||
item.mediaUrl = `/api/gallery/media/${encodeURIComponent(r.folder)}/${encodeURIComponent(r.filename)}`;
|
||||
}
|
||||
return item;
|
||||
});
|
||||
|
||||
res.json(results);
|
||||
});
|
||||
|
||||
export default router;
|
||||
Generated
+275
@@ -8,10 +8,15 @@
|
||||
"name": "ofapp-server",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"bcryptjs": "^3.0.3",
|
||||
"better-sqlite3": "^11.0.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.21.0",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"megajs": "^1.3.9",
|
||||
"multer": "^2.0.2",
|
||||
"node-fetch": "^3.3.2"
|
||||
}
|
||||
},
|
||||
@@ -28,6 +33,12 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/append-field": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
|
||||
"integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/array-flatten": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
|
||||
@@ -54,6 +65,15 @@
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bcryptjs": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz",
|
||||
"integrity": "sha512-GlF5wPWnSa/X5LKM1o0wz0suXIINz1iHRLvTS+sLyi7XPbe5ycmYI3DlZqVGZZtDgl4DmasFg7gOB3JYbphV5g==",
|
||||
"license": "BSD-3-Clause",
|
||||
"bin": {
|
||||
"bcrypt": "bin/bcrypt"
|
||||
}
|
||||
},
|
||||
"node_modules/better-sqlite3": {
|
||||
"version": "11.10.0",
|
||||
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz",
|
||||
@@ -139,6 +159,29 @@
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-equal-constant-time": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
|
||||
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/buffer-from": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
|
||||
"dependencies": {
|
||||
"streamsearch": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
@@ -225,6 +268,21 @@
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/concat-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.0.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -255,6 +313,25 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/cookie-parser": {
|
||||
"version": "1.4.7",
|
||||
"resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
|
||||
"integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cookie": "0.7.2",
|
||||
"cookie-signature": "1.0.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cookie-parser/node_modules/cookie-signature": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
|
||||
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cookie-signature": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz",
|
||||
@@ -445,6 +522,27 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/duplexify": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",
|
||||
"integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"end-of-stream": "^1.4.1",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.1.1",
|
||||
"stream-shift": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/ecdsa-sig-formatter": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
|
||||
"integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/ee-first": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
||||
@@ -881,6 +979,97 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/jsonwebtoken": {
|
||||
"version": "9.0.3",
|
||||
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz",
|
||||
"integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"jws": "^4.0.1",
|
||||
"lodash.includes": "^4.3.0",
|
||||
"lodash.isboolean": "^3.0.3",
|
||||
"lodash.isinteger": "^4.0.4",
|
||||
"lodash.isnumber": "^3.0.3",
|
||||
"lodash.isplainobject": "^4.0.6",
|
||||
"lodash.isstring": "^4.0.1",
|
||||
"lodash.once": "^4.0.0",
|
||||
"ms": "^2.1.1",
|
||||
"semver": "^7.5.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12",
|
||||
"npm": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/jsonwebtoken/node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/jwa": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",
|
||||
"integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-equal-constant-time": "^1.0.1",
|
||||
"ecdsa-sig-formatter": "1.0.11",
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/jws": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
|
||||
"integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"jwa": "^2.0.1",
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.includes": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
|
||||
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isboolean": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
|
||||
"integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isinteger": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
|
||||
"integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isnumber": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
|
||||
"integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isplainobject": {
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
|
||||
"integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isstring": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
|
||||
"integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.once": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
|
||||
"integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/math-intrinsics": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
|
||||
@@ -899,6 +1088,16 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/megajs": {
|
||||
"version": "1.3.9",
|
||||
"resolved": "https://registry.npmjs.org/megajs/-/megajs-1.3.9.tgz",
|
||||
"integrity": "sha512-91GGJbUfUu9z/KFORHcn4bugVILWcGahaoy07Q7M5GLzT6zOsrpusxkjEvEys9XCXbxntg0v+f2JN6sITrEkPQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pumpify": "^2.0.1",
|
||||
"stream-skip": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/merge-descriptors": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
|
||||
@@ -971,6 +1170,18 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/mkdirp": {
|
||||
"version": "0.5.6",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
|
||||
"integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"minimist": "^1.2.6"
|
||||
},
|
||||
"bin": {
|
||||
"mkdirp": "bin/cmd.js"
|
||||
}
|
||||
},
|
||||
"node_modules/mkdirp-classic": {
|
||||
"version": "0.5.3",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
|
||||
@@ -983,6 +1194,24 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/multer": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
|
||||
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
"concat-stream": "^2.0.0",
|
||||
"mkdirp": "^0.5.6",
|
||||
"object-assign": "^4.1.1",
|
||||
"type-is": "^1.6.18",
|
||||
"xtend": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/napi-build-utils": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
|
||||
@@ -1215,6 +1444,17 @@
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/pumpify": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz",
|
||||
"integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"duplexify": "^4.1.1",
|
||||
"inherits": "^2.0.3",
|
||||
"pump": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/qs": {
|
||||
"version": "6.14.1",
|
||||
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz",
|
||||
@@ -1498,6 +1738,26 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-shift": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",
|
||||
"integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/stream-skip": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/stream-skip/-/stream-skip-1.0.3.tgz",
|
||||
"integrity": "sha512-2rB0uBiOnYSQwJxJ3wZLher+fz0yyXQxKuKnVTsidHmkqvC8rWZ2AbX50ZVdz7fsL6zkYkqaN/pPD0RldKIbpQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
@@ -1578,6 +1838,12 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/typedarray": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
|
||||
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.22.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.22.0.tgz",
|
||||
@@ -1668,6 +1934,15 @@
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,15 @@
|
||||
"dev": "node --watch index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"bcryptjs": "^3.0.3",
|
||||
"better-sqlite3": "^11.0.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.21.0",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"megajs": "^1.3.9",
|
||||
"multer": "^2.0.2",
|
||||
"node-fetch": "^3.3.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,23 @@ async function proxyGet(ofPath, authConfig) {
|
||||
return { status: res.status, data };
|
||||
}
|
||||
|
||||
// GET /api/auth/check — validate auth by calling OF API
|
||||
router.get('/api/auth/check', async (req, res) => {
|
||||
try {
|
||||
const authConfig = getAuthConfig();
|
||||
if (!authConfig) {
|
||||
return res.json({ valid: false, error: 'No auth configured' });
|
||||
}
|
||||
const { status, data } = await proxyGet('/api2/v2/users/me', authConfig);
|
||||
if (status === 200 && data && data.id) {
|
||||
return res.json({ valid: true, user: { id: data.id, name: data.name, username: data.username } });
|
||||
}
|
||||
return res.json({ valid: false, error: data?.error?.message || data?.message || `HTTP ${status}` });
|
||||
} catch (err) {
|
||||
return res.json({ valid: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/auth
|
||||
router.get('/api/auth', (req, res) => {
|
||||
const config = getAuthConfig();
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { getAuthConfig, getAutoDownloadUsers, updateAutoDownloadLastRun, getAutoScrapeJobs, updateAutoScrapeLastRun } from './db.js';
|
||||
import { runDownload } from './download.js';
|
||||
import { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, createJob } from './scrape.js';
|
||||
|
||||
const INTERVAL = 12 * 60 * 60 * 1000; // 12 hours
|
||||
const STARTUP_DELAY = 30 * 1000; // 30 seconds
|
||||
|
||||
async function runAutoDownloads() {
|
||||
const users = getAutoDownloadUsers();
|
||||
if (users.length === 0) return;
|
||||
|
||||
const authConfig = getAuthConfig();
|
||||
if (!authConfig) {
|
||||
console.log('[scheduler] Skipping auto-downloads: no auth config');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[scheduler] Starting auto-downloads for ${users.length} user(s)`);
|
||||
|
||||
for (const user of users) {
|
||||
try {
|
||||
console.log(`[scheduler] Downloading ${user.username} (${user.user_id})`);
|
||||
await runDownload(user.user_id, authConfig, null, true, user.username);
|
||||
updateAutoDownloadLastRun(user.user_id);
|
||||
console.log(`[scheduler] Completed download for ${user.username}`);
|
||||
} catch (err) {
|
||||
console.error(`[scheduler] Error downloading ${user.username}:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[scheduler] Auto-downloads complete');
|
||||
}
|
||||
|
||||
async function runAutoScrapes() {
|
||||
const jobs = getAutoScrapeJobs();
|
||||
if (jobs.length === 0) return;
|
||||
|
||||
console.log(`[scheduler] Starting auto-scrapes for ${jobs.length} job(s)`);
|
||||
|
||||
for (const savedJob of jobs) {
|
||||
try {
|
||||
const config = JSON.parse(savedJob.config);
|
||||
config.folderName = savedJob.folder_name;
|
||||
const job = createJob(savedJob.type, config);
|
||||
|
||||
console.log(`[scheduler] Running ${savedJob.type} scrape for ${savedJob.folder_name}`);
|
||||
|
||||
if (savedJob.type === 'forum') {
|
||||
await runForumScrape(job);
|
||||
} else if (savedJob.type === 'coomer') {
|
||||
await runCoomerScrape(job);
|
||||
} else if (savedJob.type === 'medialink') {
|
||||
await runMediaLinkScrape(job);
|
||||
} else if (savedJob.type === 'mega') {
|
||||
await runMegaScrape(job);
|
||||
}
|
||||
|
||||
updateAutoScrapeLastRun(savedJob.id);
|
||||
console.log(`[scheduler] Completed scrape for ${savedJob.folder_name}`);
|
||||
} catch (err) {
|
||||
console.error(`[scheduler] Error scraping ${savedJob.folder_name}:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[scheduler] Auto-scrapes complete');
|
||||
}
|
||||
|
||||
async function runAll() {
|
||||
try {
|
||||
await runAutoDownloads();
|
||||
} catch (err) {
|
||||
console.error('[scheduler] Auto-download batch failed:', err.message);
|
||||
}
|
||||
try {
|
||||
await runAutoScrapes();
|
||||
} catch (err) {
|
||||
console.error('[scheduler] Auto-scrape batch failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
export function startScheduler() {
|
||||
// Run once shortly after startup
|
||||
setTimeout(() => {
|
||||
console.log('[scheduler] Running initial auto-download/scrape check');
|
||||
runAll();
|
||||
}, STARTUP_DELAY);
|
||||
|
||||
// Then every 12 hours
|
||||
setInterval(() => {
|
||||
console.log('[scheduler] Running scheduled auto-download/scrape');
|
||||
runAll();
|
||||
}, INTERVAL);
|
||||
|
||||
console.log('[scheduler] Scheduler started (interval: 12h)');
|
||||
}
|
||||
+429
-22
@@ -1,9 +1,14 @@
|
||||
import { Router } from 'express';
|
||||
import { mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { scrapeForumPage, getPageUrl, detectMaxPage } from './scrapers/forum.js';
|
||||
import { parseUserUrl, fetchAllPosts, downloadFiles } from './scrapers/coomer.js';
|
||||
import { parseMediaUrl, fetchAllMedia, downloadMedia } from './scrapers/medialink.js';
|
||||
import { scrapeForumPage, getPageUrl, detectMaxPage, CookieExpiredError } from './scrapers/forum.js';
|
||||
import { refreshForumCookies } from './flaresolverr.js';
|
||||
import { parseUserUrl, fetchAllPosts, fetchSearchPosts, downloadFiles } from './scrapers/coomer.js';
|
||||
import { parseMediaUrl, fetchAllMedia, fetchAllMediaFromHtml, downloadMedia } from './scrapers/medialink.js';
|
||||
import { parseMegaUrl, listAllFiles, downloadMegaFiles } from './scrapers/mega.js';
|
||||
import { runYtdlp } from './scrapers/ytdlp.js';
|
||||
import { parseLeakGalleryUrl, fetchAllMedia as fetchLeakGalleryMedia, downloadMedia as downloadLeakGalleryMedia } from './scrapers/leakgallery.js';
|
||||
import { getAutoScrapeJobs, addAutoScrapeJob, removeAutoScrapeJob, getForumSites, getForumSiteById, createForumSite, updateForumSite, deleteForumSite } from './db.js';
|
||||
|
||||
const router = Router();
|
||||
const MEDIA_PATH = process.env.MEDIA_PATH || './data/media';
|
||||
@@ -56,6 +61,8 @@ function jobToJson(job) {
|
||||
progress: job.progress,
|
||||
running: job.running,
|
||||
cancelled: job.cancelled,
|
||||
paused: job.paused || false,
|
||||
resumeAt: job.resumeAt || null,
|
||||
folderName: job.folderName,
|
||||
startedAt: job.startedAt,
|
||||
completedAt: job.completedAt,
|
||||
@@ -66,13 +73,38 @@ function jobToJson(job) {
|
||||
// --- Forum Scrape ---
|
||||
|
||||
async function runForumScrape(job) {
|
||||
const { url, startPage, endPage, delay, folderName } = job.config;
|
||||
let { url, startPage, endPage, delay, folderName, siteId, lastPageOnly } = job.config;
|
||||
let { cookies } = job.config;
|
||||
|
||||
// Load cookies from forum site record if siteId provided and no cookies passed
|
||||
if (!cookies && siteId) {
|
||||
const site = getForumSiteById(siteId);
|
||||
if (site && site.cookies) {
|
||||
cookies = site.cookies;
|
||||
job.config.cookies = cookies;
|
||||
addLog(job, `Loaded cookies from forum site: ${site.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const downloadedSet = new Set();
|
||||
let totalImages = 0;
|
||||
|
||||
// If lastPageOnly, detect the last page and only scrape that
|
||||
if (lastPageOnly) {
|
||||
addLog(job, 'Detecting last page...');
|
||||
const maxPage = await detectMaxPage(url, (msg) => addLog(job, msg), cookies);
|
||||
if (maxPage) {
|
||||
startPage = maxPage;
|
||||
endPage = maxPage;
|
||||
addLog(job, `Last page detected: ${maxPage}`);
|
||||
} else {
|
||||
addLog(job, 'Could not detect last page — falling back to page range');
|
||||
}
|
||||
}
|
||||
|
||||
addLog(job, `Starting forum scrape: pages ${startPage}-${endPage}`);
|
||||
addLog(job, `Output: ${outputDir}`);
|
||||
|
||||
@@ -88,7 +120,31 @@ async function runForumScrape(job) {
|
||||
const pageUrl = getPageUrl(url, page);
|
||||
addLog(job, `--- Page ${page}/${endPage} ---`);
|
||||
|
||||
const count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg));
|
||||
let count;
|
||||
try {
|
||||
count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
|
||||
} catch (err) {
|
||||
if (err instanceof CookieExpiredError && siteId) {
|
||||
addLog(job, `Cookie expired (HTTP ${err.statusCode}) — attempting auto-refresh via FlareSolverr...`);
|
||||
try {
|
||||
cookies = await refreshForumCookies(siteId);
|
||||
job.config.cookies = cookies;
|
||||
addLog(job, 'Cookies refreshed successfully — retrying page...');
|
||||
count = await scrapeForumPage(pageUrl, outputDir, downloadedSet, (msg) => addLog(job, msg), cookies);
|
||||
} catch (refreshErr) {
|
||||
addLog(job, `Cookie refresh failed: ${refreshErr.message}`);
|
||||
addLog(job, 'Stopping scrape — fix credentials or refresh cookies manually');
|
||||
break;
|
||||
}
|
||||
} else if (err instanceof CookieExpiredError) {
|
||||
addLog(job, `Cookie expired (HTTP ${err.statusCode}) — no siteId configured for auto-refresh`);
|
||||
addLog(job, 'Stopping scrape — refresh cookies manually and try again');
|
||||
break;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
totalImages += count;
|
||||
job.progress.completed = page - startPage + 1;
|
||||
|
||||
@@ -102,7 +158,7 @@ async function runForumScrape(job) {
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
addLog(job, `Done! ${totalImages} images saved to ${folderName}/`);
|
||||
addLog(job, `Done! ${totalImages} files saved to ${folderName}/`);
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
@@ -118,15 +174,24 @@ async function runCoomerScrape(job) {
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}`);
|
||||
|
||||
try {
|
||||
const { base, service, userId } = parseUserUrl(url);
|
||||
addLog(job, `Site: ${base}, Service: ${service}, User: ${userId}`);
|
||||
const parsed = parseUserUrl(url);
|
||||
let files;
|
||||
|
||||
// Phase 1: Collect files
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
const files = await fetchAllPosts(base, service, userId, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
if (parsed.mode === 'search') {
|
||||
addLog(job, `Site: ${parsed.base}, Search: "${parsed.query}"`);
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
files = await fetchSearchPosts(parsed.base, parsed.query, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
} else {
|
||||
addLog(job, `Site: ${parsed.base}, Service: ${parsed.service}, User: ${parsed.userId}`);
|
||||
addLog(job, `Fetching up to ${pages} pages...`);
|
||||
files = await fetchAllPosts(parsed.base, parsed.service, parsed.userId, pages,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
}
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
@@ -174,12 +239,170 @@ async function runMediaLinkScrape(job) {
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
|
||||
|
||||
try {
|
||||
const { base, userId } = parseMediaUrl(url);
|
||||
addLog(job, `Site: ${base}, User ID: ${userId}`);
|
||||
const { base, userId, mode } = parseMediaUrl(url);
|
||||
addLog(job, `Site: ${base}, ${mode === 'html' ? 'Slug' : 'User ID'}: ${userId} (${mode} mode)`);
|
||||
|
||||
// Phase 1: Collect all media via JSON API
|
||||
// Phase 1: Collect all media
|
||||
let items;
|
||||
if (mode === 'html') {
|
||||
addLog(job, `Fetching up to ${pages} pages via HTML scraping...`);
|
||||
items = await fetchAllMediaFromHtml(base, userId, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
} else {
|
||||
addLog(job, `Fetching up to ${pages} pages from API...`);
|
||||
items = await fetchAllMedia(base, userId, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
}
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
addLog(job, 'No media found');
|
||||
return;
|
||||
}
|
||||
|
||||
job.progress.total = items.length;
|
||||
addLog(job, `Found ${items.length} media items. Downloading...`);
|
||||
|
||||
// Phase 2: Download all media files
|
||||
const result = await downloadMedia(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors = errors;
|
||||
job.progress.total = total;
|
||||
},
|
||||
() => job.cancelled,
|
||||
base + '/'
|
||||
);
|
||||
|
||||
addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Mega Scrape ---
|
||||
|
||||
async function runMegaScrape(job) {
|
||||
const { url, workers, folderName } = job.config;
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
addLog(job, `Starting mega.nz scrape: ${url}`);
|
||||
addLog(job, `Workers: ${workers}`);
|
||||
|
||||
try {
|
||||
parseMegaUrl(url);
|
||||
|
||||
// Phase 1: List all files
|
||||
const { folderName: megaName, items } = await listAllFiles(url,
|
||||
(msg) => addLog(job, msg)
|
||||
);
|
||||
|
||||
if (job.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
return;
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
addLog(job, 'No files found in folder');
|
||||
return;
|
||||
}
|
||||
|
||||
job.progress.total = items.length;
|
||||
const totalSizeMb = (items.reduce((s, i) => s + i.size, 0) / (1024 * 1024)).toFixed(0);
|
||||
addLog(job, `Found ${items.length} files (${totalSizeMb} MB). Downloading...`);
|
||||
|
||||
// Phase 2: Download
|
||||
const result = await downloadMegaFiles(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors = errors;
|
||||
job.progress.total = total;
|
||||
},
|
||||
() => job.cancelled,
|
||||
(status) => {
|
||||
job.paused = status.paused;
|
||||
job.resumeAt = status.resumeAt;
|
||||
}
|
||||
);
|
||||
|
||||
addLog(job, `Done! ${result.completed} downloaded, ${result.errors} failed, ${result.skipped} skipped`);
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- yt-dlp Scrape ---
|
||||
|
||||
async function runYtdlpScrape(job) {
|
||||
const config = job.config;
|
||||
addLog(job, `Starting yt-dlp download: ${config.url}`);
|
||||
addLog(job, `Quality: ${config.quality || 'best'}, Playlist: ${config.playlist ? 'yes' : 'no'}`);
|
||||
|
||||
try {
|
||||
const result = await runYtdlp(
|
||||
config,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors) => {
|
||||
job.progress.completed = completed;
|
||||
job.progress.errors += errors;
|
||||
if (completed > job.progress.total) job.progress.total = completed;
|
||||
},
|
||||
() => job.cancelled
|
||||
);
|
||||
|
||||
if (result.cancelled) {
|
||||
addLog(job, 'Cancelled by user');
|
||||
} else {
|
||||
addLog(job, `Done! ${result.files} file${result.files !== 1 ? 's' : ''} downloaded`);
|
||||
}
|
||||
} catch (err) {
|
||||
addLog(job, `Error: ${err.message}`);
|
||||
job.progress.errors++;
|
||||
} finally {
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
pruneCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
// --- LeakGallery Scrape ---
|
||||
|
||||
async function runLeakGalleryScrape(job) {
|
||||
const { url, pages, workers, delay, folderName } = job.config;
|
||||
const outputDir = join(MEDIA_PATH, folderName);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
addLog(job, `Starting leakgallery scrape: ${url}`);
|
||||
addLog(job, `Pages: ${pages}, Workers: ${workers}, Delay: ${delay}ms`);
|
||||
|
||||
try {
|
||||
const { username } = parseLeakGalleryUrl(url);
|
||||
addLog(job, `Username: ${username}`);
|
||||
|
||||
// Phase 1: Collect all media
|
||||
addLog(job, `Fetching up to ${pages} pages from API...`);
|
||||
const items = await fetchAllMedia(base, userId, pages, delay,
|
||||
const items = await fetchLeakGalleryMedia(username, pages, delay,
|
||||
(msg) => addLog(job, msg),
|
||||
() => job.cancelled
|
||||
);
|
||||
@@ -198,7 +421,7 @@ async function runMediaLinkScrape(job) {
|
||||
addLog(job, `Found ${items.length} media items. Downloading...`);
|
||||
|
||||
// Phase 2: Download all media files
|
||||
const result = await downloadMedia(items, outputDir, workers,
|
||||
const result = await downloadLeakGalleryMedia(items, outputDir, workers,
|
||||
(msg) => addLog(job, msg),
|
||||
(completed, errors, total) => {
|
||||
job.progress.completed = completed;
|
||||
@@ -222,7 +445,7 @@ async function runMediaLinkScrape(job) {
|
||||
// --- Endpoints ---
|
||||
|
||||
router.post('/api/scrape/forum', (req, res) => {
|
||||
const { url, folderName, startPage, endPage, delay } = req.body;
|
||||
const { url, folderName, startPage, endPage, delay, cookies, siteId, lastPageOnly } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
@@ -232,6 +455,9 @@ router.post('/api/scrape/forum', (req, res) => {
|
||||
startPage: parseInt(startPage) || 1,
|
||||
endPage: parseInt(endPage) || 10,
|
||||
delay: parseFloat(delay) || 1.0,
|
||||
cookies: cookies || '',
|
||||
siteId: siteId ? parseInt(siteId, 10) : null,
|
||||
lastPageOnly: !!lastPageOnly,
|
||||
};
|
||||
|
||||
const job = createJob('forum', config);
|
||||
@@ -289,6 +515,105 @@ router.post('/api/scrape/medialink', (req, res) => {
|
||||
res.json({ jobId: job.id, message: 'MediaLink scrape started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/mega', (req, res) => {
|
||||
const { url, folderName, workers } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
try {
|
||||
parseMegaUrl(url);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: err.message });
|
||||
}
|
||||
|
||||
const config = {
|
||||
url,
|
||||
folderName,
|
||||
workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
|
||||
};
|
||||
|
||||
const job = createJob('mega', config);
|
||||
runMegaScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'Mega scrape started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/ytdlp', (req, res) => {
|
||||
const { url, quality, customFormat, embedMetadata, embedThumbnail, embedSubs,
|
||||
writeSubs, subLangs, restrictFilenames, outputTemplate,
|
||||
playlist, maxDownloads, concurrentFragments, rateLimit,
|
||||
sponsorBlock, cookiesFile } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
|
||||
const config = {
|
||||
url,
|
||||
quality: quality || 'best',
|
||||
customFormat: customFormat || '',
|
||||
embedMetadata: embedMetadata !== false,
|
||||
embedThumbnail: embedThumbnail !== false,
|
||||
embedSubs: embedSubs !== false,
|
||||
writeSubs: writeSubs || false,
|
||||
subLangs: subLangs || 'en',
|
||||
restrictFilenames: restrictFilenames !== false,
|
||||
outputTemplate: outputTemplate || '%(title)s.%(ext)s',
|
||||
playlist: playlist || false,
|
||||
maxDownloads: parseInt(maxDownloads) || 0,
|
||||
concurrentFragments: Math.min(Math.max(parseInt(concurrentFragments) || 4, 1), 16),
|
||||
rateLimit: rateLimit || '',
|
||||
sponsorBlock: sponsorBlock || 'off',
|
||||
cookiesFile: cookiesFile || '',
|
||||
folderName: (() => {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
const path = u.pathname.replace(/^\//, '').replace(/\/$/, '');
|
||||
return path ? `${u.hostname}/${path}`.slice(0, 60) : u.hostname;
|
||||
} catch { return url.slice(0, 60); }
|
||||
})(),
|
||||
};
|
||||
|
||||
const job = createJob('ytdlp', config);
|
||||
runYtdlpScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'yt-dlp download started' });
|
||||
});
|
||||
|
||||
router.post('/api/scrape/leakgallery', (req, res) => {
|
||||
const { url, folderName, pages, workers, delay } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
if (!folderName) return res.status(400).json({ error: 'Folder name is required' });
|
||||
|
||||
try {
|
||||
parseLeakGalleryUrl(url);
|
||||
} catch (err) {
|
||||
return res.status(400).json({ error: err.message });
|
||||
}
|
||||
|
||||
const config = {
|
||||
url,
|
||||
folderName,
|
||||
pages: parseInt(pages) || 100,
|
||||
workers: Math.min(Math.max(parseInt(workers) || 3, 1), 10),
|
||||
delay: parseInt(delay) || 300,
|
||||
};
|
||||
|
||||
const job = createJob('leakgallery', config);
|
||||
runLeakGalleryScrape(job).catch(err => {
|
||||
addLog(job, `Fatal error: ${err.message}`);
|
||||
job.running = false;
|
||||
job.completedAt = new Date().toISOString();
|
||||
});
|
||||
|
||||
res.json({ jobId: job.id, message: 'LeakGallery scrape started' });
|
||||
});
|
||||
|
||||
router.get('/api/scrape/jobs', (_req, res) => {
|
||||
const jobs = [...jobsMap.values()].map(jobToJson);
|
||||
jobs.sort((a, b) => new Date(b.startedAt) - new Date(a.startedAt));
|
||||
@@ -310,13 +635,95 @@ router.post('/api/scrape/jobs/:jobId/cancel', (req, res) => {
|
||||
res.json({ message: 'Cancel requested' });
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/jobs/:jobId', (req, res) => {
|
||||
const job = jobsMap.get(req.params.jobId);
|
||||
if (!job) return res.status(404).json({ error: 'Job not found' });
|
||||
job.cancelled = true;
|
||||
job.running = false;
|
||||
jobsMap.delete(req.params.jobId);
|
||||
res.json({ message: 'Job removed' });
|
||||
});
|
||||
|
||||
// Auto-detect max page for forum URLs
|
||||
router.post('/api/scrape/forum/detect-pages', async (req, res) => {
|
||||
const { url } = req.body;
|
||||
const { url, cookies } = req.body;
|
||||
if (!url) return res.status(400).json({ error: 'URL is required' });
|
||||
const logs = [];
|
||||
const maxPage = await detectMaxPage(url, (msg) => logs.push(msg));
|
||||
const maxPage = await detectMaxPage(url, (msg) => logs.push(msg), cookies);
|
||||
res.json({ maxPage, logs });
|
||||
});
|
||||
|
||||
// --- Forum Sites CRUD ---
|
||||
|
||||
router.get('/api/scrape/forum-sites', (_req, res) => {
|
||||
res.json(getForumSites());
|
||||
});
|
||||
|
||||
router.post('/api/scrape/forum-sites', (req, res) => {
|
||||
const { name, baseUrl, cookies, username, password } = req.body;
|
||||
if (!name) return res.status(400).json({ error: 'Name is required' });
|
||||
const id = createForumSite(name, baseUrl, cookies, username, password);
|
||||
res.json(getForumSiteById(id));
|
||||
});
|
||||
|
||||
router.put('/api/scrape/forum-sites/:id', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const site = getForumSiteById(id);
|
||||
if (!site) return res.status(404).json({ error: 'Forum site not found' });
|
||||
const { name, baseUrl, cookies, username, password } = req.body;
|
||||
const fields = {};
|
||||
if (name !== undefined) fields.name = name;
|
||||
if (baseUrl !== undefined) fields.base_url = baseUrl;
|
||||
if (cookies !== undefined) fields.cookies = cookies;
|
||||
if (username !== undefined) fields.username = username;
|
||||
if (password !== undefined) fields.password = password;
|
||||
updateForumSite(id, fields);
|
||||
res.json(getForumSiteById(id));
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/forum-sites/:id', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
deleteForumSite(id);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
// --- Auto-scrape CRUD ---
|
||||
|
||||
router.get('/api/scrape/auto', (_req, res) => {
|
||||
res.json(getAutoScrapeJobs());
|
||||
});
|
||||
|
||||
router.post('/api/scrape/auto', (req, res) => {
|
||||
const { type, url, folderName, config } = req.body;
|
||||
if (!type || !url || !folderName || !config) {
|
||||
return res.status(400).json({ error: 'type, url, folderName, and config are required' });
|
||||
}
|
||||
addAutoScrapeJob(type, url, folderName, config);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
router.delete('/api/scrape/auto/:id', (req, res) => {
|
||||
removeAutoScrapeJob(parseInt(req.params.id));
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
export function getActiveScrapeCount() {
|
||||
let count = 0;
|
||||
for (const job of jobsMap.values()) {
|
||||
if (job.running) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getActiveScrapesList() {
|
||||
const list = [];
|
||||
for (const job of jobsMap.values()) {
|
||||
if (job.running) {
|
||||
list.push({ type: job.type, folderName: job.folderName, progress: job.progress });
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
export { runForumScrape, runCoomerScrape, runMediaLinkScrape, runMegaScrape, runYtdlpScrape, runLeakGalleryScrape, createJob };
|
||||
export default router;
|
||||
|
||||
@@ -7,9 +7,16 @@ const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (
|
||||
export function parseUserUrl(url) {
|
||||
const parsed = new URL(url);
|
||||
const base = `${parsed.protocol}//${parsed.hostname}`;
|
||||
|
||||
// Search URL: /posts?q=query
|
||||
if (parsed.pathname === '/posts' && parsed.searchParams.get('q')) {
|
||||
return { base, mode: 'search', query: parsed.searchParams.get('q') };
|
||||
}
|
||||
|
||||
// User URL: /SERVICE/user/USER_ID
|
||||
const m = parsed.pathname.match(/^\/([^/]+)\/user\/([^/?#]+)/);
|
||||
if (!m) throw new Error(`Can't parse URL. Expected: https://coomer.su/SERVICE/user/USER_ID`);
|
||||
return { base, service: m[1], userId: m[2] };
|
||||
if (!m) throw new Error(`Can't parse URL. Expected: https://coomer.su/SERVICE/user/USER_ID or https://coomer.su/posts?q=QUERY`);
|
||||
return { base, mode: 'user', service: m[1], userId: m[2] };
|
||||
}
|
||||
|
||||
async function fetchApi(apiUrl, logFn, retries = 3) {
|
||||
@@ -150,6 +157,45 @@ export async function fetchAllPosts(base, service, userId, maxPages, logFn, chec
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
export async function fetchSearchPosts(base, query, maxPages, logFn, checkCancelled) {
|
||||
const allFiles = [];
|
||||
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
if (checkCancelled()) break;
|
||||
|
||||
const offset = page * 50;
|
||||
const apiUrl = `${base}/api/v1/posts?q=${encodeURIComponent(query)}&o=${offset}`;
|
||||
|
||||
let data;
|
||||
try {
|
||||
data = await fetchApi(apiUrl, logFn);
|
||||
} catch (err) {
|
||||
logFn(`API failed: ${err.message}`);
|
||||
break;
|
||||
}
|
||||
|
||||
// Search API returns { count, posts: [...] } not a plain array
|
||||
const posts = data?.posts || data;
|
||||
if (!posts || !Array.isArray(posts) || posts.length === 0) break;
|
||||
|
||||
const parsed = new URL(base);
|
||||
const cdnHost = `n1.${parsed.hostname}`;
|
||||
const cdnBase = `${parsed.protocol}//${cdnHost}/data`;
|
||||
|
||||
const files = collectFiles(posts, cdnBase);
|
||||
allFiles.push(...files);
|
||||
|
||||
if (page === 0 && data?.count) {
|
||||
logFn(`Search found ${data.count} total results`);
|
||||
}
|
||||
logFn(`Page ${page + 1}: ${posts.length} posts (${allFiles.length} files total)`);
|
||||
|
||||
if (posts.length < 50) break;
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
export async function downloadFiles(files, outputDir, concurrency, logFn, progressFn, checkCancelled) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
|
||||
+189
-44
@@ -1,13 +1,43 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import { createWriteStream, existsSync, mkdirSync, statSync } from 'fs';
|
||||
import { createWriteStream, existsSync, mkdirSync, statSync, writeFileSync } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
const SERVER_IP = '47.185.183.191';
|
||||
|
||||
export class CookieExpiredError extends Error {
|
||||
constructor(statusCode) {
|
||||
super(`Cookie expired or invalid (HTTP ${statusCode})`);
|
||||
this.name = 'CookieExpiredError';
|
||||
this.statusCode = statusCode;
|
||||
}
|
||||
}
|
||||
|
||||
// Replace DDoS-Guard __ddg9_ cookie IP with server's IP so cookies work from any browser
|
||||
function fixCookieIp(cookies) {
|
||||
if (!cookies) return cookies;
|
||||
return cookies.replace(/__ddg9_=[^;]+/, `__ddg9_=${SERVER_IP}`);
|
||||
}
|
||||
|
||||
const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);
|
||||
const SKIP_PATTERNS = ['avatar', 'smilie', 'emoji', 'icon', 'logo', 'button', 'sprite', 'badge', 'rank', 'star'];
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
|
||||
const SKIP_PATTERNS = ['avatar', 'smilie', 'emoji', 'icon', 'logo', 'button', 'sprite', 'badge', 'rank', 'star', 'dc_thumbnails'];
|
||||
|
||||
// External hosts that gallery-dl can resolve
|
||||
const GALLERY_DL_HOSTS = [
|
||||
/saint\d*\.\w+/i,
|
||||
/cyberdrop\.\w+/i,
|
||||
/bunkr+\.\w+/i,
|
||||
/pixeldrain\.com/i,
|
||||
/gofile\.io/i,
|
||||
/turbo\.\w+/i,
|
||||
];
|
||||
|
||||
function isImageUrl(url) {
|
||||
try {
|
||||
@@ -16,26 +46,44 @@ function isImageUrl(url) {
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
function isVideoUrl(url) {
|
||||
try {
|
||||
const path = new URL(url).pathname.toLowerCase();
|
||||
return [...VIDEO_EXTS].some(ext => path.endsWith(ext));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
function isMediaUrl(url) {
|
||||
return isImageUrl(url) || isVideoUrl(url);
|
||||
}
|
||||
|
||||
function isExternalHost(url) {
|
||||
try {
|
||||
const hostname = new URL(url).hostname.toLowerCase();
|
||||
return GALLERY_DL_HOSTS.some(p => p.test(hostname));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
export function getPageUrl(baseUrl, pageNum) {
|
||||
const url = baseUrl.replace(/page-\d+/, `page-${pageNum}`);
|
||||
return url.split('#')[0];
|
||||
}
|
||||
|
||||
export async function detectMaxPage(baseUrl, logFn) {
|
||||
export async function detectMaxPage(baseUrl, logFn, cookies) {
|
||||
try {
|
||||
const resp = await fetch(baseUrl, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(15000) });
|
||||
const headers = { 'User-Agent': UA };
|
||||
if (cookies) headers['Cookie'] = fixCookieIp(cookies);
|
||||
const resp = await fetch(baseUrl, { headers, signal: AbortSignal.timeout(15000) });
|
||||
if (!resp.ok) return null;
|
||||
const html = await resp.text();
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
let maxPage = 1;
|
||||
// XenForo-style
|
||||
$('a.pageNav-page, .pageNav a[href*="page-"], .pagination a[href*="page-"]').each((_, el) => {
|
||||
const href = $(el).attr('href') || '';
|
||||
const m = href.match(/page-(\d+)/);
|
||||
if (m) maxPage = Math.max(maxPage, parseInt(m[1], 10));
|
||||
});
|
||||
// Generic pagination text
|
||||
$('a').each((_, el) => {
|
||||
const text = $(el).text().trim();
|
||||
if (/^\d+$/.test(text)) {
|
||||
@@ -58,6 +106,7 @@ export async function detectMaxPage(baseUrl, logFn) {
|
||||
function tryFullSizeUrl(thumbUrl) {
|
||||
const candidates = [];
|
||||
if (thumbUrl.includes('.th.')) candidates.push(thumbUrl.replace('.th.', '.'));
|
||||
if (thumbUrl.includes('.md.')) candidates.push(thumbUrl.replace('.md.', '.'));
|
||||
if (/_thumb\./i.test(thumbUrl)) candidates.push(thumbUrl.replace(/_thumb\./i, '.'));
|
||||
if (thumbUrl.includes('/thumbs/')) {
|
||||
candidates.push(thumbUrl.replace('/thumbs/', '/images/'));
|
||||
@@ -74,7 +123,7 @@ function tryFullSizeUrl(thumbUrl) {
|
||||
return candidates;
|
||||
}
|
||||
|
||||
async function downloadImage(url, outputDir, downloadedSet, logFn) {
|
||||
async function downloadImage(url, outputDir, downloadedSet, logFn, cookies) {
|
||||
if (downloadedSet.has(url)) return false;
|
||||
if (!isImageUrl(url)) return false;
|
||||
const lower = url.toLowerCase();
|
||||
@@ -83,47 +132,34 @@ async function downloadImage(url, outputDir, downloadedSet, logFn) {
|
||||
downloadedSet.add(url);
|
||||
|
||||
let filename;
|
||||
try {
|
||||
filename = basename(new URL(url).pathname);
|
||||
} catch { return false; }
|
||||
try { filename = basename(new URL(url).pathname); } catch { return false; }
|
||||
if (!filename) return false;
|
||||
filename = filename.replace('.th.', '.').replace('.md.', '.');
|
||||
|
||||
filename = filename.replace('.th.', '.');
|
||||
|
||||
let filepath = join(outputDir, filename);
|
||||
const filepath = join(outputDir, filename);
|
||||
if (existsSync(filepath)) {
|
||||
const ext = extname(filename);
|
||||
const name = filename.slice(0, -ext.length);
|
||||
let i = 1;
|
||||
while (existsSync(filepath)) {
|
||||
filepath = join(outputDir, `${name}_${i}${ext}`);
|
||||
i++;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
const dlHeaders = { 'User-Agent': UA };
|
||||
if (cookies) dlHeaders['Cookie'] = fixCookieIp(cookies);
|
||||
const resp = await fetch(url, { headers: dlHeaders, signal: AbortSignal.timeout(30000) });
|
||||
if (!resp.ok) {
|
||||
logFn(`FAILED (${resp.status}): ${url}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Read full body to check size
|
||||
const buf = Buffer.from(await resp.arrayBuffer());
|
||||
if (buf.length < 1000) {
|
||||
downloadedSet.delete(url);
|
||||
return false;
|
||||
}
|
||||
|
||||
const { writeFileSync } = await import('fs');
|
||||
writeFileSync(filepath, buf);
|
||||
|
||||
const savedName = basename(filepath);
|
||||
const folderName = basename(outputDir);
|
||||
try { upsertMediaFile(folderName, savedName, 'image', buf.length, Date.now(), null); } catch { /* ignore */ }
|
||||
try { upsertMediaFile(folderName, savedName, 'image', buf.length, Date.now(), null); } catch {}
|
||||
|
||||
const sizeKb = (buf.length / 1024).toFixed(1);
|
||||
logFn(`Downloaded: ${savedName} (${sizeKb} KB)`);
|
||||
@@ -134,28 +170,101 @@ async function downloadImage(url, outputDir, downloadedSet, logFn) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn) {
|
||||
// Use gallery-dl to download from external hosts (bunkr, saint, cyberdrop, etc.)
|
||||
async function downloadFromExternalHost(url, outputDir, downloadedSet, logFn) {
|
||||
if (downloadedSet.has(url)) return 0;
|
||||
downloadedSet.add(url);
|
||||
|
||||
logFn(`Resolving via gallery-dl: ${url}`);
|
||||
|
||||
try {
|
||||
const args = [
|
||||
'-d', outputDir,
|
||||
'--filename', '{filename}.{extension}',
|
||||
'--no-mtime',
|
||||
'-o', 'directory=[]',
|
||||
url,
|
||||
];
|
||||
|
||||
const { stdout, stderr } = await execFileAsync('gallery-dl', args, {
|
||||
timeout: 300000, // 5 min per external link
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
|
||||
let count = 0;
|
||||
const lines = (stdout + '\n' + stderr).split('\n').filter(Boolean);
|
||||
for (const line of lines) {
|
||||
// gallery-dl outputs file paths for downloaded files
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith(outputDir) || trimmed.startsWith('/')) {
|
||||
const filePath = trimmed.replace(/^# /, '');
|
||||
if (existsSync(filePath)) {
|
||||
const stat = statSync(filePath);
|
||||
const savedName = basename(filePath);
|
||||
const folderName = basename(outputDir);
|
||||
const ext = extname(savedName).toLowerCase();
|
||||
const type = VIDEO_EXTS.has(ext) ? 'video' : 'image';
|
||||
const sizeStr = type === 'video'
|
||||
? `${(stat.size / (1024 * 1024)).toFixed(1)} MB`
|
||||
: `${(stat.size / 1024).toFixed(1)} KB`;
|
||||
|
||||
try { upsertMediaFile(folderName, savedName, type, stat.size, Date.now(), null); } catch {}
|
||||
logFn(`Downloaded: ${savedName} (${sizeStr}) [${type}]`);
|
||||
count++;
|
||||
}
|
||||
} else if (trimmed.includes('Downloading') || trimmed.includes('Skipping')) {
|
||||
logFn(` ${trimmed}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (count === 0) {
|
||||
// gallery-dl doesn't always output paths clearly, check stderr for errors
|
||||
const errLines = stderr ? stderr.split('\n').filter(l => l.trim()) : [];
|
||||
for (const line of errLines) {
|
||||
if (line.includes('ERROR') || line.includes('error')) {
|
||||
logFn(` gallery-dl: ${line.trim()}`);
|
||||
}
|
||||
}
|
||||
logFn(` gallery-dl finished but no files detected from output`);
|
||||
}
|
||||
|
||||
return count;
|
||||
} catch (err) {
|
||||
if (err.stderr) {
|
||||
const errMsg = err.stderr.split('\n').find(l => l.includes('ERROR') || l.includes('error')) || err.stderr.slice(0, 200);
|
||||
logFn(`gallery-dl error: ${errMsg.trim()}`);
|
||||
} else {
|
||||
logFn(`gallery-dl error: ${err.message}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn, cookies) {
|
||||
logFn(`Fetching page: ${pageUrl}`);
|
||||
|
||||
let html;
|
||||
try {
|
||||
const resp = await fetch(pageUrl, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
const headers = { 'User-Agent': UA };
|
||||
if (cookies) headers['Cookie'] = fixCookieIp(cookies);
|
||||
const resp = await fetch(pageUrl, { headers, signal: AbortSignal.timeout(15000) });
|
||||
if (!resp.ok) {
|
||||
// SimpCity returns 404 for expired sessions, 403 for blocked
|
||||
if (cookies && (resp.status === 404 || resp.status === 403)) {
|
||||
throw new CookieExpiredError(resp.status);
|
||||
}
|
||||
logFn(`Failed to fetch page (${resp.status})`);
|
||||
return 0;
|
||||
}
|
||||
html = await resp.text();
|
||||
} catch (err) {
|
||||
if (err instanceof CookieExpiredError) throw err;
|
||||
logFn(`Failed to fetch page: ${err.message}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// Try known content selectors, fall back to whole page
|
||||
const selectors = '.message-body, .post-body, .post_body, .postcontent, .messageContent, .bbWrapper, article, .entry-content, .post_message, .post-content, #posts, .threadBody';
|
||||
let contentAreas = $(selectors).toArray();
|
||||
if (contentAreas.length === 0) {
|
||||
@@ -163,6 +272,7 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
|
||||
}
|
||||
|
||||
const imageUrls = [];
|
||||
const externalUrls = new Set();
|
||||
|
||||
for (const area of contentAreas) {
|
||||
const $area = $(area);
|
||||
@@ -176,7 +286,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
|
||||
let absSrc;
|
||||
try { absSrc = new URL(src, pageUrl).href; } catch { return; }
|
||||
|
||||
// Check parent <a> for direct image link
|
||||
const $parentA = $img.closest('a');
|
||||
if ($parentA.length && $parentA.attr('href')) {
|
||||
try {
|
||||
@@ -188,7 +297,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Try to derive full-size from thumbnail URL
|
||||
const fullCandidates = tryFullSizeUrl(absSrc);
|
||||
if (fullCandidates.length > 0) {
|
||||
imageUrls.push(...fullCandidates);
|
||||
@@ -196,7 +304,6 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
|
||||
imageUrls.push(absSrc);
|
||||
}
|
||||
|
||||
// Also check data attributes
|
||||
for (const attr of ['data-src', 'data-url', 'data-orig', 'data-original', 'data-full-url', 'data-zoom-src']) {
|
||||
const val = $img.attr(attr);
|
||||
if (val && val !== src) {
|
||||
@@ -205,26 +312,64 @@ export async function scrapeForumPage(pageUrl, outputDir, downloadedSet, logFn)
|
||||
}
|
||||
});
|
||||
|
||||
// Pass 2: <a href> pointing directly to images (no child <img>)
|
||||
// Pass 2: <a href> links — images + external hosts
|
||||
$area.find('a[href]').each((_, el) => {
|
||||
const $a = $(el);
|
||||
if ($a.find('img').length) return;
|
||||
let href;
|
||||
try { href = new URL($a.attr('href'), pageUrl).href; } catch { return; }
|
||||
|
||||
// Skip same-forum links
|
||||
try {
|
||||
const href = new URL($a.attr('href'), pageUrl).href;
|
||||
if (isImageUrl(href)) imageUrls.push(href);
|
||||
if (new URL(href).hostname === new URL(pageUrl).hostname) return;
|
||||
} catch {}
|
||||
|
||||
// Direct image link (without child img — those are handled in Pass 1)
|
||||
if (isImageUrl(href) && $a.find('img').length === 0) {
|
||||
imageUrls.push(href);
|
||||
return;
|
||||
}
|
||||
|
||||
// Direct video link
|
||||
if (isVideoUrl(href)) {
|
||||
externalUrls.add(href);
|
||||
return;
|
||||
}
|
||||
|
||||
// External file host (bunkr, saint, cyberdrop, etc.)
|
||||
if (isExternalHost(href)) {
|
||||
externalUrls.add(href);
|
||||
}
|
||||
});
|
||||
|
||||
// Pass 3: iframe embeds
|
||||
$area.find('iframe[src]').each((_, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (src) {
|
||||
try {
|
||||
const absUrl = new URL(src, pageUrl).href;
|
||||
if (isExternalHost(absUrl)) externalUrls.add(absUrl);
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logFn(`Found ${imageUrls.length} candidate URLs`);
|
||||
logFn(`Found ${imageUrls.length} images, ${externalUrls.size} external links`);
|
||||
|
||||
let count = 0;
|
||||
|
||||
// Download images
|
||||
for (const imgUrl of imageUrls) {
|
||||
if (await downloadImage(imgUrl, outputDir, downloadedSet, logFn)) {
|
||||
if (await downloadImage(imgUrl, outputDir, downloadedSet, logFn, cookies)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
logFn(`${count} images from this page`);
|
||||
// Download from external hosts via gallery-dl
|
||||
for (const extUrl of externalUrls) {
|
||||
const dlCount = await downloadFromExternalHost(extUrl, outputDir, downloadedSet, logFn);
|
||||
count += dlCount;
|
||||
}
|
||||
|
||||
logFn(`${count} files from this page`);
|
||||
return count;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { existsSync, writeFileSync, mkdirSync } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
|
||||
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
const API_BASE = 'https://api.leakgallery.com';
|
||||
const CDN_BASE = 'https://cdn.leakgallery.com';
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
|
||||
|
||||
export function parseLeakGalleryUrl(url) {
|
||||
const parsed = new URL(url);
|
||||
if (!parsed.hostname.includes('leakgallery.com')) {
|
||||
throw new Error('Not a leakgallery.com URL');
|
||||
}
|
||||
// URL format: https://leakgallery.com/{username}
|
||||
const m = parsed.pathname.match(/^\/([a-zA-Z0-9_.-]+)\/?$/);
|
||||
if (!m) throw new Error('Expected URL format: https://leakgallery.com/username');
|
||||
return { username: m[1] };
|
||||
}
|
||||
|
||||
async function fetchPage(username, page, logFn) {
|
||||
// Page 1: /profile/{username}?type=All&sort=MostRecent
|
||||
// Page 2+: /profile/{username}/{page}?type=All&sort=MostRecent
|
||||
const pagePath = page <= 1 ? '' : `/${page}`;
|
||||
const apiUrl = `${API_BASE}/profile/${username}${pagePath}?type=All&sort=MostRecent`;
|
||||
|
||||
try {
|
||||
const resp = await fetch(apiUrl, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Accept': 'application/json',
|
||||
'Origin': 'https://leakgallery.com',
|
||||
'Referer': 'https://leakgallery.com/',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
if (resp.status === 404) return null;
|
||||
logFn(`API error (${resp.status}): ${apiUrl}`);
|
||||
return null;
|
||||
}
|
||||
return await resp.json();
|
||||
} catch (err) {
|
||||
logFn(`API fetch error: ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchAllMedia(username, maxPages, delay, logFn, checkCancelled) {
|
||||
const allItems = [];
|
||||
const seen = new Set();
|
||||
let totalCount = 0;
|
||||
|
||||
for (let page = 1; page <= maxPages; page++) {
|
||||
if (checkCancelled()) break;
|
||||
|
||||
logFn(`Fetching page ${page}...`);
|
||||
const data = await fetchPage(username, page, logFn);
|
||||
|
||||
if (!data) {
|
||||
logFn(`Page ${page}: no data — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
// First page includes mediaCount
|
||||
if (page === 1 && data.mediaCount) {
|
||||
totalCount = data.mediaCount;
|
||||
logFn(`Profile has ${totalCount} total media items`);
|
||||
}
|
||||
|
||||
const medias = data.medias;
|
||||
if (!medias || !Array.isArray(medias) || medias.length === 0) {
|
||||
logFn(`Page ${page}: no more items — done`);
|
||||
break;
|
||||
}
|
||||
|
||||
let newCount = 0;
|
||||
for (const item of medias) {
|
||||
if (seen.has(item.id)) continue;
|
||||
seen.add(item.id);
|
||||
newCount++;
|
||||
|
||||
// file_path is relative, e.g. content4/username/watermark_hash__username__id_580px.webp
|
||||
// Full-size: remove _580px.webp suffix, use .jpg (or .mp4 for videos)
|
||||
const isVideo = !!item.is_video;
|
||||
let fullUrl;
|
||||
let filename;
|
||||
|
||||
if (isVideo) {
|
||||
// Videos: file_path is already the video file
|
||||
fullUrl = `${CDN_BASE}/${item.file_path}`;
|
||||
filename = basename(item.file_path);
|
||||
} else {
|
||||
// Images: thumbnail has _580px.webp — convert to full-size .jpg
|
||||
const filePath = item.file_path || item.thumbnail_path || '';
|
||||
const fullPath = filePath
|
||||
.replace(/_580px\.webp$/, '.jpg')
|
||||
.replace(/_300px\.webp$/, '.jpg');
|
||||
fullUrl = `${CDN_BASE}/${fullPath}`;
|
||||
filename = basename(fullPath);
|
||||
}
|
||||
|
||||
allItems.push({
|
||||
id: item.id,
|
||||
url: fullUrl,
|
||||
filename,
|
||||
type: isVideo ? 'video' : 'image',
|
||||
});
|
||||
}
|
||||
|
||||
if (newCount === 0) {
|
||||
logFn(`Page ${page}: all duplicates — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
logFn(`Page ${page}: ${medias.length} items (${newCount} new, ${allItems.length} total)`);
|
||||
|
||||
if (page < maxPages && !checkCancelled()) {
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
async function tryFetch(url) {
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Referer': 'https://leakgallery.com/',
|
||||
},
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!resp.ok) return null;
|
||||
const buf = Buffer.from(await resp.arrayBuffer());
|
||||
if (buf.length < 500) return null;
|
||||
return buf;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
let index = 0;
|
||||
|
||||
async function processNext() {
|
||||
while (index < items.length) {
|
||||
if (checkCancelled()) return;
|
||||
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
const filename = item.filename || `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
|
||||
const filepath = join(outputDir, filename);
|
||||
|
||||
if (existsSync(filepath)) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
const buf = await tryFetch(item.url);
|
||||
if (buf) {
|
||||
writeFileSync(filepath, buf);
|
||||
const folderName = basename(outputDir);
|
||||
const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
|
||||
try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
|
||||
completed++;
|
||||
logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} else {
|
||||
logFn(`FAILED: ${filename}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < Math.min(workers, items.length); i++) {
|
||||
workerPromises.push(processNext());
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
return { completed, errors, skipped, total: items.length };
|
||||
}
|
||||
+222
-59
@@ -1,6 +1,7 @@
|
||||
import { existsSync, writeFileSync, mkdirSync } from 'fs';
|
||||
import { existsSync, writeFileSync, mkdirSync, unlinkSync } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
import { load as cheerioLoad } from 'cheerio';
|
||||
import { upsertMediaFile, removeMediaFile } from '../db.js';
|
||||
|
||||
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
@@ -9,10 +10,13 @@ const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
|
||||
export function parseMediaUrl(url) {
|
||||
const parsed = new URL(url);
|
||||
const base = `${parsed.protocol}//${parsed.hostname}`;
|
||||
// Support /model/{id} or /media/{id}
|
||||
// Support /model/{id} or /media/{id} (fapello.to JSON API)
|
||||
const m = parsed.pathname.match(/\/(?:model|media)\/(\d+)/);
|
||||
if (!m) throw new Error(`Can't parse URL. Expected: https://fapello.to/model/12345`);
|
||||
return { base, userId: m[1] };
|
||||
if (m) return { base, userId: m[1], mode: 'api' };
|
||||
// Support fapello.com profile slug URLs like /josie-hamming-41/
|
||||
const slugMatch = parsed.pathname.match(/^\/([a-zA-Z0-9_-]+)\/?$/);
|
||||
if (slugMatch) return { base, userId: slugMatch[1], mode: 'html' };
|
||||
throw new Error(`Can't parse URL. Expected: https://fapello.to/model/12345 or https://fapello.com/username/`);
|
||||
}
|
||||
|
||||
// Fetch JSON from the API endpoint
|
||||
@@ -73,6 +77,7 @@ export async function fetchAllMedia(base, userId, maxPages, delay, logFn, checkC
|
||||
allItems.push({
|
||||
id: item.id,
|
||||
url: fullUrl,
|
||||
thumbUrl: item.newUrlThumb || null,
|
||||
type: isVideo ? 'video' : 'image',
|
||||
});
|
||||
}
|
||||
@@ -92,13 +97,171 @@ export async function fetchAllMedia(base, userId, maxPages, delay, logFn, checkC
|
||||
return allItems;
|
||||
}
|
||||
|
||||
// --- HTML-based scraping (fapello.com profile pages) ---
|
||||
|
||||
function parseMediaFromHtml(html, base) {
|
||||
const $ = cheerioLoad(html);
|
||||
const items = [];
|
||||
|
||||
// Find all image thumbnails in the grid
|
||||
$('img[src*="_300px."]').each((_, el) => {
|
||||
const thumbUrl = $(el).attr('src');
|
||||
if (!thumbUrl) return;
|
||||
// Convert thumbnail to full-size: remove _300px
|
||||
const fullUrl = thumbUrl.replace(/_300px\./, '.');
|
||||
const absUrl = fullUrl.startsWith('http') ? fullUrl : `${base}${fullUrl}`;
|
||||
items.push({ url: absUrl, type: 'image' });
|
||||
});
|
||||
|
||||
// Find video elements (source tags with .mp4)
|
||||
$('video source[src*=".mp4"], video[src*=".mp4"]').each((_, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (!src) return;
|
||||
const absUrl = src.startsWith('http') ? src : `${base}${src}`;
|
||||
items.push({ url: absUrl, type: 'video' });
|
||||
});
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function fetchAllMediaFromHtml(base, slug, maxPages, delay, logFn, checkCancelled) {
|
||||
const allItems = [];
|
||||
const seen = new Set();
|
||||
let totalPages = maxPages;
|
||||
|
||||
// Phase 1: Fetch initial profile page to get data-max
|
||||
logFn(`Fetching profile page: ${base}/${slug}/`);
|
||||
try {
|
||||
const resp = await fetch(`${base}/${slug}/`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
logFn(`Profile page error (${resp.status})`);
|
||||
return allItems;
|
||||
}
|
||||
const html = await resp.text();
|
||||
const $ = cheerioLoad(html);
|
||||
|
||||
// Get max pages from data-max attribute
|
||||
const dataMax = $('#showmore').attr('data-max');
|
||||
if (dataMax) {
|
||||
totalPages = Math.min(parseInt(dataMax, 10) || maxPages, maxPages);
|
||||
logFn(`Detected ${totalPages} pages`);
|
||||
}
|
||||
|
||||
// Parse initial page content
|
||||
const initialItems = parseMediaFromHtml(html, base);
|
||||
for (const item of initialItems) {
|
||||
if (!seen.has(item.url)) {
|
||||
seen.add(item.url);
|
||||
allItems.push({ ...item, id: seen.size });
|
||||
}
|
||||
}
|
||||
logFn(`Page 1: ${initialItems.length} items (${allItems.length} total)`);
|
||||
} catch (err) {
|
||||
logFn(`Error fetching profile: ${err.message}`);
|
||||
return allItems;
|
||||
}
|
||||
|
||||
// Phase 2: Paginate through AJAX pages
|
||||
for (let page = 2; page <= totalPages; page++) {
|
||||
if (checkCancelled()) break;
|
||||
|
||||
const ajaxUrl = `${base}/ajax/model/${slug}/page-${page}/`;
|
||||
try {
|
||||
const resp = await fetch(ajaxUrl, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'Referer': `${base}/${slug}/`,
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
if (resp.status === 404) {
|
||||
logFn(`Page ${page}: 404 — done`);
|
||||
break;
|
||||
}
|
||||
logFn(`Page ${page}: error (${resp.status})`);
|
||||
continue;
|
||||
}
|
||||
const html = await resp.text();
|
||||
if (!html || html.trim().length === 0) {
|
||||
logFn(`Page ${page}: empty — done`);
|
||||
break;
|
||||
}
|
||||
|
||||
const pageItems = parseMediaFromHtml(html, base);
|
||||
let newCount = 0;
|
||||
for (const item of pageItems) {
|
||||
if (!seen.has(item.url)) {
|
||||
seen.add(item.url);
|
||||
allItems.push({ ...item, id: seen.size });
|
||||
newCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (newCount === 0) {
|
||||
logFn(`Page ${page}: all duplicates — stopping`);
|
||||
break;
|
||||
}
|
||||
|
||||
logFn(`Page ${page}: ${pageItems.length} items (${newCount} new, ${allItems.length} total)`);
|
||||
} catch (err) {
|
||||
logFn(`Page ${page}: error — ${err.message}`);
|
||||
}
|
||||
|
||||
if (page < totalPages && !checkCancelled()) {
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
// Helper: derive filename from URL, with fallback
|
||||
function filenameFromUrl(url, item) {
|
||||
try {
|
||||
const name = basename(new URL(url).pathname);
|
||||
if (name && name !== '/') return name;
|
||||
} catch {}
|
||||
return `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
|
||||
}
|
||||
|
||||
// Helper: add _md suffix before extension
|
||||
function mdFilename(filename) {
|
||||
const ext = extname(filename);
|
||||
return filename.slice(0, -ext.length) + '_md' + ext;
|
||||
}
|
||||
|
||||
// Helper: try fetching a URL, return buffer or null
|
||||
async function tryFetch(url, referer) {
|
||||
if (!url) return null;
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: { 'User-Agent': UA, 'Referer': referer || 'https://fapello.to/' },
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!resp.ok) return null;
|
||||
const buf = Buffer.from(await resp.arrayBuffer());
|
||||
if (buf.length < 500) return null;
|
||||
return buf;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Download all collected media items with concurrency
|
||||
export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled) {
|
||||
// Fallback: if full-res URL fails, download medium (thumbUrl) with _md suffix.
|
||||
// Upgrade: if _md file exists, try full-res again; replace _md on success.
|
||||
export async function downloadMedia(items, outputDir, workers, logFn, progressFn, checkCancelled, referer) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
let upgraded = 0;
|
||||
let index = 0;
|
||||
|
||||
async function processNext() {
|
||||
@@ -108,72 +271,71 @@ export async function downloadMedia(items, outputDir, workers, logFn, progressFn
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
|
||||
let filename;
|
||||
try {
|
||||
filename = basename(new URL(item.url).pathname);
|
||||
if (!filename || filename === '/') {
|
||||
filename = `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
|
||||
}
|
||||
} catch {
|
||||
filename = `${item.id}.${item.type === 'video' ? 'mp4' : 'jpg'}`;
|
||||
}
|
||||
const filename = filenameFromUrl(item.url, item);
|
||||
const filepath = join(outputDir, filename);
|
||||
const mdName = mdFilename(filename);
|
||||
const mdPath = join(outputDir, mdName);
|
||||
|
||||
let filepath = join(outputDir, filename);
|
||||
// Full-res already exists — skip
|
||||
if (existsSync(filepath)) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(item.url, {
|
||||
headers: {
|
||||
'User-Agent': UA,
|
||||
'Referer': 'https://fapello.to/',
|
||||
},
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
logFn(`FAILED (${resp.status}): ${filename}`);
|
||||
errors++;
|
||||
// Medium version exists — try to upgrade to full-res
|
||||
if (existsSync(mdPath)) {
|
||||
const buf = await tryFetch(item.url, referer);
|
||||
if (buf) {
|
||||
writeFileSync(filepath, buf);
|
||||
try { unlinkSync(mdPath); } catch {}
|
||||
const folderName = basename(outputDir);
|
||||
const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
|
||||
try { removeMediaFile(folderName, mdName); } catch {}
|
||||
try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
|
||||
upgraded++;
|
||||
completed++;
|
||||
logFn(`[${completed}/${items.length}] ${filename} (upgraded from _md, ${(buf.length / 1024).toFixed(1)} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
const buf = Buffer.from(await resp.arrayBuffer());
|
||||
if (buf.length < 500) {
|
||||
} else {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Neither exists — try full-res, then fallback to medium
|
||||
const buf = await tryFetch(item.url, referer);
|
||||
if (buf) {
|
||||
writeFileSync(filepath, buf);
|
||||
const folderName = basename(outputDir);
|
||||
const fileType = VIDEO_EXTS.has(extname(filename).toLowerCase()) ? 'video' : 'image';
|
||||
try { upsertMediaFile(folderName, filename, fileType, buf.length, Date.now(), null); } catch {}
|
||||
completed++;
|
||||
logFn(`[${completed}/${items.length}] ${filename} (${(buf.length / 1024).toFixed(1)} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Full-res failed — try medium (thumbUrl)
|
||||
if (item.thumbUrl) {
|
||||
const mdBuf = await tryFetch(item.thumbUrl, referer);
|
||||
if (mdBuf) {
|
||||
writeFileSync(mdPath, mdBuf);
|
||||
const folderName = basename(outputDir);
|
||||
const fileType = VIDEO_EXTS.has(extname(mdName).toLowerCase()) ? 'video' : 'image';
|
||||
try { upsertMediaFile(folderName, mdName, fileType, mdBuf.length, Date.now(), null); } catch {}
|
||||
completed++;
|
||||
logFn(`[${completed}/${items.length}] ${mdName} (medium fallback, ${(mdBuf.length / 1024).toFixed(1)} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle filename collision
|
||||
if (existsSync(filepath)) {
|
||||
const ext = extname(filename);
|
||||
const name = filename.slice(0, -ext.length);
|
||||
let i = 1;
|
||||
while (existsSync(filepath)) {
|
||||
filepath = join(outputDir, `${name}_${i}${ext}`);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(filepath, buf);
|
||||
const savedName = basename(filepath);
|
||||
const folderName = basename(outputDir);
|
||||
const fileExt = extname(savedName).toLowerCase();
|
||||
const fileType = VIDEO_EXTS.has(fileExt) ? 'video' : 'image';
|
||||
try { upsertMediaFile(folderName, savedName, fileType, buf.length, Date.now(), null); } catch {}
|
||||
|
||||
completed++;
|
||||
const sizeKb = (buf.length / 1024).toFixed(1);
|
||||
logFn(`[${completed}/${items.length}] ${savedName} (${sizeKb} KB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} catch (err) {
|
||||
logFn(`FAILED: ${filename} - ${err.message}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
|
||||
// Both failed
|
||||
logFn(`FAILED: ${filename} — full-res and medium both unavailable`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,5 +345,6 @@ export async function downloadMedia(items, outputDir, workers, logFn, progressFn
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
if (upgraded > 0) logFn(`Upgraded ${upgraded} files from medium to full resolution`);
|
||||
return { completed, errors, skipped, total: items.length };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { File } from 'megajs';
|
||||
import { existsSync, mkdirSync, statSync, unlinkSync } from 'fs';
|
||||
import { createWriteStream } from 'fs';
|
||||
import { basename, join, extname } from 'path';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { upsertMediaFile } from '../db.js';
|
||||
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v']);
|
||||
const IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff']);
|
||||
|
||||
export function parseMegaUrl(url) {
|
||||
// Validate it's a mega.nz folder URL
|
||||
const parsed = new URL(url);
|
||||
if (!parsed.hostname.includes('mega.nz') && !parsed.hostname.includes('mega.co.nz')) {
|
||||
throw new Error('Not a mega.nz URL');
|
||||
}
|
||||
if (!parsed.pathname.includes('/folder/')) {
|
||||
throw new Error('Expected a mega.nz folder URL (e.g. https://mega.nz/folder/ABC#key)');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
// Load shared folder and list all files recursively
|
||||
export async function listAllFiles(url, logFn) {
|
||||
logFn('Loading shared folder...');
|
||||
const folder = File.fromURL(url);
|
||||
await folder.loadAttributes();
|
||||
|
||||
const folderName = folder.name || 'mega_folder';
|
||||
logFn(`Folder: ${folderName}`);
|
||||
|
||||
// Recursively get all non-directory files
|
||||
const allFiles = folder.filter(f => !f.directory, true);
|
||||
logFn(`Found ${allFiles.length} files across all subfolders`);
|
||||
|
||||
// Build items with subfolder paths
|
||||
const items = [];
|
||||
for (const file of allFiles) {
|
||||
const ext = extname(file.name).toLowerCase();
|
||||
let type = 'other';
|
||||
if (IMAGE_EXTS.has(ext)) type = 'image';
|
||||
else if (VIDEO_EXTS.has(ext)) type = 'video';
|
||||
|
||||
// Build relative path from parent folders
|
||||
let subfolder = '';
|
||||
let parent = file.parent;
|
||||
const parts = [];
|
||||
while (parent && parent !== folder) {
|
||||
parts.unshift(parent.name);
|
||||
parent = parent.parent;
|
||||
}
|
||||
subfolder = parts.join('/');
|
||||
|
||||
items.push({
|
||||
file,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type,
|
||||
subfolder,
|
||||
});
|
||||
}
|
||||
|
||||
return { folderName, items };
|
||||
}
|
||||
|
||||
// Parse bandwidth limit wait time from error message
|
||||
function parseBandwidthWait(errMsg) {
|
||||
const m = errMsg.match(/(\d+)\s*seconds?\s*until/i);
|
||||
if (m) return parseInt(m[1], 10);
|
||||
if (/bandwidth/i.test(errMsg)) return 3600; // default 1hr if can't parse
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Download all files with concurrency + bandwidth limit auto-retry
|
||||
export async function downloadMegaFiles(items, outputDir, workers, logFn, progressFn, checkCancelled, statusFn) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let skipped = 0;
|
||||
let index = 0;
|
||||
let bandwidthPaused = false;
|
||||
|
||||
async function processNext() {
|
||||
while (index < items.length) {
|
||||
if (checkCancelled()) return;
|
||||
|
||||
// If another worker hit the bandwidth limit, wait for it to clear
|
||||
if (bandwidthPaused) return;
|
||||
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
|
||||
// All files go to root output dir (flatten subfolders)
|
||||
const filepath = join(outputDir, item.name);
|
||||
|
||||
// Skip if file exists AND is non-empty (0-byte = failed partial download)
|
||||
if (existsSync(filepath)) {
|
||||
try {
|
||||
const st = statSync(filepath);
|
||||
if (st.size > 0) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
// Remove 0-byte leftover from previous failed download
|
||||
unlinkSync(filepath);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = item.file.download();
|
||||
await pipeline(stream, createWriteStream(filepath));
|
||||
|
||||
// Verify the file was actually written
|
||||
let actualSize = item.size;
|
||||
try { actualSize = statSync(filepath).size; } catch {}
|
||||
|
||||
const folderName = basename(outputDir);
|
||||
const ext = extname(item.name).toLowerCase();
|
||||
const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
|
||||
try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
|
||||
|
||||
completed++;
|
||||
const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
|
||||
logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} catch (err) {
|
||||
// Clean up partial/empty file on any error
|
||||
try { unlinkSync(filepath); } catch {}
|
||||
|
||||
const waitSecs = parseBandwidthWait(err.message);
|
||||
if (waitSecs > 0) {
|
||||
// Bandwidth limit — put this item back and pause all workers
|
||||
index = current; // rewind so this file gets retried
|
||||
bandwidthPaused = true;
|
||||
const waitMins = Math.ceil(waitSecs / 60);
|
||||
const resumeAt = Date.now() + waitSecs * 1000;
|
||||
logFn(`Bandwidth limit reached — waiting ${waitMins} minutes for quota reset...`);
|
||||
if (statusFn) statusFn({ paused: true, resumeAt });
|
||||
await new Promise(r => setTimeout(r, waitSecs * 1000));
|
||||
if (checkCancelled()) return;
|
||||
if (statusFn) statusFn({ paused: false, resumeAt: null });
|
||||
logFn('Quota reset — resuming downloads...');
|
||||
bandwidthPaused = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
logFn(`FAILED: ${item.name} — ${err.message}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerPromises = [];
|
||||
for (let i = 0; i < Math.min(workers, items.length); i++) {
|
||||
workerPromises.push(processNext());
|
||||
}
|
||||
await Promise.all(workerPromises);
|
||||
|
||||
// If we paused for bandwidth and there are remaining files, run single-threaded to finish
|
||||
while (index < items.length && !checkCancelled()) {
|
||||
const current = index++;
|
||||
const item = items[current];
|
||||
const filepath = join(outputDir, item.name);
|
||||
|
||||
if (existsSync(filepath)) {
|
||||
try {
|
||||
const st = statSync(filepath);
|
||||
if (st.size > 0) {
|
||||
skipped++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
continue;
|
||||
}
|
||||
unlinkSync(filepath);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = item.file.download();
|
||||
await pipeline(stream, createWriteStream(filepath));
|
||||
|
||||
let actualSize = item.size;
|
||||
try { actualSize = statSync(filepath).size; } catch {}
|
||||
|
||||
const folderName = basename(outputDir);
|
||||
const ext = extname(item.name).toLowerCase();
|
||||
const fileType = VIDEO_EXTS.has(ext) ? 'video' : IMAGE_EXTS.has(ext) ? 'image' : 'other';
|
||||
try { upsertMediaFile(folderName, item.name, fileType, actualSize, Date.now(), null); } catch {}
|
||||
|
||||
completed++;
|
||||
const sizeMb = (item.size / (1024 * 1024)).toFixed(1);
|
||||
logFn(`[${completed}/${items.length}] ${item.subfolder ? item.subfolder + '/' : ''}${item.name} (${sizeMb} MB)`);
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
} catch (err) {
|
||||
try { unlinkSync(filepath); } catch {}
|
||||
|
||||
const waitSecs = parseBandwidthWait(err.message);
|
||||
if (waitSecs > 0) {
|
||||
index = current;
|
||||
const waitMins = Math.ceil(waitSecs / 60);
|
||||
const resumeAt = Date.now() + waitSecs * 1000;
|
||||
logFn(`Bandwidth limit reached — waiting ${waitMins} minutes...`);
|
||||
if (statusFn) statusFn({ paused: true, resumeAt });
|
||||
await new Promise(r => setTimeout(r, waitSecs * 1000));
|
||||
if (checkCancelled()) break;
|
||||
if (statusFn) statusFn({ paused: false, resumeAt: null });
|
||||
logFn('Quota reset — resuming...');
|
||||
continue;
|
||||
}
|
||||
logFn(`FAILED: ${item.name} — ${err.message}`);
|
||||
errors++;
|
||||
progressFn(completed + skipped, errors, items.length);
|
||||
}
|
||||
}
|
||||
|
||||
return { completed, errors, skipped, total: items.length };
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
import { spawn } from 'child_process';
|
||||
import { basename, extname, join } from 'path';
|
||||
import { existsSync, statSync, readdirSync } from 'fs';
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { insertVideo, getVideoByPath } from '../db.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
|
||||
|
||||
// Quality presets mapped to yt-dlp format strings
|
||||
const QUALITY_PRESETS = {
|
||||
best: 'bestvideo+bestaudio/best',
|
||||
'2160p': 'bestvideo[height<=2160]+bestaudio/best[height<=2160]',
|
||||
'1080p': 'bestvideo[height<=1080]+bestaudio/best[height<=1080]',
|
||||
'720p': 'bestvideo[height<=720]+bestaudio/best[height<=720]',
|
||||
'480p': 'bestvideo[height<=480]+bestaudio/best[height<=480]',
|
||||
audio: 'bestaudio/best',
|
||||
};
|
||||
|
||||
async function probeVideo(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'format=duration,bit_rate',
|
||||
'-show_entries', 'stream=codec_name,width,height,r_frame_rate,codec_type',
|
||||
'-of', 'json',
|
||||
filePath,
|
||||
], { timeout: 60000 });
|
||||
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = info.streams?.find(s => s.codec_type === 'video');
|
||||
const audioStream = info.streams?.find(s => s.codec_type === 'audio');
|
||||
const duration = parseFloat(info.format?.duration || '0');
|
||||
const bitrate = parseInt(info.format?.bit_rate || '0', 10);
|
||||
|
||||
let fps = null;
|
||||
if (videoStream?.r_frame_rate) {
|
||||
const [num, den] = videoStream.r_frame_rate.split('/');
|
||||
if (den && parseInt(den, 10) > 0) {
|
||||
fps = Math.round((parseInt(num, 10) / parseInt(den, 10)) * 100) / 100;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
duration: duration || null,
|
||||
width: videoStream?.width || null,
|
||||
height: videoStream?.height || null,
|
||||
fps,
|
||||
codec: videoStream?.codec_name || null,
|
||||
bitrate: bitrate || null,
|
||||
has_audio: audioStream ? 1 : 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function generateThumbnail(filePath) {
|
||||
const thumbDir = join(VIDEOS_PATH, '.thumbnails');
|
||||
const filename = basename(filePath);
|
||||
const thumbName = `${Date.now()}_${filename.replace(/\.[^.]+$/, '.jpg')}`;
|
||||
const thumbPath = join(thumbDir, thumbName);
|
||||
|
||||
let duration = 0;
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', filePath,
|
||||
], { timeout: 15000 });
|
||||
duration = parseFloat(stdout.trim()) || 0;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const seekTime = duration > 2 ? '1' : '0';
|
||||
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-ss', seekTime, '-i', filePath,
|
||||
'-frames:v', '1', '-vf', 'scale=480:-1', '-q:v', '4', '-y', '-update', '1',
|
||||
thumbPath,
|
||||
], { timeout: 30000 });
|
||||
|
||||
return thumbPath;
|
||||
}
|
||||
|
||||
// Register a downloaded video file into the videos DB table
|
||||
async function registerVideo(filePath, log) {
|
||||
try {
|
||||
if (getVideoByPath(filePath)) {
|
||||
log(`Already indexed: ${basename(filePath)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const stat = statSync(filePath);
|
||||
const filename = basename(filePath);
|
||||
|
||||
let probe;
|
||||
try {
|
||||
probe = await probeVideo(filePath);
|
||||
} catch (err) {
|
||||
log(`Probe failed for ${filename}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
let thumbPath = null;
|
||||
try {
|
||||
thumbPath = await generateThumbnail(filePath);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const title = basename(filename, extname(filename))
|
||||
.replace(/[_.-]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
insertVideo({
|
||||
title,
|
||||
filename,
|
||||
file_path: filePath,
|
||||
file_size: stat.size,
|
||||
...probe,
|
||||
thumbnail_path: thumbPath,
|
||||
status: 'ready',
|
||||
});
|
||||
|
||||
log(`Registered in library: ${title}`);
|
||||
} catch (err) {
|
||||
log(`Failed to register ${basename(filePath)}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Build yt-dlp arguments from config
|
||||
function buildArgs(config) {
|
||||
const { url, quality, customFormat, embedMetadata, embedThumbnail, embedSubs,
|
||||
writeSubs, subLangs, restrictFilenames, outputTemplate,
|
||||
playlist, maxDownloads, concurrentFragments, rateLimit,
|
||||
sponsorBlock, cookiesFile } = config;
|
||||
|
||||
const args = [];
|
||||
|
||||
// Format
|
||||
if (customFormat) {
|
||||
args.push('-f', customFormat);
|
||||
} else {
|
||||
args.push('-f', QUALITY_PRESETS[quality] || QUALITY_PRESETS.best);
|
||||
}
|
||||
|
||||
// Merge to mp4 when possible
|
||||
if (quality !== 'audio') {
|
||||
args.push('--merge-output-format', 'mp4');
|
||||
} else {
|
||||
args.push('-x', '--audio-format', 'mp3');
|
||||
}
|
||||
|
||||
// Embed options
|
||||
if (embedMetadata) args.push('--embed-metadata');
|
||||
if (embedThumbnail) args.push('--embed-thumbnail');
|
||||
if (embedSubs) args.push('--embed-subs');
|
||||
if (writeSubs) args.push('--write-subs');
|
||||
if (subLangs) args.push('--sub-langs', subLangs);
|
||||
|
||||
// Filename
|
||||
if (restrictFilenames) args.push('--restrict-filenames');
|
||||
args.push('-o', join(VIDEOS_PATH, outputTemplate || '%(title)s.%(ext)s'));
|
||||
|
||||
// Playlist
|
||||
if (playlist) {
|
||||
args.push('--yes-playlist');
|
||||
if (maxDownloads) args.push('--max-downloads', String(maxDownloads));
|
||||
} else {
|
||||
args.push('--no-playlist');
|
||||
}
|
||||
|
||||
// Performance
|
||||
if (concurrentFragments && concurrentFragments > 1) {
|
||||
args.push('--concurrent-fragments', String(concurrentFragments));
|
||||
}
|
||||
if (rateLimit) args.push('--rate-limit', rateLimit);
|
||||
|
||||
// SponsorBlock
|
||||
if (sponsorBlock === 'remove') args.push('--sponsorblock-remove', 'all');
|
||||
else if (sponsorBlock === 'mark') args.push('--sponsorblock-mark', 'all');
|
||||
|
||||
// Cookies
|
||||
if (cookiesFile) args.push('--cookies', cookiesFile);
|
||||
|
||||
// Progress & output
|
||||
args.push('--newline', '--no-colors', '--no-overwrites');
|
||||
// Print downloaded file paths
|
||||
args.push('--print', 'after_move:filepath');
|
||||
|
||||
args.push(url);
|
||||
return args;
|
||||
}
|
||||
|
||||
// Run yt-dlp download. Returns a promise. Progress/logs via callbacks.
|
||||
export function runYtdlp(config, log, onProgress, isCancelled) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args = buildArgs(config);
|
||||
log(`yt-dlp ${args.join(' ')}`);
|
||||
|
||||
const proc = spawn('yt-dlp', args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
const downloadedFiles = [];
|
||||
let currentFile = '';
|
||||
let fileCount = 0;
|
||||
|
||||
proc.stdout.on('data', (data) => {
|
||||
const lines = data.toString().split('\n').filter(Boolean);
|
||||
for (const line of lines) {
|
||||
// yt-dlp --print after_move:filepath outputs the final file path on its own line
|
||||
// These lines don't start with [ and are absolute paths
|
||||
if (line.startsWith('/') && existsSync(line.trim())) {
|
||||
const filePath = line.trim();
|
||||
if (!downloadedFiles.includes(filePath)) {
|
||||
downloadedFiles.push(filePath);
|
||||
fileCount++;
|
||||
onProgress(fileCount, 0);
|
||||
log(`Downloaded: ${basename(filePath)}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse progress lines: [download] 45.2% of 250.00MiB at 5.00MiB/s ETA 00:25
|
||||
const progressMatch = line.match(/\[download\]\s+([\d.]+)%\s+of\s+~?([\d.]+\w+)\s+at\s+([\d.]+\w+\/s|Unknown)\s+ETA\s+(\S+)/);
|
||||
if (progressMatch) {
|
||||
const pct = parseFloat(progressMatch[1]);
|
||||
const size = progressMatch[2];
|
||||
const speed = progressMatch[3];
|
||||
const eta = progressMatch[4];
|
||||
log(`[download] ${pct.toFixed(1)}% of ${size} at ${speed} ETA ${eta}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Destination line: [download] Destination: filename.mp4
|
||||
const destMatch = line.match(/\[download\] Destination:\s+(.+)/);
|
||||
if (destMatch) {
|
||||
currentFile = basename(destMatch[1]);
|
||||
log(`Downloading: ${currentFile}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Already downloaded
|
||||
if (line.includes('has already been downloaded')) {
|
||||
log(line.trim());
|
||||
onProgress(fileCount, 0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Log other yt-dlp output
|
||||
if (line.trim()) {
|
||||
log(line.trim());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
proc.stderr.on('data', (data) => {
|
||||
const lines = data.toString().split('\n').filter(Boolean);
|
||||
for (const line of lines) {
|
||||
if (line.includes('WARNING:')) {
|
||||
log(`Warning: ${line.replace(/WARNING:\s*/, '')}`);
|
||||
} else if (line.includes('ERROR:')) {
|
||||
log(`ERROR: ${line.replace(/ERROR:\s*/, '')}`);
|
||||
onProgress(fileCount, 1);
|
||||
} else if (line.trim()) {
|
||||
log(line.trim());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Check for cancellation
|
||||
const cancelCheck = setInterval(() => {
|
||||
if (isCancelled()) {
|
||||
proc.kill('SIGTERM');
|
||||
clearInterval(cancelCheck);
|
||||
}
|
||||
}, 500);
|
||||
|
||||
proc.on('close', async (code) => {
|
||||
clearInterval(cancelCheck);
|
||||
|
||||
// Register downloaded video files in the library
|
||||
for (const filePath of downloadedFiles) {
|
||||
const ext = extname(filePath).toLowerCase();
|
||||
if (VIDEO_EXTS.has(ext)) {
|
||||
await registerVideo(filePath, log);
|
||||
}
|
||||
}
|
||||
|
||||
if (code === 0) {
|
||||
resolve({ files: downloadedFiles.length, errors: 0 });
|
||||
} else if (isCancelled()) {
|
||||
resolve({ files: downloadedFiles.length, errors: 0, cancelled: true });
|
||||
} else {
|
||||
resolve({ files: downloadedFiles.length, errors: 1 });
|
||||
}
|
||||
});
|
||||
|
||||
proc.on('error', (err) => {
|
||||
clearInterval(cancelCheck);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
import { Router } from 'express';
|
||||
import { join, extname } from 'path';
|
||||
import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, readdirSync, rmSync } from 'fs';
|
||||
import { execFile, spawn } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { getVideoById } from './db.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const router = Router();
|
||||
|
||||
const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
|
||||
const CACHE_DIR = join(VIDEOS_PATH, '.hls-cache');
|
||||
const SEGMENT_DURATION = 10;
|
||||
const MAX_CONCURRENT_TRANSCODES = 2;
|
||||
const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
// Ensure cache dir exists
|
||||
if (!existsSync(CACHE_DIR)) {
|
||||
mkdirSync(CACHE_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Quality tiers (no "original" copy mode — always transcode for reliable HLS)
|
||||
const QUALITY_TIERS = {
|
||||
'480p': { maxW: 854, maxH: 480, videoBitrate: '1500k', maxrate: '2000k', bufsize: '3000k', audioBitrate: '96k' },
|
||||
'720p': { maxW: 1280, maxH: 720, videoBitrate: '3000k', maxrate: '4000k', bufsize: '6000k', audioBitrate: '128k' },
|
||||
'1080p': { maxW: 1920, maxH: 1080, videoBitrate: '6000k', maxrate: '8000k', bufsize: '12000k', audioBitrate: '192k' },
|
||||
};
|
||||
|
||||
// Compute output dimensions preserving source aspect ratio, clamped to maxW x maxH
|
||||
function fitDimensions(srcW, srcH, maxW, maxH) {
|
||||
if (srcW <= maxW && srcH <= maxH) {
|
||||
// Already fits — round to even
|
||||
let w = srcW, h = srcH;
|
||||
w += w % 2; h += h % 2;
|
||||
return { w, h };
|
||||
}
|
||||
const scale = Math.min(maxW / srcW, maxH / srcH);
|
||||
let w = Math.round(srcW * scale);
|
||||
let h = Math.round(srcH * scale);
|
||||
w += w % 2; h += h % 2; // ensure even (required for encoding)
|
||||
return { w, h };
|
||||
}
|
||||
|
||||
// Hardware acceleration detection: VAAPI > QSV > libx264
|
||||
// Alpine ships intel-media-driver (VA-API) but not oneVPL GPU runtime (QSV),
|
||||
// so VAAPI is the preferred HW accel path for Intel iGPUs.
|
||||
let hwAccel = null; // 'vaapi' | 'qsv' | null
|
||||
|
||||
async function detectHwAccel() {
|
||||
if (hwAccel !== null) return hwAccel;
|
||||
|
||||
// Try VAAPI first (works on Alpine with intel-media-driver)
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner',
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'format=nv12,hwupload',
|
||||
'-c:v', 'h264_vaapi', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000, env: { ...process.env, LIBVA_DRIVER_NAME: 'iHD' } });
|
||||
hwAccel = 'vaapi';
|
||||
console.log('[video-hls] Intel VAAPI hardware acceleration available');
|
||||
return hwAccel;
|
||||
} catch {
|
||||
// VAAPI failed, try QSV
|
||||
}
|
||||
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-hide_banner', '-init_hw_device', 'qsv=hw',
|
||||
'-f', 'lavfi', '-i', 'nullsrc=s=256x256:d=1',
|
||||
'-vf', 'hwupload=extra_hw_frames=64,format=qsv',
|
||||
'-c:v', 'h264_qsv', '-frames:v', '1', '-f', 'null', '-',
|
||||
], { timeout: 10000 });
|
||||
hwAccel = 'qsv';
|
||||
console.log('[video-hls] Intel QSV hardware acceleration available');
|
||||
return hwAccel;
|
||||
} catch {
|
||||
// QSV also failed
|
||||
}
|
||||
|
||||
hwAccel = null;
|
||||
console.log('[video-hls] No hardware acceleration available, using libx264 fallback');
|
||||
return hwAccel;
|
||||
}
|
||||
|
||||
// Detect on startup
|
||||
detectHwAccel();
|
||||
|
||||
// --- Transcode semaphore ---
|
||||
|
||||
let activeTranscodes = 0;
|
||||
const transcodeQueue = [];
|
||||
|
||||
function acquireTranscodeSlot() {
|
||||
return new Promise((resolve) => {
|
||||
if (activeTranscodes < MAX_CONCURRENT_TRANSCODES) {
|
||||
activeTranscodes++;
|
||||
resolve();
|
||||
} else {
|
||||
transcodeQueue.push(resolve);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function releaseTranscodeSlot() {
|
||||
activeTranscodes--;
|
||||
if (transcodeQueue.length > 0) {
|
||||
activeTranscodes++;
|
||||
transcodeQueue.shift()();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Cache cleanup (hourly) ---
|
||||
|
||||
function cleanupCache() {
|
||||
try {
|
||||
if (!existsSync(CACHE_DIR)) return;
|
||||
const videoDirs = readdirSync(CACHE_DIR, { withFileTypes: true });
|
||||
const now = Date.now();
|
||||
|
||||
for (const dir of videoDirs) {
|
||||
if (!dir.isDirectory()) continue;
|
||||
const videoCacheDir = join(CACHE_DIR, dir.name);
|
||||
|
||||
// Find newest segment mtime across all quality dirs
|
||||
let newestMtime = 0;
|
||||
try {
|
||||
const qualityDirs = readdirSync(videoCacheDir, { withFileTypes: true });
|
||||
for (const qDir of qualityDirs) {
|
||||
if (!qDir.isDirectory()) continue;
|
||||
const qPath = join(videoCacheDir, qDir.name);
|
||||
const files = readdirSync(qPath);
|
||||
for (const f of files) {
|
||||
try {
|
||||
const st = statSync(join(qPath, f));
|
||||
if (st.mtimeMs > newestMtime) newestMtime = st.mtimeMs;
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
} catch { continue; }
|
||||
|
||||
if (newestMtime > 0 && (now - newestMtime) > CACHE_MAX_AGE_MS) {
|
||||
try {
|
||||
rmSync(videoCacheDir, { recursive: true });
|
||||
console.log(`[video-hls] Cleaned cache for video ${dir.name}`);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[video-hls] Cache cleanup error:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Run cleanup every hour
|
||||
setInterval(cleanupCache, 60 * 60 * 1000);
|
||||
|
||||
// --- Probe video duration ---
|
||||
|
||||
async function getVideoDuration(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error', '-show_entries', 'format=duration',
|
||||
'-of', 'csv=p=0', filePath,
|
||||
], { timeout: 15000 });
|
||||
return parseFloat(stdout.trim()) || 0;
|
||||
}
|
||||
|
||||
async function getVideoInfo(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'stream=codec_type,width,height',
|
||||
'-show_entries', 'format=duration',
|
||||
'-of', 'json',
|
||||
filePath,
|
||||
], { timeout: 15000 });
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = info.streams?.find(s => s.codec_type === 'video');
|
||||
return {
|
||||
duration: parseFloat(info.format?.duration || '0'),
|
||||
width: videoStream?.width || 0,
|
||||
height: videoStream?.height || 0,
|
||||
hasAudio: !!info.streams?.find(s => s.codec_type === 'audio'),
|
||||
};
|
||||
}
|
||||
|
||||
// --- Master playlist ---
|
||||
|
||||
// GET /api/video-hls/:id/master.m3u8
|
||||
router.get('/api/video-hls/:id/master.m3u8', async (req, res) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
|
||||
|
||||
const info = await getVideoInfo(video.file_path);
|
||||
const sourceWidth = info.width || video.width || 1920;
|
||||
const sourceHeight = info.height || video.height || 1080;
|
||||
|
||||
let playlist = '#EXTM3U\n';
|
||||
|
||||
// Add quality tiers at or below source resolution (all transcoded, aspect-ratio preserved)
|
||||
for (const [name, tier] of Object.entries(QUALITY_TIERS)) {
|
||||
if (tier.maxH <= sourceHeight) {
|
||||
const { w, h } = fitDimensions(sourceWidth, sourceHeight, tier.maxW, tier.maxH);
|
||||
const bandwidth = parseInt(tier.videoBitrate) * 1000 + parseInt(tier.audioBitrate) * 1000;
|
||||
playlist += `#EXT-X-STREAM-INF:BANDWIDTH=${bandwidth},RESOLUTION=${w}x${h},NAME="${name}"\n`;
|
||||
playlist += `${name}/playlist.m3u8\n`;
|
||||
}
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[video-hls] Master playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate master playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Variant playlist ---
|
||||
|
||||
// GET /api/video-hls/:id/:quality/playlist.m3u8
|
||||
router.get('/api/video-hls/:id/:quality/playlist.m3u8', async (req, res) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const { quality } = req.params;
|
||||
|
||||
if (!QUALITY_TIERS[quality]) {
|
||||
return res.status(400).json({ error: 'Invalid quality' });
|
||||
}
|
||||
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
|
||||
|
||||
const duration = await getVideoDuration(video.file_path);
|
||||
if (!duration || duration <= 0) {
|
||||
return res.status(500).json({ error: 'Could not determine video duration' });
|
||||
}
|
||||
|
||||
const segmentCount = Math.ceil(duration / SEGMENT_DURATION);
|
||||
|
||||
let playlist = '#EXTM3U\n#EXT-X-VERSION:3\n';
|
||||
playlist += `#EXT-X-TARGETDURATION:${SEGMENT_DURATION}\n`;
|
||||
playlist += '#EXT-X-MEDIA-SEQUENCE:0\n';
|
||||
|
||||
for (let i = 0; i < segmentCount; i++) {
|
||||
const remaining = duration - i * SEGMENT_DURATION;
|
||||
const segDuration = Math.min(SEGMENT_DURATION, remaining);
|
||||
playlist += `#EXTINF:${segDuration.toFixed(3)},\n`;
|
||||
playlist += `segment-${i}.ts\n`;
|
||||
}
|
||||
|
||||
playlist += '#EXT-X-ENDLIST\n';
|
||||
|
||||
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.send(playlist);
|
||||
} catch (err) {
|
||||
console.error('[video-hls] Variant playlist error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to generate variant playlist' });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Segment transcoding ---
|
||||
|
||||
// GET /api/video-hls/:id/:quality/segment-:index.ts
|
||||
router.get('/api/video-hls/:id/:quality/segment-:index.ts', async (req, res) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const { quality } = req.params;
|
||||
const segIndex = parseInt(req.params.index, 10);
|
||||
|
||||
if (isNaN(segIndex) || segIndex < 0) {
|
||||
return res.status(400).json({ error: 'Invalid segment index' });
|
||||
}
|
||||
if (!QUALITY_TIERS[quality]) {
|
||||
return res.status(400).json({ error: 'Invalid quality' });
|
||||
}
|
||||
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
if (!existsSync(video.file_path)) return res.status(404).json({ error: 'Video file missing' });
|
||||
|
||||
// Check cache first
|
||||
const segmentCacheDir = join(CACHE_DIR, String(id), quality);
|
||||
const segmentCachePath = join(segmentCacheDir, `segment-${segIndex}.ts`);
|
||||
|
||||
if (existsSync(segmentCachePath)) {
|
||||
const stat = statSync(segmentCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segmentCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
// Transcode on-demand
|
||||
await acquireTranscodeSlot();
|
||||
|
||||
// Check cache again after acquiring slot (another request may have cached it)
|
||||
if (existsSync(segmentCachePath)) {
|
||||
releaseTranscodeSlot();
|
||||
const stat = statSync(segmentCachePath);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'video/MP2T',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=3600',
|
||||
});
|
||||
createReadStream(segmentCachePath).pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const offset = segIndex * SEGMENT_DURATION;
|
||||
const accel = await detectHwAccel();
|
||||
const tier = QUALITY_TIERS[quality];
|
||||
|
||||
// Compute output dimensions preserving source aspect ratio
|
||||
const srcW = video.width || 1920;
|
||||
const srcH = video.height || 1080;
|
||||
const { w: outW, h: outH } = fitDimensions(srcW, srcH, tier.maxW, tier.maxH);
|
||||
|
||||
// -output_ts_offset ensures PTS continuity across segments (each segment's PTS
|
||||
// starts where the previous one ended, required for smooth HLS playback)
|
||||
let ffmpegArgs;
|
||||
if (accel === 'vaapi') {
|
||||
ffmpegArgs = [
|
||||
'-init_hw_device', 'vaapi=va:/dev/dri/renderD128',
|
||||
'-filter_hw_device', 'va',
|
||||
'-ss', String(offset),
|
||||
'-i', video.file_path,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `format=nv12,hwupload,scale_vaapi=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_vaapi',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else if (accel === 'qsv') {
|
||||
ffmpegArgs = [
|
||||
'-hwaccel', 'qsv', '-hwaccel_output_format', 'qsv',
|
||||
'-ss', String(offset),
|
||||
'-i', video.file_path,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale_qsv=w=${outW}:h=${outH}`,
|
||||
'-c:v', 'h264_qsv',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
} else {
|
||||
ffmpegArgs = [
|
||||
'-ss', String(offset),
|
||||
'-i', video.file_path,
|
||||
'-t', String(SEGMENT_DURATION),
|
||||
'-output_ts_offset', String(offset),
|
||||
'-vf', `scale=${outW}:${outH}`,
|
||||
'-c:v', 'libx264', '-preset', 'veryfast',
|
||||
'-b:v', tier.videoBitrate, '-maxrate', tier.maxrate, '-bufsize', tier.bufsize,
|
||||
'-c:a', 'aac', '-b:a', tier.audioBitrate, '-ac', '2',
|
||||
'-f', 'mpegts',
|
||||
'pipe:1',
|
||||
];
|
||||
}
|
||||
|
||||
const spawnEnv = accel === 'vaapi'
|
||||
? { ...process.env, LIBVA_DRIVER_NAME: 'iHD' }
|
||||
: undefined;
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', ffmpegArgs, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
...(spawnEnv && { env: spawnEnv }),
|
||||
});
|
||||
|
||||
// Stream directly to client while also collecting chunks for cache
|
||||
res.setHeader('Content-Type', 'video/MP2T');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
|
||||
const cacheChunks = [];
|
||||
let aborted = false;
|
||||
|
||||
ffmpeg.stdout.on('data', (chunk) => {
|
||||
cacheChunks.push(chunk);
|
||||
if (!res.destroyed) res.write(chunk);
|
||||
});
|
||||
|
||||
req.on('close', () => {
|
||||
if (!ffmpeg.killed) {
|
||||
aborted = true;
|
||||
ffmpeg.kill('SIGKILL');
|
||||
releaseTranscodeSlot();
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (aborted) return;
|
||||
releaseTranscodeSlot();
|
||||
|
||||
// Write cache file on success
|
||||
if (code === 0 && cacheChunks.length > 0) {
|
||||
try {
|
||||
if (!existsSync(segmentCacheDir)) mkdirSync(segmentCacheDir, { recursive: true });
|
||||
const cacheStream = createWriteStream(segmentCachePath);
|
||||
for (const chunk of cacheChunks) cacheStream.write(chunk);
|
||||
cacheStream.end();
|
||||
} catch { /* ignore cache write failure */ }
|
||||
}
|
||||
|
||||
if (!res.destroyed) res.end();
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (!aborted) releaseTranscodeSlot();
|
||||
console.error('[video-hls] ffmpeg error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: 'Transcoding failed' });
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[video-hls] Segment error:', err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -0,0 +1,445 @@
|
||||
import { Router } from 'express';
|
||||
import multer from 'multer';
|
||||
import { join, extname, basename } from 'path';
|
||||
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync, createReadStream, rmSync } from 'fs';
|
||||
import { execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import {
|
||||
insertVideo, getVideoById, getVideoByPath, updateVideo, deleteVideoById,
|
||||
searchVideos, getOrCreateTag, getAllTags, setVideoTags, getVideoTags,
|
||||
} from './db.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const router = Router();
|
||||
|
||||
const VIDEOS_PATH = process.env.VIDEOS_PATH || '/data/videos';
|
||||
const VIDEO_EXTS = new Set(['.mp4', '.mov', '.avi', '.webm', '.mkv', '.m4v', '.wmv', '.flv', '.ts']);
|
||||
|
||||
// Ensure videos dir exists
|
||||
if (!existsSync(VIDEOS_PATH)) {
|
||||
mkdirSync(VIDEOS_PATH, { recursive: true });
|
||||
}
|
||||
|
||||
// Multer config for uploads
|
||||
const storage = multer.diskStorage({
|
||||
destination: (req, file, cb) => cb(null, VIDEOS_PATH),
|
||||
filename: (req, file, cb) => {
|
||||
// Preserve original name, avoid collisions
|
||||
let name = file.originalname;
|
||||
const filePath = join(VIDEOS_PATH, name);
|
||||
if (existsSync(filePath)) {
|
||||
const ext = extname(name);
|
||||
const base = basename(name, ext);
|
||||
name = `${base}_${Date.now()}${ext}`;
|
||||
}
|
||||
cb(null, name);
|
||||
},
|
||||
});
|
||||
|
||||
const upload = multer({
|
||||
storage,
|
||||
fileFilter: (req, file, cb) => {
|
||||
const ext = extname(file.originalname).toLowerCase();
|
||||
if (VIDEO_EXTS.has(ext)) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(new Error(`Unsupported file type: ${ext}`));
|
||||
}
|
||||
},
|
||||
limits: { fileSize: 50 * 1024 * 1024 * 1024 }, // 50 GB
|
||||
});
|
||||
|
||||
// --- ffprobe helper ---
|
||||
|
||||
async function probeVideo(filePath) {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-show_entries', 'format=duration,bit_rate',
|
||||
'-show_entries', 'stream=codec_name,width,height,r_frame_rate,codec_type',
|
||||
'-of', 'json',
|
||||
filePath,
|
||||
], { timeout: 60000 });
|
||||
|
||||
const info = JSON.parse(stdout);
|
||||
const videoStream = info.streams?.find(s => s.codec_type === 'video');
|
||||
const audioStream = info.streams?.find(s => s.codec_type === 'audio');
|
||||
const duration = parseFloat(info.format?.duration || '0');
|
||||
const bitrate = parseInt(info.format?.bit_rate || '0', 10);
|
||||
|
||||
let fps = null;
|
||||
if (videoStream?.r_frame_rate) {
|
||||
const [num, den] = videoStream.r_frame_rate.split('/');
|
||||
if (den && parseInt(den, 10) > 0) {
|
||||
fps = Math.round((parseInt(num, 10) / parseInt(den, 10)) * 100) / 100;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
duration: duration || null,
|
||||
width: videoStream?.width || null,
|
||||
height: videoStream?.height || null,
|
||||
fps,
|
||||
codec: videoStream?.codec_name || null,
|
||||
bitrate: bitrate || null,
|
||||
has_audio: audioStream ? 1 : 0,
|
||||
};
|
||||
}
|
||||
|
||||
// --- thumbnail generation ---
|
||||
|
||||
async function generateVideoThumbnail(filePath, outputPath) {
|
||||
const dir = join(VIDEOS_PATH, '.thumbnails');
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Seek 1s in for a better frame
|
||||
let duration = 0;
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', filePath,
|
||||
], { timeout: 15000 });
|
||||
duration = parseFloat(stdout.trim()) || 0;
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const seekTime = duration > 2 ? '1' : '0';
|
||||
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-ss', seekTime,
|
||||
'-i', filePath,
|
||||
'-frames:v', '1',
|
||||
'-vf', 'scale=480:-1',
|
||||
'-q:v', '4',
|
||||
'-y',
|
||||
'-update', '1',
|
||||
outputPath,
|
||||
], { timeout: 30000 });
|
||||
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
// --- Scan state ---
|
||||
|
||||
let scanState = { running: false, total: 0, done: 0, added: 0, skipped: 0, errors: 0 };
|
||||
|
||||
// GET /api/videos — browse/search
|
||||
router.get('/api/videos', (req, res, next) => {
|
||||
try {
|
||||
const { search, sort, offset, limit, minDuration, maxDuration, minWidth } = req.query;
|
||||
const tagsParam = req.query.tags;
|
||||
const tagsArr = tagsParam
|
||||
? tagsParam.split(',').map(t => t.trim()).filter(Boolean)
|
||||
: undefined;
|
||||
|
||||
const result = searchVideos({
|
||||
search: search || undefined,
|
||||
tags: tagsArr,
|
||||
minDuration: minDuration || undefined,
|
||||
maxDuration: maxDuration || undefined,
|
||||
minWidth: minWidth || undefined,
|
||||
sort: sort || 'latest',
|
||||
offset: parseInt(offset || '0', 10),
|
||||
limit: parseInt(limit || '48', 10),
|
||||
});
|
||||
|
||||
// Attach tags to each video
|
||||
const videos = result.rows.map(v => ({
|
||||
...v,
|
||||
tags: getVideoTags(v.id),
|
||||
}));
|
||||
|
||||
res.json({ total: result.total, offset: parseInt(offset || '0', 10), videos });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/videos/tags — all tags with counts
|
||||
router.get('/api/videos/tags', (req, res, next) => {
|
||||
try {
|
||||
const { search } = req.query;
|
||||
const tags = getAllTags(search || undefined);
|
||||
res.json(tags);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/videos/scan/status
|
||||
router.get('/api/videos/scan/status', (req, res) => {
|
||||
res.json(scanState);
|
||||
});
|
||||
|
||||
// GET /api/videos/:id
|
||||
router.get('/api/videos/:id', (req, res, next) => {
|
||||
try {
|
||||
const video = getVideoById(parseInt(req.params.id, 10));
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
video.tags = getVideoTags(video.id);
|
||||
res.json(video);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// PUT /api/videos/:id — update title, description, tags
|
||||
router.put('/api/videos/:id', (req, res, next) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
|
||||
const { title, description, tags } = req.body;
|
||||
const updates = {};
|
||||
if (title !== undefined) updates.title = title;
|
||||
if (description !== undefined) updates.description = description;
|
||||
|
||||
if (Object.keys(updates).length > 0) {
|
||||
updateVideo(id, updates);
|
||||
}
|
||||
if (Array.isArray(tags)) {
|
||||
setVideoTags(id, tags);
|
||||
}
|
||||
|
||||
const updated = getVideoById(id);
|
||||
updated.tags = getVideoTags(id);
|
||||
res.json(updated);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/videos/:id
|
||||
router.delete('/api/videos/:id', (req, res, next) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
|
||||
// Delete file
|
||||
if (existsSync(video.file_path)) {
|
||||
try { unlinkSync(video.file_path); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Delete thumbnail
|
||||
if (video.thumbnail_path && existsSync(video.thumbnail_path)) {
|
||||
try { unlinkSync(video.thumbnail_path); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Delete HLS cache
|
||||
const hlsCacheDir = join(VIDEOS_PATH, '.hls-cache', String(id));
|
||||
if (existsSync(hlsCacheDir)) {
|
||||
try { rmSync(hlsCacheDir, { recursive: true }); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
deleteVideoById(id);
|
||||
res.json({ ok: true });
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/videos/upload — multipart file upload
|
||||
router.post('/api/videos/upload', upload.single('video'), async (req, res) => {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({ error: 'No video file provided' });
|
||||
}
|
||||
|
||||
const filePath = req.file.path;
|
||||
const filename = req.file.filename;
|
||||
|
||||
try {
|
||||
// Check for dupe
|
||||
const existing = getVideoByPath(filePath);
|
||||
if (existing) {
|
||||
return res.json({ video: existing, duplicate: true });
|
||||
}
|
||||
|
||||
const stat = statSync(filePath);
|
||||
const probe = await probeVideo(filePath);
|
||||
|
||||
// Generate thumbnail
|
||||
const thumbName = filename.replace(/\.[^.]+$/, '.jpg');
|
||||
const thumbPath = join(VIDEOS_PATH, '.thumbnails', thumbName);
|
||||
let thumbResult = null;
|
||||
try {
|
||||
thumbResult = await generateVideoThumbnail(filePath, thumbPath);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const title = basename(filename, extname(filename))
|
||||
.replace(/[_.-]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
const videoId = insertVideo({
|
||||
title,
|
||||
filename,
|
||||
file_path: filePath,
|
||||
file_size: stat.size,
|
||||
...probe,
|
||||
thumbnail_path: thumbResult || null,
|
||||
status: 'ready',
|
||||
});
|
||||
|
||||
const video = getVideoById(videoId);
|
||||
video.tags = [];
|
||||
res.json({ video });
|
||||
} catch (err) {
|
||||
console.error('[videos] Upload processing failed:', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/videos/scan — scan VIDEOS_PATH for new files
|
||||
router.post('/api/videos/scan', (req, res) => {
|
||||
if (scanState.running) {
|
||||
return res.json({ status: 'already_running', ...scanState });
|
||||
}
|
||||
|
||||
scanState = { running: true, total: 0, done: 0, added: 0, skipped: 0, errors: 0 };
|
||||
res.json({ status: 'started' });
|
||||
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
// Collect all video files
|
||||
const videoFiles = [];
|
||||
const collectFiles = (dir) => {
|
||||
let entries;
|
||||
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
collectFiles(fullPath);
|
||||
} else {
|
||||
const ext = extname(entry.name).toLowerCase();
|
||||
if (VIDEO_EXTS.has(ext)) {
|
||||
videoFiles.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
collectFiles(VIDEOS_PATH);
|
||||
|
||||
scanState.total = videoFiles.length;
|
||||
console.log(`[videos] Scan found ${videoFiles.length} video files`);
|
||||
|
||||
for (const filePath of videoFiles) {
|
||||
try {
|
||||
// Skip if already indexed
|
||||
const existing = getVideoByPath(filePath);
|
||||
if (existing) {
|
||||
scanState.skipped++;
|
||||
scanState.done++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const stat = statSync(filePath);
|
||||
const filename = basename(filePath);
|
||||
|
||||
// Probe metadata
|
||||
let probe;
|
||||
try {
|
||||
probe = await probeVideo(filePath);
|
||||
} catch (err) {
|
||||
console.error(`[videos] Probe failed for ${filename}:`, err.message);
|
||||
scanState.errors++;
|
||||
scanState.done++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate thumbnail
|
||||
const thumbName = `${Date.now()}_${filename.replace(/\.[^.]+$/, '.jpg')}`;
|
||||
const thumbPath = join(VIDEOS_PATH, '.thumbnails', thumbName);
|
||||
let thumbResult = null;
|
||||
try {
|
||||
thumbResult = await generateVideoThumbnail(filePath, thumbPath);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const title = basename(filename, extname(filename))
|
||||
.replace(/[_.-]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
insertVideo({
|
||||
title,
|
||||
filename,
|
||||
file_path: filePath,
|
||||
file_size: stat.size,
|
||||
...probe,
|
||||
thumbnail_path: thumbResult || null,
|
||||
status: 'ready',
|
||||
});
|
||||
|
||||
scanState.added++;
|
||||
scanState.done++;
|
||||
} catch (err) {
|
||||
console.error(`[videos] Scan error for ${filePath}:`, err.message);
|
||||
scanState.errors++;
|
||||
scanState.done++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[videos] Scan complete: ${scanState.added} added, ${scanState.skipped} skipped, ${scanState.errors} errors`);
|
||||
} catch (err) {
|
||||
console.error('[videos] Scan failed:', err.message);
|
||||
} finally {
|
||||
scanState.running = false;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// GET /api/videos/:id/thumbnail
|
||||
router.get('/api/videos/:id/thumbnail', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
|
||||
if (video.thumbnail_path && existsSync(video.thumbnail_path)) {
|
||||
const stat = statSync(video.thumbnail_path);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Content-Length': stat.size,
|
||||
'Cache-Control': 'public, max-age=86400',
|
||||
});
|
||||
createReadStream(video.thumbnail_path).pipe(res);
|
||||
} else {
|
||||
// Return a placeholder
|
||||
res.status(404).json({ error: 'No thumbnail' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /api/videos/:id/stream — direct file serve for grid wall playback
|
||||
router.get('/api/videos/:id/stream', (req, res) => {
|
||||
const id = parseInt(req.params.id, 10);
|
||||
const video = getVideoById(id);
|
||||
if (!video) return res.status(404).json({ error: 'Video not found' });
|
||||
if (!existsSync(video.file_path)) return res.status(404).json({ error: 'File not found' });
|
||||
|
||||
const stat = statSync(video.file_path);
|
||||
const ext = extname(video.file_path).toLowerCase();
|
||||
const mimeTypes = { '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime', '.mkv': 'video/x-matroska', '.m4v': 'video/mp4' };
|
||||
const contentType = mimeTypes[ext] || 'video/mp4';
|
||||
|
||||
// Support range requests
|
||||
const range = req.headers.range;
|
||||
if (range) {
|
||||
const parts = range.replace(/bytes=/, '').split('-');
|
||||
const start = parseInt(parts[0], 10);
|
||||
const end = parts[1] ? parseInt(parts[1], 10) : stat.size - 1;
|
||||
res.writeHead(206, {
|
||||
'Content-Range': `bytes ${start}-${end}/${stat.size}`,
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Length': end - start + 1,
|
||||
'Content-Type': contentType,
|
||||
});
|
||||
createReadStream(video.file_path, { start, end }).pipe(res);
|
||||
} else {
|
||||
res.writeHead(200, {
|
||||
'Content-Length': stat.size,
|
||||
'Content-Type': contentType,
|
||||
'Accept-Ranges': 'bytes',
|
||||
});
|
||||
createReadStream(video.file_path).pipe(res);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
Reference in New Issue
Block a user