Files
OFApp/server/db.js
Trey t 1e5f54f60b Add DRM downloads, scrapers, gallery index, and UI improvements
- DRM video download pipeline with pywidevine subprocess for Widevine key acquisition
- Scraper system: forum threads, Coomer/Kemono API, and MediaLink (Fapello) scrapers
- SQLite-backed media index for instant gallery loads with startup scan
- Duplicate detection and gallery filtering/sorting
- HLS video component, log viewer, and scrape management UI
- Dockerfile updated for Python/pywidevine, docker-compose volume for CDM

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 11:29:11 -06:00

247 lines
7.5 KiB
JavaScript

import Database from 'better-sqlite3';
import { mkdirSync, existsSync } from 'fs';
import { dirname } from 'path';
const DB_PATH = process.env.DB_PATH || './data/db/ofapp.db';
const dir = dirname(DB_PATH);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
const db = new Database(DB_PATH);
db.pragma('journal_mode = WAL');
db.exec(`
CREATE TABLE IF NOT EXISTS auth_config (
user_id TEXT,
cookie TEXT,
x_bc TEXT,
app_token TEXT,
x_of_rev TEXT,
user_agent TEXT
);
CREATE TABLE IF NOT EXISTS download_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT,
post_id TEXT,
media_id TEXT,
media_type TEXT,
filename TEXT,
downloaded_at TEXT
);
CREATE TABLE IF NOT EXISTS download_cursors (
user_id TEXT UNIQUE,
cursor TEXT,
posts_downloaded INTEGER
);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT
);
CREATE TABLE IF NOT EXISTS media_files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
folder TEXT NOT NULL,
filename TEXT NOT NULL,
type TEXT NOT NULL,
size INTEGER NOT NULL,
modified REAL NOT NULL,
posted_at TEXT,
created_at TEXT DEFAULT (datetime('now')),
UNIQUE(folder, filename)
);
CREATE INDEX IF NOT EXISTS idx_media_folder ON media_files(folder);
CREATE INDEX IF NOT EXISTS idx_media_type ON media_files(type);
CREATE INDEX IF NOT EXISTS idx_media_modified ON media_files(modified);
CREATE INDEX IF NOT EXISTS idx_media_posted_at ON media_files(posted_at);
`);
// Migration: add posted_at column if missing
const cols = db.prepare("PRAGMA table_info(download_history)").all().map((c) => c.name);
if (!cols.includes('posted_at')) {
db.exec('ALTER TABLE download_history ADD COLUMN posted_at TEXT');
}
export function getAuthConfig() {
const row = db.prepare('SELECT * FROM auth_config LIMIT 1').get();
return row || null;
}
export function saveAuthConfig(config) {
const del = db.prepare('DELETE FROM auth_config');
const ins = db.prepare(
'INSERT INTO auth_config (user_id, cookie, x_bc, app_token, x_of_rev, user_agent) VALUES (?, ?, ?, ?, ?, ?)'
);
const upsert = db.transaction((c) => {
del.run();
ins.run(c.user_id, c.cookie, c.x_bc, c.app_token, c.x_of_rev, c.user_agent);
});
upsert(config);
}
export function isMediaDownloaded(mediaId) {
const row = db.prepare('SELECT 1 FROM download_history WHERE media_id = ? LIMIT 1').get(String(mediaId));
return !!row;
}
export function recordDownload(userId, postId, mediaId, mediaType, filename, postedAt) {
db.prepare(
'INSERT INTO download_history (user_id, post_id, media_id, media_type, filename, downloaded_at, posted_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
).run(String(userId), String(postId), String(mediaId), mediaType, filename, new Date().toISOString(), postedAt || null);
}
export function getDownloadHistory(userId) {
return db.prepare('SELECT * FROM download_history WHERE user_id = ? ORDER BY downloaded_at DESC').all(String(userId));
}
export function saveCursor(userId, cursor, postsDownloaded) {
db.prepare(
'INSERT INTO download_cursors (user_id, cursor, posts_downloaded) VALUES (?, ?, ?) ON CONFLICT(user_id) DO UPDATE SET cursor = excluded.cursor, posts_downloaded = excluded.posts_downloaded'
).run(String(userId), cursor, postsDownloaded);
}
export function getCursor(userId) {
return db.prepare('SELECT cursor, posts_downloaded FROM download_cursors WHERE user_id = ?').get(String(userId)) || null;
}
export function clearCursor(userId) {
db.prepare('DELETE FROM download_cursors WHERE user_id = ?').run(String(userId));
}
export function getPostDateByFilename(filename) {
const row = db.prepare('SELECT posted_at FROM download_history WHERE filename = ? LIMIT 1').get(filename);
return row?.posted_at || null;
}
export function getSetting(key) {
const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key);
return row ? row.value : null;
}
export function setSetting(key, value) {
db.prepare(
'INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value'
).run(key, value);
}
export function getAllSettings() {
const rows = db.prepare('SELECT key, value FROM settings').all();
const obj = {};
for (const row of rows) obj[row.key] = row.value;
return obj;
}
export function getDownloadStats() {
return db.prepare(
'SELECT user_id, COUNT(*) as file_count, MAX(downloaded_at) as last_download FROM download_history GROUP BY user_id'
).all();
}
// --- media_files helpers ---
const upsertMediaStmt = db.prepare(`
INSERT INTO media_files (folder, filename, type, size, modified, posted_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(folder, filename) DO UPDATE SET
size = excluded.size,
modified = excluded.modified,
posted_at = COALESCE(excluded.posted_at, media_files.posted_at)
`);
export function upsertMediaFile(folder, filename, type, size, modified, postedAt) {
upsertMediaStmt.run(folder, filename, type, size, modified, postedAt || null);
}
export const upsertMediaFileBatch = db.transaction((files) => {
for (const f of files) {
upsertMediaStmt.run(f.folder, f.filename, f.type, f.size, f.modified, f.postedAt || null);
}
});
export function removeMediaFile(folder, filename) {
db.prepare('DELETE FROM media_files WHERE folder = ? AND filename = ?').run(folder, filename);
}
export function getMediaFolders() {
return db.prepare(`
SELECT folder AS name,
COUNT(*) AS total,
SUM(CASE WHEN type = 'image' THEN 1 ELSE 0 END) AS images,
SUM(CASE WHEN type = 'video' THEN 1 ELSE 0 END) AS videos
FROM media_files
GROUP BY folder
ORDER BY folder
`).all();
}
export function getMediaFiles({ folder, folders, type, sort, offset, limit }) {
const conditions = [];
const params = [];
if (folder) {
conditions.push('folder = ?');
params.push(folder);
} else if (folders && folders.length > 0) {
conditions.push(`folder IN (${folders.map(() => '?').join(',')})`);
params.push(...folders);
}
if (type && type !== 'all') {
conditions.push('type = ?');
params.push(type);
}
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
const countRow = db.prepare(`SELECT COUNT(*) AS total FROM media_files ${where}`).get(...params);
const total = countRow.total;
let orderBy;
if (sort === 'shuffle') {
orderBy = 'ORDER BY RANDOM()';
} else {
// 'latest' — prefer posted_at, fall back to modified
orderBy = 'ORDER BY COALESCE(posted_at, datetime(modified / 1000, \'unixepoch\')) DESC';
}
const rows = db.prepare(`
SELECT folder, filename, type, size, modified, posted_at
FROM media_files
${where}
${orderBy}
LIMIT ? OFFSET ?
`).all(...params, limit || 50, offset || 0);
return { total, rows };
}
export function getAllIndexedFolders() {
return db.prepare('SELECT DISTINCT folder FROM media_files').all().map(r => r.folder);
}
export function removeStaleFiles(folder, existingFilenames) {
const rows = db.prepare('SELECT filename FROM media_files WHERE folder = ?').all(folder);
const existing = new Set(existingFilenames);
const toDelete = rows.filter(r => !existing.has(r.filename));
if (toDelete.length > 0) {
const del = db.prepare('DELETE FROM media_files WHERE folder = ? AND filename = ?');
const batch = db.transaction((files) => {
for (const f of files) del.run(folder, f.filename);
});
batch(toDelete);
}
return toDelete.length;
}
export function getMediaFileCount() {
return db.prepare('SELECT COUNT(*) AS count FROM media_files').get().count;
}