/* * Per-scrape storage: one directory, one SQLite index, one media folder. * Layout and rules: docs/media.md. */ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { fileURLToPath } from "node:url"; import { extractMedia } from "./extract.js"; import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js"; /** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */ export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url)); export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string { return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT; } export function scrapeDir(root: string, id: string): string { return join(root, id); } export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void { writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n"); } export function readScrapeMeta(dir: string): ScrapeMeta | null { try { return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta; } catch { return null; } } /** Every scrape on disk, newest first. */ export function listScrapes(root: string): ScrapeMeta[] { let entries: string[]; try { entries = readdirSync(root); } catch { return []; } const metas: ScrapeMeta[] = []; for (const entry of entries) { const meta = readScrapeMeta(join(root, entry)); if (meta) metas.push(meta); } return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1)); } /** Combined size of a directory tree, used by the storage limit. */ export function dirSizeBytes(dir: string): number { let entries; try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return 0; } let total = 0; for (const entry of entries) { const path = join(dir, entry.name); if (entry.isDirectory()) { total += dirSizeBytes(path); } else { try { total += statSync(path).size; } catch { // A file that vanished mid-walk counts as zero. } } } return total; } export function createScrapeDirs(root: string, id: string): string { const dir = scrapeDir(root, id); mkdirSync(join(dir, "media"), { recursive: true }); mkdirSync(join(dir, "thumbs"), { recursive: true }); return dir; } /** `-`, with a numeric suffix on a same-second collision. */ export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string { const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, ""); const base = `${channelId}-${stamp}`; let id = base; let suffix = 2; while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`; return id; } export interface MediaRow { id: number; kind: MediaKind; source: string | null; url: string; provider: string | null; content_type: string | null; size: number | null; width: number | null; height: number | null; duration_secs: number | null; downloaded: number; preview_url: string | null; file_hash: string | null; preview_hash: string | null; file_bytes: number | null; preview_bytes: number | null; file_path: string | null; download_error: string | null; preview_error: string | null; created_at: string; } const SCHEMA = ` CREATE TABLE IF NOT EXISTS messages ( id TEXT PRIMARY KEY, author_id TEXT NOT NULL, author_name TEXT NOT NULL, timestamp TEXT NOT NULL, content TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS media ( id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, source TEXT, url TEXT NOT NULL UNIQUE, provider TEXT, content_type TEXT, size INTEGER, width INTEGER, height INTEGER, duration_secs REAL, downloaded INTEGER NOT NULL DEFAULT 0, preview_url TEXT, file_hash TEXT, preview_hash TEXT, file_bytes INTEGER, preview_bytes INTEGER, file_path TEXT, download_error TEXT, preview_error TEXT, created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS media_sources ( media_id INTEGER NOT NULL REFERENCES media(id), message_id TEXT NOT NULL, PRIMARY KEY (media_id, message_id) ); CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind); CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id); `; /** One open SQLite index. The caller closes it. */ export class ScrapeStore { private readonly db: DatabaseSync; constructor(readonly dir: string) { this.db = new DatabaseSync(join(dir, "index.db")); this.db.exec(SCHEMA); this.migrate(); } /** Add columns introduced after a scrape was created. */ private migrate(): void { const columns = this.db.prepare("PRAGMA table_info(media)").all() as unknown as Array<{ name: string }>; const names = new Set(columns.map(column => column.name)); if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER"); if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER"); if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT"); if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT"); if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT"); } close(): void { this.db.close(); } /** Write one page of messages in a transaction. Returns the newly inserted media per kind. */ insertPage(messages: CoreMessage[]): Record { const added = emptyMediaCounts(); const insertMessage = this.db.prepare( "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)" ); const insertMedia = this.db.prepare( "INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ); const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?"); const linkSource = this.db.prepare( "INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)" ); const createdAt = new Date().toISOString(); this.db.exec("BEGIN"); try { for (const message of messages) { insertMessage.run( message.id, message.author.id, message.author.username, message.timestamp, message.content ); for (const item of extractMedia(message)) { const result = insertMedia.run( item.kind, item.source, item.url, item.provider, item.contentType, item.size, item.width, item.height, item.durationSecs, item.downloaded ? 1 : 0, item.previewUrl, createdAt ); if (result.changes > 0) added[item.kind]++; const row = findMedia.get(item.url) as { id: number } | undefined; if (row) linkSource.run(row.id, message.id); } } this.db.exec("COMMIT"); } catch (error) { this.db.exec("ROLLBACK"); throw error; } return added; } /** Bytes actually on disk per kind (downloaded files and cached previews). */ storedBytesByKind(): Record { const rows = this.db .prepare( "SELECT kind, COALESCE(SUM(COALESCE(file_bytes, 0) + COALESCE(preview_bytes, 0)), 0) AS bytes FROM media GROUP BY kind" ) .all() as unknown as Array<{ kind: string; bytes: number }>; const result = emptyMediaCounts(); for (const row of rows) { if (row.kind in result) result[row.kind as MediaKind] = row.bytes; } return result; } counts(): Record { const rows = this.db .prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind") .all() as unknown as Array<{ kind: string; n: number }>; const result = emptyMediaCounts(); for (const row of rows) { if (row.kind in result) result[row.kind as MediaKind] = row.n; } return result; } /** Media rows that still need bytes: never downloaded and not skipped. */ pendingDownloads(limit = 5000): MediaRow[] { return this.db .prepare( "SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?" ) .all(limit) as unknown as MediaRow[]; } markDownloaded( id: number, file: { path: string; hash: string; bytes: number } ): void { this.db .prepare( "UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?" ) .run(file.path, file.hash, file.bytes, id); } markDownloadError(id: number, error: string): void { this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id); } /** * Rows that still need a cached preview. Attachment/sticker images and GIFs always; * embed thumbnails only when `includeEmbeds` is set (download mode). */ pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] { const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))"; const condition = includeEmbeds ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))` : local; return this.db .prepare( `SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?` ) .all(limit) as unknown as MediaRow[]; } markPreview(id: number, preview: { hash: string; bytes: number }): void { this.db .prepare("UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_error = NULL WHERE id = ?") .run(preview.hash, preview.bytes, id); } markPreviewError(id: number, error: string): void { this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id); } getMedia(id: number): MediaRow | null { const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as | MediaRow | undefined; return row ?? null; } queryMedia( options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {} ): MediaRow[] { const limit = Math.max(1, Math.min(options.limit ?? 50, 500)); const offset = Math.max(0, options.offset ?? 0); const conditions: string[] = []; const values: Array = []; if (options.kind) { conditions.push("kind = ?"); values.push(options.kind); } if (options.source) { conditions.push("source = ?"); values.push(options.source); } const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; const rows = this.db .prepare(`SELECT * FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`) .all(...values, limit, offset); return rows as unknown as MediaRow[]; } }