/* * Per-scrape storage: one directory, one SQLite index, one media folder. * Layout and rules: docs/media.md. */ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { fileURLToPath } from "node:url"; import { extractMedia } from "./extract.js"; import { aspectCondition, resolutionCondition } from "./filter-presets.js"; import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js"; /** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */ export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url)); export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string { return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT; } export function scrapeDir(root: string, id: string): string { return join(root, id); } export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void { writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n"); } export function readScrapeMeta(dir: string): ScrapeMeta | null { try { return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta; } catch { return null; } } /** Every scrape on disk, newest first. */ export function listScrapes(root: string): ScrapeMeta[] { let entries: string[]; try { entries = readdirSync(root); } catch { return []; } const metas: ScrapeMeta[] = []; for (const entry of entries) { const meta = readScrapeMeta(join(root, entry)); if (meta) metas.push(meta); } return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1)); } /** Combined size of a directory tree, used by the storage limit. */ export function dirSizeBytes(dir: string): number { let entries; try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return 0; } let total = 0; for (const entry of entries) { const path = join(dir, entry.name); if (entry.isDirectory()) { total += dirSizeBytes(path); } else { try { total += statSync(path).size; } catch { // A file that vanished mid-walk counts as zero. } } } return total; } /** Width / height, rounded to three decimals. Null when either side is missing or zero. */ export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null { if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null; return Math.round(((width as number) / (height as number)) * 1000) / 1000; } export function createScrapeDirs(root: string, id: string): string { const dir = scrapeDir(root, id); mkdirSync(join(dir, "media"), { recursive: true }); mkdirSync(join(dir, "thumbs"), { recursive: true }); return dir; } /** `-`, with a numeric suffix on a same-second collision. */ export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string { const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, ""); const base = `${channelId}-${stamp}`; let id = base; let suffix = 2; while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`; return id; } export interface MediaFilterOptions { kind?: MediaKind; source?: string; authorId?: string; authorIds?: string[]; cachedOnly?: boolean; aspect?: string; resolution?: string; after?: string; before?: string; limit?: number; offset?: number; } export interface MediaRow { id: number; kind: MediaKind; source: string | null; author_id: string | null; author_name: string | null; posted_at: string | null; url: string; play_url: string | null; provider: string | null; content_type: string | null; size: number | null; width: number | null; height: number | null; aspect_ratio: number | null; duration_secs: number | null; downloaded: number; preview_url: string | null; file_hash: string | null; preview_hash: string | null; preview_ext: string | null; anim_hash: string | null; file_bytes: number | null; preview_bytes: number | null; file_path: string | null; download_error: string | null; preview_error: string | null; created_at: string; } const SCHEMA = ` CREATE TABLE IF NOT EXISTS messages ( id TEXT PRIMARY KEY, author_id TEXT NOT NULL, author_name TEXT NOT NULL, timestamp TEXT NOT NULL, content TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS media ( id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, source TEXT, author_id TEXT, author_name TEXT, posted_at TEXT, url TEXT NOT NULL UNIQUE, play_url TEXT, provider TEXT, content_type TEXT, size INTEGER, width INTEGER, height INTEGER, aspect_ratio REAL, duration_secs REAL, downloaded INTEGER NOT NULL DEFAULT 0, preview_url TEXT, file_hash TEXT, preview_hash TEXT, preview_ext TEXT, anim_hash TEXT, file_bytes INTEGER, preview_bytes INTEGER, file_path TEXT, download_error TEXT, preview_error TEXT, created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS media_sources ( media_id INTEGER NOT NULL REFERENCES media(id), message_id TEXT NOT NULL, PRIMARY KEY (media_id, message_id) ); CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind); CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at); CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id); `; /** One open SQLite index. The caller closes it. */ export class ScrapeStore { private readonly db: DatabaseSync; constructor(readonly dir: string) { this.db = new DatabaseSync(join(dir, "index.db")); this.db.exec(SCHEMA); this.migrate(); } /** Add columns introduced after a scrape was created. */ private migrate(): void { const columns = this.db.prepare("PRAGMA table_info(media)").all() as unknown as Array<{ name: string }>; const names = new Set(columns.map(column => column.name)); if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER"); if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER"); if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT"); if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT"); if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT"); if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT"); if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT"); if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT"); if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT"); if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT"); if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT"); if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL"); } close(): void { this.db.close(); } /** Write one page of messages in a transaction. Returns the newly inserted media per kind. */ insertPage(messages: CoreMessage[]): Record { const added = emptyMediaCounts(); const insertMessage = this.db.prepare( "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)" ); const insertMedia = this.db.prepare( "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ); const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?"); const linkSource = this.db.prepare( "INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)" ); const createdAt = new Date().toISOString(); this.db.exec("BEGIN"); try { for (const message of messages) { insertMessage.run( message.id, message.author.id, message.author.username, message.timestamp, message.content ); for (const item of extractMedia(message)) { const result = insertMedia.run( item.kind, item.source, message.author.id, message.author.username, message.timestamp, item.url, item.playUrl, item.provider, item.contentType, item.size, item.width, item.height, aspectRatioOf(item.width, item.height), item.durationSecs, item.downloaded ? 1 : 0, item.previewUrl, createdAt ); if (result.changes > 0) added[item.kind]++; const row = findMedia.get(item.url) as { id: number } | undefined; if (row) linkSource.run(row.id, message.id); } } this.db.exec("COMMIT"); } catch (error) { this.db.exec("ROLLBACK"); throw error; } return added; } /** Bytes actually on disk per kind (downloaded files and cached previews). */ storedBytesByKind(): Record { const rows = this.db .prepare( "SELECT kind, COALESCE(SUM(COALESCE(file_bytes, 0) + COALESCE(preview_bytes, 0)), 0) AS bytes FROM media GROUP BY kind" ) .all() as unknown as Array<{ kind: string; bytes: number }>; const result = emptyMediaCounts(); for (const row of rows) { if (row.kind in result) result[row.kind as MediaKind] = row.bytes; } return result; } counts(): Record { const rows = this.db .prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind") .all() as unknown as Array<{ kind: string; n: number }>; const result = emptyMediaCounts(); for (const row of rows) { if (row.kind in result) result[row.kind as MediaKind] = row.n; } return result; } /** Media rows that still need bytes: never downloaded and not skipped. */ pendingDownloads(limit = 5000): MediaRow[] { return this.db .prepare( "SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?" ) .all(limit) as unknown as MediaRow[]; } markDownloaded( id: number, file: { path: string; hash: string; bytes: number } ): void { this.db .prepare( "UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?" ) .run(file.path, file.hash, file.bytes, id); } markDownloadError(id: number, error: string): void { this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id); } /** * Rows that still need a cached preview. Attachment/sticker images and GIFs always; * embed thumbnails only when `includeEmbeds` is set (download mode). */ pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] { const local = "((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))"; const condition = includeEmbeds ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))` : local; return this.db .prepare( `SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?` ) .all(limit) as unknown as MediaRow[]; } markPreview(id: number, preview: { hash: string; bytes: number; ext?: string; animHash?: string }): void { if (preview.animHash) { this.db .prepare( "UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, anim_hash = ?, preview_error = NULL WHERE id = ?" ) .run(preview.hash, preview.bytes, preview.ext ?? "webp", preview.animHash, id); return; } this.db .prepare( "UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?" ) .run(preview.hash, preview.bytes, preview.ext ?? "webp", id); } markPreviewError(id: number, error: string): void { this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id); } /** Refresh stored URLs after an expired CDN link is re-fetched. */ updateUrls(id: number, urls: { url?: string; playUrl?: string | null }): void { const sets: string[] = []; const values: Array = []; if (urls.url) { sets.push("url = ?"); values.push(urls.url); } if (urls.playUrl !== undefined) { sets.push("play_url = ?"); values.push(urls.playUrl); } if (!sets.length) return; this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id); } /** Rows whose preview failed, with the reason, newest first. */ previewFailures( limit = 50 ): Array<{ id: number; kind: MediaKind; provider: string | null; url: string; error: string }> { const rows = this.db .prepare( "SELECT id, kind, provider, url, preview_error FROM media WHERE preview_error IS NOT NULL ORDER BY id DESC LIMIT ?" ) .all(limit) as unknown as Array<{ id: number; kind: MediaKind; provider: string | null; url: string; preview_error: string; }>; return rows.map(row => ({ id: row.id, kind: row.kind, provider: row.provider, url: row.url, error: row.preview_error })); } /** The message a media row came from, for URL refreshes. */ sourceMessageId(mediaId: number): string | null { const row = this.db .prepare("SELECT message_id FROM media_sources WHERE media_id = ? LIMIT 1") .get(mediaId) as unknown as { message_id: string } | undefined; return row?.message_id ?? null; } /** Fill in dimensions and duration after a download probe. */ updateMediaInfo( id: number, info: { width?: number | null; height?: number | null; durationSecs?: number | null } ): void { const sets: string[] = []; const values: Array = []; if (info.width != null && info.height != null) { sets.push("width = ?", "height = ?", "aspect_ratio = ?"); values.push(info.width, info.height, aspectRatioOf(info.width, info.height)); } if (info.durationSecs != null) { sets.push("duration_secs = ?"); values.push(info.durationSecs); } if (!sets.length) return; this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id); } /** Distinct posters with their item counts, biggest first. */ authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> { const rows = this.db .prepare( "SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?" ) .all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>; return rows; } getMedia(id: number): MediaRow | null { const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as | MediaRow | undefined; return row ?? null; } private mediaWhere(options: MediaFilterOptions): { where: string; values: Array } { const conditions: string[] = []; const values: Array = []; if (options.kind) { conditions.push("kind = ?"); values.push(options.kind); } if (options.source) { conditions.push("source = ?"); values.push(options.source); } if (options.authorId) { conditions.push("author_id = ?"); values.push(options.authorId); } if (options.authorIds?.length) { conditions.push(`author_id IN (${options.authorIds.map(() => "?").join(", ")})`); values.push(...options.authorIds); } if (options.cachedOnly) { conditions.push("preview_hash IS NOT NULL"); } if (options.after) { conditions.push("posted_at >= ?"); values.push(options.after); } if (options.before) { conditions.push("posted_at <= ?"); values.push(options.before); } const aspect = options.aspect ? aspectCondition(options.aspect) : null; if (aspect) conditions.push(aspect); const resolution = options.resolution ? resolutionCondition(options.resolution) : null; if (resolution) conditions.push(resolution); return { where: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "", values }; } queryMedia(options: MediaFilterOptions = {}): MediaRow[] { const limit = Math.max(1, Math.min(options.limit ?? 50, 500)); const offset = Math.max(0, options.offset ?? 0); const { where, values } = this.mediaWhere(options); const rows = this.db .prepare(`SELECT * FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`) .all(...values, limit, offset); return rows as unknown as MediaRow[]; } }