527 lines
19 KiB
TypeScript
527 lines
19 KiB
TypeScript
/*
|
|
* Per-scrape storage: one directory, one SQLite index, one media folder.
|
|
* Layout and rules: docs/media.md.
|
|
*/
|
|
|
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { fileURLToPath } from "node:url";
|
|
import { extractMedia } from "./extract.js";
|
|
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
|
|
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
|
|
|
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
|
export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url));
|
|
|
|
export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string {
|
|
return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT;
|
|
}
|
|
|
|
export function scrapeDir(root: string, id: string): string {
|
|
return join(root, id);
|
|
}
|
|
|
|
export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void {
|
|
writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n");
|
|
}
|
|
|
|
export function readScrapeMeta(dir: string): ScrapeMeta | null {
|
|
try {
|
|
return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Every scrape on disk, newest first. */
|
|
export function listScrapes(root: string): ScrapeMeta[] {
|
|
let entries: string[];
|
|
try {
|
|
entries = readdirSync(root);
|
|
} catch {
|
|
return [];
|
|
}
|
|
|
|
const metas: ScrapeMeta[] = [];
|
|
for (const entry of entries) {
|
|
const meta = readScrapeMeta(join(root, entry));
|
|
if (meta) metas.push(meta);
|
|
}
|
|
|
|
return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
|
|
}
|
|
|
|
/** Combined size of a directory tree, used by the storage limit. */
|
|
export function dirSizeBytes(dir: string): number {
|
|
let entries;
|
|
try {
|
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
} catch {
|
|
return 0;
|
|
}
|
|
|
|
let total = 0;
|
|
for (const entry of entries) {
|
|
const path = join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
total += dirSizeBytes(path);
|
|
} else {
|
|
try {
|
|
total += statSync(path).size;
|
|
} catch {
|
|
// A file that vanished mid-walk counts as zero.
|
|
}
|
|
}
|
|
}
|
|
|
|
return total;
|
|
}
|
|
|
|
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
|
|
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
|
|
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
|
|
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
|
|
}
|
|
|
|
export function createScrapeDirs(root: string, id: string): string {
|
|
const dir = scrapeDir(root, id);
|
|
mkdirSync(join(dir, "media"), { recursive: true });
|
|
mkdirSync(join(dir, "thumbs"), { recursive: true });
|
|
return dir;
|
|
}
|
|
|
|
/** `<channelId>-<YYYYMMDDHHMMSS>`, with a numeric suffix on a same-second collision. */
|
|
export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string {
|
|
const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, "");
|
|
const base = `${channelId}-${stamp}`;
|
|
|
|
let id = base;
|
|
let suffix = 2;
|
|
while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`;
|
|
return id;
|
|
}
|
|
|
|
export interface MediaFilterOptions {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
cachedOnly?: boolean;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
}
|
|
|
|
export interface MediaRow {
|
|
id: number;
|
|
kind: MediaKind;
|
|
source: string | null;
|
|
author_id: string | null;
|
|
author_name: string | null;
|
|
posted_at: string | null;
|
|
url: string;
|
|
play_url: string | null;
|
|
provider: string | null;
|
|
content_type: string | null;
|
|
size: number | null;
|
|
width: number | null;
|
|
height: number | null;
|
|
aspect_ratio: number | null;
|
|
duration_secs: number | null;
|
|
downloaded: number;
|
|
preview_url: string | null;
|
|
file_hash: string | null;
|
|
preview_hash: string | null;
|
|
preview_ext: string | null;
|
|
anim_hash: string | null;
|
|
file_bytes: number | null;
|
|
preview_bytes: number | null;
|
|
file_path: string | null;
|
|
download_error: string | null;
|
|
preview_error: string | null;
|
|
created_at: string;
|
|
}
|
|
|
|
const SCHEMA = `
|
|
CREATE TABLE IF NOT EXISTS messages (
|
|
id TEXT PRIMARY KEY,
|
|
author_id TEXT NOT NULL,
|
|
author_name TEXT NOT NULL,
|
|
timestamp TEXT NOT NULL,
|
|
content TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS media (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL,
|
|
source TEXT,
|
|
author_id TEXT,
|
|
author_name TEXT,
|
|
posted_at TEXT,
|
|
url TEXT NOT NULL UNIQUE,
|
|
play_url TEXT,
|
|
provider TEXT,
|
|
content_type TEXT,
|
|
size INTEGER,
|
|
width INTEGER,
|
|
height INTEGER,
|
|
aspect_ratio REAL,
|
|
duration_secs REAL,
|
|
downloaded INTEGER NOT NULL DEFAULT 0,
|
|
preview_url TEXT,
|
|
file_hash TEXT,
|
|
preview_hash TEXT,
|
|
preview_ext TEXT,
|
|
anim_hash TEXT,
|
|
file_bytes INTEGER,
|
|
preview_bytes INTEGER,
|
|
file_path TEXT,
|
|
download_error TEXT,
|
|
preview_error TEXT,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS media_sources (
|
|
media_id INTEGER NOT NULL REFERENCES media(id),
|
|
message_id TEXT NOT NULL,
|
|
PRIMARY KEY (media_id, message_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
|
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
|
`;
|
|
|
|
/** One open SQLite index. The caller closes it. */
|
|
export class ScrapeStore {
|
|
private readonly db: DatabaseSync;
|
|
|
|
constructor(readonly dir: string) {
|
|
this.db = new DatabaseSync(join(dir, "index.db"));
|
|
this.db.exec(SCHEMA);
|
|
this.migrate();
|
|
}
|
|
|
|
/** Add columns introduced after a scrape was created. */
|
|
private migrate(): void {
|
|
const columns = this.db.prepare("PRAGMA table_info(media)").all() as unknown as Array<{ name: string }>;
|
|
const names = new Set(columns.map(column => column.name));
|
|
|
|
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
|
|
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
|
|
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
|
|
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
|
|
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
|
|
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
|
|
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
|
|
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
|
|
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
|
|
if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT");
|
|
if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT");
|
|
if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
|
|
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
|
|
}
|
|
|
|
close(): void {
|
|
this.db.close();
|
|
}
|
|
|
|
/** Write one page of messages in a transaction. Returns the newly inserted media per kind. */
|
|
insertPage(messages: CoreMessage[]): Record<MediaKind, number> {
|
|
const added = emptyMediaCounts();
|
|
const insertMessage = this.db.prepare(
|
|
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
|
);
|
|
const insertMedia = this.db.prepare(
|
|
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
);
|
|
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
|
const linkSource = this.db.prepare(
|
|
"INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)"
|
|
);
|
|
const createdAt = new Date().toISOString();
|
|
|
|
this.db.exec("BEGIN");
|
|
try {
|
|
for (const message of messages) {
|
|
insertMessage.run(
|
|
message.id,
|
|
message.author.id,
|
|
message.author.username,
|
|
message.timestamp,
|
|
message.content
|
|
);
|
|
|
|
for (const item of extractMedia(message)) {
|
|
const result = insertMedia.run(
|
|
item.kind,
|
|
item.source,
|
|
message.author.id,
|
|
message.author.username,
|
|
message.timestamp,
|
|
item.url,
|
|
item.playUrl,
|
|
item.provider,
|
|
item.contentType,
|
|
item.size,
|
|
item.width,
|
|
item.height,
|
|
aspectRatioOf(item.width, item.height),
|
|
item.durationSecs,
|
|
item.downloaded ? 1 : 0,
|
|
item.previewUrl,
|
|
createdAt
|
|
);
|
|
if (result.changes > 0) added[item.kind]++;
|
|
|
|
const row = findMedia.get(item.url) as { id: number } | undefined;
|
|
if (row) linkSource.run(row.id, message.id);
|
|
}
|
|
}
|
|
this.db.exec("COMMIT");
|
|
} catch (error) {
|
|
this.db.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
|
|
return added;
|
|
}
|
|
|
|
/** Bytes actually on disk per kind (downloaded files and cached previews). */
|
|
storedBytesByKind(): Record<MediaKind, number> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT kind, COALESCE(SUM(COALESCE(file_bytes, 0) + COALESCE(preview_bytes, 0)), 0) AS bytes FROM media GROUP BY kind"
|
|
)
|
|
.all() as unknown as Array<{ kind: string; bytes: number }>;
|
|
|
|
const result = emptyMediaCounts();
|
|
for (const row of rows) {
|
|
if (row.kind in result) result[row.kind as MediaKind] = row.bytes;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
counts(): Record<MediaKind, number> {
|
|
const rows = this.db
|
|
.prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")
|
|
.all() as unknown as Array<{ kind: string; n: number }>;
|
|
|
|
const result = emptyMediaCounts();
|
|
for (const row of rows) {
|
|
if (row.kind in result) result[row.kind as MediaKind] = row.n;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** Media rows that still need bytes: never downloaded and not skipped. */
|
|
pendingDownloads(limit = 5000): MediaRow[] {
|
|
return this.db
|
|
.prepare(
|
|
"SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
markDownloaded(
|
|
id: number,
|
|
file: { path: string; hash: string; bytes: number }
|
|
): void {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?"
|
|
)
|
|
.run(file.path, file.hash, file.bytes, id);
|
|
}
|
|
|
|
markDownloadError(id: number, error: string): void {
|
|
this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/**
|
|
* Rows that still need a cached preview. Attachment/sticker images and GIFs always;
|
|
* embed thumbnails only when `includeEmbeds` is set (download mode).
|
|
*/
|
|
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
|
|
const local =
|
|
"((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))";
|
|
const condition = includeEmbeds
|
|
? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
|
|
: local;
|
|
|
|
return this.db
|
|
.prepare(
|
|
`SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
|
|
)
|
|
.all(limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
markPreview(id: number, preview: { hash: string; bytes: number; ext?: string; animHash?: string }): void {
|
|
if (preview.animHash) {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, anim_hash = ?, preview_error = NULL WHERE id = ?"
|
|
)
|
|
.run(preview.hash, preview.bytes, preview.ext ?? "webp", preview.animHash, id);
|
|
return;
|
|
}
|
|
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?"
|
|
)
|
|
.run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
|
|
}
|
|
|
|
markPreviewError(id: number, error: string): void {
|
|
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/** Refresh stored URLs after an expired CDN link is re-fetched. */
|
|
updateUrls(id: number, urls: { url?: string; playUrl?: string | null }): void {
|
|
const sets: string[] = [];
|
|
const values: Array<string | null> = [];
|
|
|
|
if (urls.url) {
|
|
sets.push("url = ?");
|
|
values.push(urls.url);
|
|
}
|
|
if (urls.playUrl !== undefined) {
|
|
sets.push("play_url = ?");
|
|
values.push(urls.playUrl);
|
|
}
|
|
|
|
if (!sets.length) return;
|
|
this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
|
|
}
|
|
|
|
/** Rows whose preview failed, with the reason, newest first. */
|
|
previewFailures(
|
|
limit = 50
|
|
): Array<{ id: number; kind: MediaKind; provider: string | null; url: string; error: string }> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT id, kind, provider, url, preview_error FROM media WHERE preview_error IS NOT NULL ORDER BY id DESC LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as Array<{
|
|
id: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
preview_error: string;
|
|
}>;
|
|
|
|
return rows.map(row => ({
|
|
id: row.id,
|
|
kind: row.kind,
|
|
provider: row.provider,
|
|
url: row.url,
|
|
error: row.preview_error
|
|
}));
|
|
}
|
|
|
|
/** The message a media row came from, for URL refreshes. */
|
|
sourceMessageId(mediaId: number): string | null {
|
|
const row = this.db
|
|
.prepare("SELECT message_id FROM media_sources WHERE media_id = ? LIMIT 1")
|
|
.get(mediaId) as unknown as { message_id: string } | undefined;
|
|
return row?.message_id ?? null;
|
|
}
|
|
|
|
/** Fill in dimensions and duration after a download probe. */
|
|
updateMediaInfo(
|
|
id: number,
|
|
info: { width?: number | null; height?: number | null; durationSecs?: number | null }
|
|
): void {
|
|
const sets: string[] = [];
|
|
const values: Array<number | null> = [];
|
|
|
|
if (info.width != null && info.height != null) {
|
|
sets.push("width = ?", "height = ?", "aspect_ratio = ?");
|
|
values.push(info.width, info.height, aspectRatioOf(info.width, info.height));
|
|
}
|
|
if (info.durationSecs != null) {
|
|
sets.push("duration_secs = ?");
|
|
values.push(info.durationSecs);
|
|
}
|
|
|
|
if (!sets.length) return;
|
|
this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
|
|
}
|
|
|
|
/** Distinct posters with their item counts, biggest first. */
|
|
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
|
|
return rows;
|
|
}
|
|
|
|
getMedia(id: number): MediaRow | null {
|
|
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
|
|
| MediaRow
|
|
| undefined;
|
|
return row ?? null;
|
|
}
|
|
|
|
private mediaWhere(options: MediaFilterOptions): { where: string; values: Array<string | number> } {
|
|
const conditions: string[] = [];
|
|
const values: Array<string | number> = [];
|
|
|
|
if (options.kind) {
|
|
conditions.push("kind = ?");
|
|
values.push(options.kind);
|
|
}
|
|
if (options.source) {
|
|
conditions.push("source = ?");
|
|
values.push(options.source);
|
|
}
|
|
if (options.authorId) {
|
|
conditions.push("author_id = ?");
|
|
values.push(options.authorId);
|
|
}
|
|
if (options.authorIds?.length) {
|
|
conditions.push(`author_id IN (${options.authorIds.map(() => "?").join(", ")})`);
|
|
values.push(...options.authorIds);
|
|
}
|
|
if (options.cachedOnly) {
|
|
conditions.push("preview_hash IS NOT NULL");
|
|
}
|
|
if (options.after) {
|
|
conditions.push("posted_at >= ?");
|
|
values.push(options.after);
|
|
}
|
|
if (options.before) {
|
|
conditions.push("posted_at <= ?");
|
|
values.push(options.before);
|
|
}
|
|
|
|
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
|
|
if (aspect) conditions.push(aspect);
|
|
|
|
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
|
|
if (resolution) conditions.push(resolution);
|
|
|
|
return { where: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "", values };
|
|
}
|
|
|
|
queryMedia(options: MediaFilterOptions = {}): MediaRow[] {
|
|
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
|
const offset = Math.max(0, options.offset ?? 0);
|
|
const { where, values } = this.mediaWhere(options);
|
|
|
|
const rows = this.db
|
|
.prepare(`SELECT * FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`)
|
|
.all(...values, limit, offset);
|
|
|
|
return rows as unknown as MediaRow[];
|
|
}
|
|
|
|
}
|