1050 lines
41 KiB
TypeScript
1050 lines
41 KiB
TypeScript
/*
|
|
* Per-scrape storage: one directory, one SQLite index, one media folder.
|
|
* Layout and rules: docs/media.md.
|
|
*/
|
|
|
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { fileURLToPath } from "node:url";
|
|
import { extractMedia, normalizeYouTubeUrl } from "./extract.js";
|
|
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
|
|
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
|
|
|
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
|
export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url));
|
|
|
|
export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string {
|
|
return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT;
|
|
}
|
|
|
|
export function scrapeDir(root: string, id: string): string {
|
|
return join(root, id);
|
|
}
|
|
|
|
export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void {
|
|
writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n");
|
|
}
|
|
|
|
export function readScrapeMeta(dir: string): ScrapeMeta | null {
|
|
try {
|
|
return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Every scrape on disk, newest first. */
|
|
export function listScrapes(root: string): ScrapeMeta[] {
|
|
let entries: string[];
|
|
try {
|
|
entries = readdirSync(root);
|
|
} catch {
|
|
return [];
|
|
}
|
|
|
|
const metas: ScrapeMeta[] = [];
|
|
for (const entry of entries) {
|
|
const meta = readScrapeMeta(join(root, entry));
|
|
if (meta) metas.push(meta);
|
|
}
|
|
|
|
return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
|
|
}
|
|
|
|
/** Combined size of a directory tree, used by the storage limit. */
|
|
export function dirSizeBytes(dir: string): number {
|
|
let entries;
|
|
try {
|
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
} catch {
|
|
return 0;
|
|
}
|
|
|
|
let total = 0;
|
|
for (const entry of entries) {
|
|
const path = join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
total += dirSizeBytes(path);
|
|
} else {
|
|
try {
|
|
total += statSync(path).size;
|
|
} catch {
|
|
// A file that vanished mid-walk counts as zero.
|
|
}
|
|
}
|
|
}
|
|
|
|
return total;
|
|
}
|
|
|
|
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
|
|
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
|
|
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
|
|
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
|
|
}
|
|
|
|
export function createScrapeDirs(root: string, id: string): string {
|
|
const dir = scrapeDir(root, id);
|
|
mkdirSync(join(dir, "media"), { recursive: true });
|
|
mkdirSync(join(dir, "thumbs"), { recursive: true });
|
|
return dir;
|
|
}
|
|
|
|
/** `<channelId>-<YYYYMMDDHHMMSS>`, with a numeric suffix on a same-second collision. */
|
|
export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string {
|
|
const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, "");
|
|
const base = `${channelId}-${stamp}`;
|
|
|
|
let id = base;
|
|
let suffix = 2;
|
|
while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`;
|
|
return id;
|
|
}
|
|
|
|
/** Global ordering applied once candidates from every scrape are merged. */
|
|
export type MediaSort = "newest" | "oldest" | "user" | "size" | "length";
|
|
|
|
export interface MediaFilterOptions {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
cachedOnly?: boolean;
|
|
exts?: string[];
|
|
excludeHosts?: string[];
|
|
/** URL extension or content type to hide. */
|
|
excludeExts?: string[];
|
|
/** Provider hostnames to hide (youtube, twitch, ...). */
|
|
excludeProviders?: string[];
|
|
/** Row sources to hide (attachment, sticker, embed). */
|
|
excludeSources?: string[];
|
|
/** Keep only providers matching one of these substrings; empty keeps all. */
|
|
includeProviders?: string[];
|
|
/** Keep only these row sources; empty keeps all. */
|
|
includeSources?: string[];
|
|
/** Bytes; rows with no known size drop out while this is set. */
|
|
sizeLessThan?: number;
|
|
sizeGreaterThan?: number;
|
|
/** Seconds; rows with no known duration drop out while this is set. */
|
|
lengthLessThan?: number;
|
|
lengthGreaterThan?: number;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
sort?: MediaSort;
|
|
/** Title search: multi-word prefix tokens by default, a phrase when `titleExact`. */
|
|
titleQuery?: string;
|
|
titleExact?: boolean;
|
|
}
|
|
|
|
export interface MediaRow {
|
|
id: number;
|
|
kind: MediaKind;
|
|
source: string | null;
|
|
author_id: string | null;
|
|
author_name: string | null;
|
|
posted_at: string | null;
|
|
url: string;
|
|
play_url: string | null;
|
|
provider: string | null;
|
|
content_type: string | null;
|
|
size: number | null;
|
|
width: number | null;
|
|
height: number | null;
|
|
aspect_ratio: number | null;
|
|
duration_secs: number | null;
|
|
downloaded: number;
|
|
preview_url: string | null;
|
|
file_hash: string | null;
|
|
preview_hash: string | null;
|
|
preview_ext: string | null;
|
|
anim_hash: string | null;
|
|
file_bytes: number | null;
|
|
preview_bytes: number | null;
|
|
file_path: string | null;
|
|
download_error: string | null;
|
|
preview_error: string | null;
|
|
preview_attempts: number;
|
|
/** First message that carried this media; null on rows written before linking. */
|
|
message_id: string | null;
|
|
/** Embed title from Discord (YouTube and friends). */
|
|
title: string | null;
|
|
title_attempts: number;
|
|
title_error: string | null;
|
|
/** YouTube channel name from oEmbed, kept even if the video later disappears. */
|
|
title_author: string | null;
|
|
/** Last suspect (possibly throttled) preview failure; retried once, then left alone. */
|
|
preview_suspect: string | null;
|
|
/** Epoch ms when the suspect row may be retried. Null once confirmed unavailable. */
|
|
preview_retry_at: number | null;
|
|
/** Lowercased, punctuation-collapsed title for token-prefix search. */
|
|
title_key: string | null;
|
|
created_at: string;
|
|
}
|
|
|
|
/** Normalize a title for prefix-token search: lowercase, punctuation to spaces, collapsed. */
|
|
export function normalizeTitleKey(title: string): string {
|
|
return title
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, " ")
|
|
.trim()
|
|
.replace(/\s+/g, " ");
|
|
}
|
|
|
|
/** Lowercase a title query and collapse punctuation, but keep `*` as a wildcard marker. */
|
|
export function normalizeTitleQuery(query: string): string {
|
|
return query
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9*]+/g, " ")
|
|
.trim()
|
|
.replace(/\s+/g, " ");
|
|
}
|
|
|
|
/** Escape LIKE metacharacters, then turn `*` into `%`. */
|
|
function titleLikePattern(value: string): string {
|
|
return value.replace(/[\\%_]/g, match => `\\${match}`).replace(/\*/g, "%");
|
|
}
|
|
|
|
const SCHEMA = `
|
|
CREATE TABLE IF NOT EXISTS messages (
|
|
id TEXT PRIMARY KEY,
|
|
author_id TEXT NOT NULL,
|
|
author_name TEXT NOT NULL,
|
|
timestamp TEXT NOT NULL,
|
|
content TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS media (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL,
|
|
source TEXT,
|
|
author_id TEXT,
|
|
author_name TEXT,
|
|
posted_at TEXT,
|
|
url TEXT NOT NULL UNIQUE,
|
|
play_url TEXT,
|
|
provider TEXT,
|
|
content_type TEXT,
|
|
size INTEGER,
|
|
width INTEGER,
|
|
height INTEGER,
|
|
aspect_ratio REAL,
|
|
duration_secs REAL,
|
|
downloaded INTEGER NOT NULL DEFAULT 0,
|
|
preview_url TEXT,
|
|
file_hash TEXT,
|
|
preview_hash TEXT,
|
|
preview_ext TEXT,
|
|
anim_hash TEXT,
|
|
file_bytes INTEGER,
|
|
preview_bytes INTEGER,
|
|
file_path TEXT,
|
|
download_error TEXT,
|
|
preview_error TEXT,
|
|
preview_attempts INTEGER NOT NULL DEFAULT 0,
|
|
title TEXT,
|
|
title_key TEXT,
|
|
title_attempts INTEGER NOT NULL DEFAULT 0,
|
|
title_error TEXT,
|
|
title_author TEXT,
|
|
title_author_checked INTEGER NOT NULL DEFAULT 0,
|
|
preview_suspect TEXT,
|
|
preview_retry_at INTEGER,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS media_sources (
|
|
media_id INTEGER NOT NULL REFERENCES media(id),
|
|
message_id TEXT NOT NULL,
|
|
PRIMARY KEY (media_id, message_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
|
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
|
`;
|
|
|
|
/** One open SQLite index. The caller closes it. */
|
|
export class ScrapeStore {
|
|
private readonly db: DatabaseSync;
|
|
|
|
constructor(readonly dir: string) {
|
|
this.db = new DatabaseSync(join(dir, "index.db"));
|
|
this.db.exec(SCHEMA);
|
|
this.migrate();
|
|
}
|
|
|
|
/** Add columns introduced after a scrape was created. */
|
|
private migrate(): void {
|
|
const columns = this.db.prepare("PRAGMA table_info(media)").all() as unknown as Array<{ name: string }>;
|
|
const names = new Set(columns.map(column => column.name));
|
|
|
|
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
|
|
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
|
|
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
|
|
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
|
|
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
|
|
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
|
|
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
|
|
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
|
|
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
|
|
if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT");
|
|
if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT");
|
|
if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
|
|
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
|
|
if (!names.has("preview_attempts")) {
|
|
this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0");
|
|
}
|
|
if (!names.has("title")) {
|
|
this.db.exec("ALTER TABLE media ADD COLUMN title TEXT");
|
|
this.db.exec("ALTER TABLE media ADD COLUMN title_key TEXT");
|
|
}
|
|
if (!names.has("title_attempts")) {
|
|
this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0");
|
|
this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
|
|
}
|
|
if (!names.has("title_author")) this.db.exec("ALTER TABLE media ADD COLUMN title_author TEXT");
|
|
if (!names.has("title_author_checked")) {
|
|
this.db.exec("ALTER TABLE media ADD COLUMN title_author_checked INTEGER NOT NULL DEFAULT 0");
|
|
}
|
|
if (!names.has("preview_suspect")) {
|
|
this.db.exec("ALTER TABLE media ADD COLUMN preview_suspect TEXT");
|
|
this.db.exec("ALTER TABLE media ADD COLUMN preview_retry_at INTEGER");
|
|
}
|
|
|
|
this.migrateYouTubeShorts();
|
|
}
|
|
|
|
/**
|
|
* One-time rewrite of youtube.com/shorts links to watch URLs. The embed player needs `v=`,
|
|
* so old rows with a shorts URL would never play. `PRAGMA user_version` makes it idempotent
|
|
* per scrape database.
|
|
*/
|
|
private migrateYouTubeShorts(): void {
|
|
const version = this.db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined;
|
|
if ((version?.user_version ?? 0) >= 1) return;
|
|
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT * FROM media WHERE url LIKE '%youtube.com/shorts/%' OR play_url LIKE '%youtube.com/shorts/%'"
|
|
)
|
|
.all() as unknown as MediaRow[];
|
|
|
|
if (rows.length) {
|
|
const update = this.db.prepare("UPDATE media SET url = ?, play_url = ? WHERE id = ?");
|
|
const findTarget = this.db.prepare("SELECT id FROM media WHERE url = ? AND id != ?");
|
|
const merge = this.db.prepare(
|
|
`UPDATE media SET
|
|
preview_hash = COALESCE(preview_hash, ?),
|
|
preview_bytes = COALESCE(preview_bytes, ?),
|
|
preview_ext = COALESCE(preview_ext, ?),
|
|
file_path = COALESCE(file_path, ?),
|
|
file_hash = COALESCE(file_hash, ?),
|
|
file_bytes = COALESCE(file_bytes, ?),
|
|
downloaded = MAX(downloaded, ?),
|
|
title = COALESCE(title, ?),
|
|
title_key = COALESCE(title_key, ?),
|
|
play_url = COALESCE(play_url, ?)
|
|
WHERE id = ?`
|
|
);
|
|
const relink = this.db.prepare(
|
|
"UPDATE OR IGNORE media_sources SET media_id = ? WHERE media_id = ?"
|
|
);
|
|
const dropLinks = this.db.prepare("DELETE FROM media_sources WHERE media_id = ?");
|
|
const drop = this.db.prepare("DELETE FROM media WHERE id = ?");
|
|
|
|
this.db.exec("BEGIN");
|
|
try {
|
|
for (const row of rows) {
|
|
const url = normalizeYouTubeUrl(row.url);
|
|
const playUrl = row.play_url ? normalizeYouTubeUrl(row.play_url) : null;
|
|
const target = findTarget.get(url, row.id) as { id: number } | undefined;
|
|
|
|
// The same video was already indexed under its watch URL: fold this row in.
|
|
if (!target) {
|
|
update.run(url, playUrl, row.id);
|
|
continue;
|
|
}
|
|
|
|
merge.run(
|
|
row.preview_hash,
|
|
row.preview_bytes,
|
|
row.preview_ext,
|
|
row.file_path,
|
|
row.file_hash,
|
|
row.file_bytes,
|
|
row.downloaded,
|
|
row.title,
|
|
row.title_key,
|
|
playUrl,
|
|
target.id
|
|
);
|
|
relink.run(target.id, row.id);
|
|
dropLinks.run(row.id);
|
|
drop.run(row.id);
|
|
}
|
|
this.db.exec("COMMIT");
|
|
} catch (error) {
|
|
this.db.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
this.db.exec("PRAGMA user_version = 1");
|
|
}
|
|
|
|
close(): void {
|
|
this.db.close();
|
|
}
|
|
|
|
/** Write one page of messages in a transaction. Returns the newly inserted media per kind. */
|
|
insertPage(messages: CoreMessage[]): Record<MediaKind, number> {
|
|
const added = emptyMediaCounts();
|
|
const insertMessage = this.db.prepare(
|
|
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
|
);
|
|
const insertMedia = this.db.prepare(
|
|
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, title_author, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
);
|
|
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
|
const linkSource = this.db.prepare(
|
|
"INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)"
|
|
);
|
|
const createdAt = new Date().toISOString();
|
|
|
|
this.db.exec("BEGIN");
|
|
try {
|
|
for (const message of messages) {
|
|
insertMessage.run(
|
|
message.id,
|
|
message.author.id,
|
|
message.author.username,
|
|
message.timestamp,
|
|
message.content
|
|
);
|
|
|
|
for (const item of extractMedia(message)) {
|
|
const result = insertMedia.run(
|
|
item.kind,
|
|
item.source,
|
|
message.author.id,
|
|
message.author.username,
|
|
message.timestamp,
|
|
item.url,
|
|
item.playUrl,
|
|
item.provider,
|
|
item.contentType,
|
|
item.size,
|
|
item.width,
|
|
item.height,
|
|
aspectRatioOf(item.width, item.height),
|
|
item.durationSecs,
|
|
item.downloaded ? 1 : 0,
|
|
item.previewUrl,
|
|
item.title,
|
|
item.title ? normalizeTitleKey(item.title) : null,
|
|
item.titleAuthor,
|
|
createdAt
|
|
);
|
|
if (result.changes > 0) added[item.kind]++;
|
|
|
|
const row = findMedia.get(item.url) as { id: number } | undefined;
|
|
if (row) linkSource.run(row.id, message.id);
|
|
}
|
|
}
|
|
this.db.exec("COMMIT");
|
|
} catch (error) {
|
|
this.db.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
|
|
return added;
|
|
}
|
|
|
|
/** Bytes actually on disk per kind (downloaded files and cached previews). */
|
|
storedBytesByKind(): Record<MediaKind, number> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT kind, COALESCE(SUM(COALESCE(file_bytes, 0) + COALESCE(preview_bytes, 0)), 0) AS bytes FROM media GROUP BY kind"
|
|
)
|
|
.all() as unknown as Array<{ kind: string; bytes: number }>;
|
|
|
|
const result = emptyMediaCounts();
|
|
for (const row of rows) {
|
|
if (row.kind in result) result[row.kind as MediaKind] = row.bytes;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
counts(): Record<MediaKind, number> {
|
|
const rows = this.db
|
|
.prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")
|
|
.all() as unknown as Array<{ kind: string; n: number }>;
|
|
|
|
const result = emptyMediaCounts();
|
|
for (const row of rows) {
|
|
if (row.kind in result) result[row.kind as MediaKind] = row.n;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** Media rows that still need bytes: never downloaded and not skipped. */
|
|
pendingDownloads(limit = 5000): MediaRow[] {
|
|
return this.db
|
|
.prepare(
|
|
"SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
markDownloaded(
|
|
id: number,
|
|
file: { path: string; hash: string; bytes: number }
|
|
): void {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?"
|
|
)
|
|
.run(file.path, file.hash, file.bytes, id);
|
|
}
|
|
|
|
markDownloadError(id: number, error: string): void {
|
|
this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/**
|
|
* Rows that still need a cached preview. Attachment/sticker images and GIFs always;
|
|
* embed thumbnails only when `includeEmbeds` is set (download mode).
|
|
*/
|
|
/**
|
|
* Rows whose preview failed in a way that may be throttling rather than a dead link. Each is
|
|
* retried once after a delay; confirmed rows stay here and are skipped by normal passes, so a
|
|
* finished backfill never churns over genuinely unavailable media.
|
|
*/
|
|
dueSuspects(limit = 500, nowMs = Date.now()): MediaRow[] {
|
|
return this.db
|
|
.prepare(
|
|
`SELECT * FROM media
|
|
WHERE preview_hash IS NULL AND preview_error IS NULL
|
|
AND preview_suspect IS NOT NULL
|
|
AND preview_retry_at IS NOT NULL
|
|
AND preview_retry_at <= ?
|
|
ORDER BY preview_retry_at LIMIT ?`
|
|
)
|
|
.all(nowMs, limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
/** Suspect rows that still have a retry coming. */
|
|
suspectCount(): number {
|
|
const row = this.db
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_suspect IS NOT NULL AND preview_retry_at IS NOT NULL"
|
|
)
|
|
.get() as { count: number | null } | undefined;
|
|
return row?.count ?? 0;
|
|
}
|
|
|
|
/** Suspect rows whose single retry failed too; skipped by normal passes. */
|
|
unverifiedCount(): number {
|
|
const row = this.db
|
|
.prepare(
|
|
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_suspect IS NOT NULL AND preview_retry_at IS NULL"
|
|
)
|
|
.get() as { count: number | null } | undefined;
|
|
return row?.count ?? 0;
|
|
}
|
|
|
|
/** Schedule the single delayed retry for a suspect row. */
|
|
markSuspect(id: number, error: string, retryAtMs: number): void {
|
|
this.db
|
|
.prepare("UPDATE media SET preview_suspect = ?, preview_retry_at = ? WHERE id = ?")
|
|
.run(error.slice(0, 200), retryAtMs, id);
|
|
}
|
|
|
|
/** The delayed retry failed too: keep the note but stop retrying. */
|
|
confirmSuspect(id: number, error: string): void {
|
|
this.db
|
|
.prepare("UPDATE media SET preview_suspect = ?, preview_retry_at = NULL WHERE id = ?")
|
|
.run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/** The retry succeeded or the preview arrived another way. */
|
|
clearSuspect(id: number): void {
|
|
this.db.prepare("UPDATE media SET preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?").run(id);
|
|
}
|
|
|
|
/**
|
|
* Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows
|
|
* whose single retry already failed (they may have been hit by the same stale-URL bug).
|
|
*/
|
|
discordPreviewFailures(limit = 5000, nowMs = Date.now()): MediaRow[] {
|
|
return this.db
|
|
.prepare(
|
|
`SELECT * FROM media
|
|
WHERE preview_hash IS NULL
|
|
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
|
|
AND ((preview_error LIKE 'http %'
|
|
AND preview_error NOT LIKE 'unavailable after recovery%')
|
|
OR (preview_suspect IS NOT NULL
|
|
AND (preview_retry_at IS NULL OR preview_retry_at <= ?)))
|
|
ORDER BY id LIMIT ?`
|
|
)
|
|
.all(nowMs, limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
clearPreviewError(id: number): void {
|
|
this.db.prepare("UPDATE media SET preview_error = NULL WHERE id = ?").run(id);
|
|
}
|
|
|
|
/** The recovery task refreshed and retried this row; it is genuinely unavailable now. */
|
|
markUnavailableAfterRecovery(id: number): void {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET preview_error = 'unavailable after recovery', preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?"
|
|
)
|
|
.run(id);
|
|
}
|
|
|
|
discordPreviewFailureCount(): number {
|
|
const row = this.db
|
|
.prepare(
|
|
`SELECT COUNT(*) AS count FROM media
|
|
WHERE preview_hash IS NULL
|
|
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
|
|
AND ((preview_error LIKE 'http %'
|
|
AND preview_error NOT LIKE 'unavailable after recovery%')
|
|
OR preview_suspect IS NOT NULL)`
|
|
)
|
|
.get() as { count: number | null } | undefined;
|
|
return row?.count ?? 0;
|
|
}
|
|
|
|
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
|
|
const local =
|
|
"((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))";
|
|
// With embeds on, a thumbnail-less embed still needs a pass: the engine resolves one
|
|
// from the provider or the page (or the Discord proxy for CDN attachment embeds).
|
|
const condition = includeEmbeds ? `(${local} OR source = 'embed')` : local;
|
|
|
|
return this.db
|
|
.prepare(
|
|
`SELECT * FROM media
|
|
WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL
|
|
AND preview_suspect IS NULL
|
|
ORDER BY id LIMIT ?`
|
|
)
|
|
.all(limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
markPreview(id: number, preview: { hash: string; bytes: number; ext?: string; animHash?: string }): void {
|
|
if (preview.animHash) {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, anim_hash = ?, preview_error = NULL WHERE id = ?"
|
|
)
|
|
.run(preview.hash, preview.bytes, preview.ext ?? "webp", preview.animHash, id);
|
|
return;
|
|
}
|
|
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?"
|
|
)
|
|
.run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
|
|
}
|
|
|
|
/** Message span of the walk, for completeness checks and incremental scrapes. */
|
|
dbSummary(): {
|
|
messages: number;
|
|
oldest: string | null;
|
|
newest: string | null;
|
|
newestId: string | null;
|
|
} {
|
|
const row = this.db
|
|
.prepare(
|
|
"SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest, MAX(id) AS newestId FROM messages"
|
|
)
|
|
.get() as
|
|
| { messages: number; oldest: string | null; newest: string | null; newestId: string | null }
|
|
| undefined;
|
|
|
|
return {
|
|
messages: row?.messages ?? 0,
|
|
oldest: row?.oldest ?? null,
|
|
newest: row?.newest ?? null,
|
|
newestId: row?.newestId ?? null
|
|
};
|
|
}
|
|
|
|
/** Preview state for the whole scrape and per kind, for the backfill UI. */
|
|
previewCounts(): {
|
|
total: number;
|
|
pending: number;
|
|
cached: number;
|
|
failed: number;
|
|
/** YouTube-like rows still missing a title. */
|
|
titlePending: number;
|
|
/** Rows with a stored title. */
|
|
titled: number;
|
|
/** Rows closed out without a title. */
|
|
titleFailed: number;
|
|
/** Preview failures kept for one delayed retry. */
|
|
suspect: number;
|
|
/** Suspect rows whose retry failed too. */
|
|
unverified: number;
|
|
byKind: Record<MediaKind, { total: number; pending: number; cached: number; failed: number }>;
|
|
} {
|
|
const empty = () => ({ total: 0, pending: 0, cached: 0, failed: 0 });
|
|
const byKind: Record<MediaKind, { total: number; pending: number; cached: number; failed: number }> = {
|
|
image: empty(),
|
|
gif: empty(),
|
|
video: empty(),
|
|
audio: empty()
|
|
};
|
|
|
|
const rows = this.db
|
|
.prepare(
|
|
`SELECT kind,
|
|
COUNT(*) AS total,
|
|
SUM(preview_hash IS NOT NULL) AS cached,
|
|
SUM(preview_error IS NOT NULL) AS failed,
|
|
SUM(preview_hash IS NULL AND preview_error IS NULL) AS pending
|
|
FROM media
|
|
GROUP BY kind`
|
|
)
|
|
.all() as unknown as Array<{
|
|
kind: MediaKind;
|
|
total: number;
|
|
cached: number;
|
|
failed: number;
|
|
pending: number;
|
|
}>;
|
|
|
|
const totals = empty();
|
|
for (const row of rows) {
|
|
const bucket = byKind[row.kind] ?? empty();
|
|
bucket.total += row.total;
|
|
bucket.pending += row.pending ?? 0;
|
|
bucket.cached += row.cached ?? 0;
|
|
bucket.failed += row.failed ?? 0;
|
|
byKind[row.kind] = bucket;
|
|
|
|
totals.total += row.total;
|
|
totals.pending += row.pending ?? 0;
|
|
totals.cached += row.cached ?? 0;
|
|
totals.failed += row.failed ?? 0;
|
|
}
|
|
|
|
const titles = this.db
|
|
.prepare(
|
|
`SELECT
|
|
SUM(title_error IS NULL
|
|
AND (title IS NULL OR (title_author IS NULL AND title_author_checked = 0))
|
|
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
|
|
OR lower(url) LIKE '%youtube.com/%'
|
|
OR lower(url) LIKE '%youtu.be/%')) AS pending,
|
|
SUM(title IS NOT NULL) AS titled,
|
|
SUM(title IS NULL AND title_error IS NOT NULL) AS failed
|
|
FROM media`
|
|
)
|
|
.get() as { pending: number | null; titled: number | null; failed: number | null } | undefined;
|
|
|
|
return {
|
|
...totals,
|
|
titlePending: titles?.pending ?? 0,
|
|
titled: titles?.titled ?? 0,
|
|
titleFailed: titles?.failed ?? 0,
|
|
suspect: this.suspectCount(),
|
|
unverified: this.unverifiedCount(),
|
|
byKind
|
|
};
|
|
}
|
|
|
|
/** Count one transient preview failure and return the new total. */
|
|
/**
|
|
* YouTube-like rows whose title Discord did not provide, oldest first. Resolved through the
|
|
* provider oEmbed in the backfill queue.
|
|
*/
|
|
pendingTitles(limit = 500): MediaRow[] {
|
|
// Missing title, or a title whose channel was never fetched (one attempt per row).
|
|
return this.db
|
|
.prepare(
|
|
`SELECT * FROM media
|
|
WHERE title_error IS NULL
|
|
AND (title IS NULL OR (title_author IS NULL AND title_author_checked = 0))
|
|
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
|
|
OR lower(url) LIKE '%youtube.com/%'
|
|
OR lower(url) LIKE '%youtu.be/%')
|
|
ORDER BY id LIMIT ?`
|
|
)
|
|
.all(limit) as unknown as MediaRow[];
|
|
}
|
|
|
|
/** Store a resolved title, its search key, and the provider's channel name. */
|
|
markTitle(id: number, title: string, author?: string | null): void {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE media SET title = ?, title_key = ?, title_error = NULL, title_author = COALESCE(?, title_author), title_author_checked = 1 WHERE id = ?"
|
|
)
|
|
.run(title.slice(0, 300), normalizeTitleKey(title), author ?? null, id);
|
|
}
|
|
|
|
/** Count one transient title failure and return the new total. */
|
|
bumpTitleAttempt(id: number): number {
|
|
this.db.prepare("UPDATE media SET title_attempts = title_attempts + 1 WHERE id = ?").run(id);
|
|
const row = this.db.prepare("SELECT title_attempts FROM media WHERE id = ?").get(id) as
|
|
| { title_attempts: number | null }
|
|
| undefined;
|
|
|
|
return row?.title_attempts ?? 0;
|
|
}
|
|
|
|
markTitleError(id: number, error: string): void {
|
|
this.db
|
|
.prepare("UPDATE media SET title_error = ?, title_author_checked = 1 WHERE id = ?")
|
|
.run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/** Count one transient preview failure and return the new total. */
|
|
bumpPreviewAttempt(id: number): number {
|
|
this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id);
|
|
const row = this.db.prepare("SELECT preview_attempts FROM media WHERE id = ?").get(id) as
|
|
| { preview_attempts: number | null }
|
|
| undefined;
|
|
|
|
return row?.preview_attempts ?? 0;
|
|
}
|
|
|
|
markPreviewError(id: number, error: string): void {
|
|
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
|
}
|
|
|
|
/** Refresh stored URLs after an expired CDN link is re-fetched. */
|
|
updateUrls(id: number, urls: { url?: string; playUrl?: string | null }): void {
|
|
const sets: string[] = [];
|
|
const values: Array<string | null> = [];
|
|
|
|
if (urls.url) {
|
|
sets.push("url = ?");
|
|
values.push(urls.url);
|
|
}
|
|
if (urls.playUrl !== undefined) {
|
|
sets.push("play_url = ?");
|
|
values.push(urls.playUrl);
|
|
}
|
|
|
|
if (!sets.length) return;
|
|
this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
|
|
}
|
|
|
|
/** Rows whose preview failed, with the reason, newest first. */
|
|
previewFailures(
|
|
limit = 50
|
|
): Array<{ id: number; kind: MediaKind; provider: string | null; url: string; error: string }> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT id, kind, provider, url, preview_error FROM media WHERE preview_error IS NOT NULL ORDER BY id DESC LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as Array<{
|
|
id: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
preview_error: string;
|
|
}>;
|
|
|
|
return rows.map(row => ({
|
|
id: row.id,
|
|
kind: row.kind,
|
|
provider: row.provider,
|
|
url: row.url,
|
|
error: row.preview_error
|
|
}));
|
|
}
|
|
|
|
/** The message a media row came from, for URL refreshes. */
|
|
sourceMessageId(mediaId: number): string | null {
|
|
const row = this.db
|
|
.prepare("SELECT message_id FROM media_sources WHERE media_id = ? LIMIT 1")
|
|
.get(mediaId) as unknown as { message_id: string } | undefined;
|
|
return row?.message_id ?? null;
|
|
}
|
|
|
|
/** Fill in dimensions and duration after a download probe. */
|
|
updateMediaInfo(
|
|
id: number,
|
|
info: { width?: number | null; height?: number | null; durationSecs?: number | null }
|
|
): void {
|
|
const sets: string[] = [];
|
|
const values: Array<number | null> = [];
|
|
|
|
if (info.width != null && info.height != null) {
|
|
sets.push("width = ?", "height = ?", "aspect_ratio = ?");
|
|
values.push(info.width, info.height, aspectRatioOf(info.width, info.height));
|
|
}
|
|
if (info.durationSecs != null) {
|
|
sets.push("duration_secs = ?");
|
|
values.push(info.durationSecs);
|
|
}
|
|
|
|
if (!sets.length) return;
|
|
this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
|
|
}
|
|
|
|
/** Distinct posters with their item counts, biggest first. */
|
|
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
|
|
const rows = this.db
|
|
.prepare(
|
|
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
|
|
)
|
|
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
|
|
return rows;
|
|
}
|
|
|
|
getMedia(id: number): MediaRow | null {
|
|
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
|
|
| MediaRow
|
|
| undefined;
|
|
return row ?? null;
|
|
}
|
|
|
|
private mediaWhere(options: MediaFilterOptions): { where: string; values: Array<string | number> } {
|
|
const conditions: string[] = [];
|
|
const values: Array<string | number> = [];
|
|
|
|
if (options.kind) {
|
|
conditions.push("kind = ?");
|
|
values.push(options.kind);
|
|
}
|
|
if (options.source) {
|
|
conditions.push("source = ?");
|
|
values.push(options.source);
|
|
}
|
|
if (options.authorId) {
|
|
conditions.push("author_id = ?");
|
|
values.push(options.authorId);
|
|
}
|
|
if (options.authorIds?.length) {
|
|
conditions.push(`author_id IN (${options.authorIds.map(() => "?").join(", ")})`);
|
|
values.push(...options.authorIds);
|
|
}
|
|
if (options.cachedOnly) {
|
|
conditions.push("preview_hash IS NOT NULL");
|
|
}
|
|
if (options.exts?.length) {
|
|
const clauses = options.exts.map(
|
|
() => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
|
|
);
|
|
conditions.push(`(${clauses.join(" OR ")})`);
|
|
for (const ext of options.exts) {
|
|
values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
|
|
}
|
|
}
|
|
for (const host of options.excludeHosts ?? []) {
|
|
conditions.push("lower(url) NOT LIKE ?");
|
|
values.push(`%${host.toLowerCase()}%`);
|
|
}
|
|
if (options.excludeExts?.length) {
|
|
const clauses = options.excludeExts.map(
|
|
() => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
|
|
);
|
|
conditions.push(`NOT (${clauses.join(" OR ")})`);
|
|
for (const ext of options.excludeExts) {
|
|
values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
|
|
}
|
|
}
|
|
for (const provider of options.excludeProviders ?? []) {
|
|
conditions.push("lower(COALESCE(provider, '')) NOT LIKE ?");
|
|
values.push(`%${provider.toLowerCase()}%`);
|
|
}
|
|
if (options.excludeSources?.length) {
|
|
conditions.push(`COALESCE(source, '') NOT IN (${options.excludeSources.map(() => "?").join(", ")})`);
|
|
values.push(...options.excludeSources);
|
|
}
|
|
if (options.includeProviders?.length) {
|
|
const clauses = options.includeProviders.map(() => "lower(COALESCE(provider, '')) LIKE ?");
|
|
conditions.push(`(${clauses.join(" OR ")})`);
|
|
for (const provider of options.includeProviders) {
|
|
values.push(`%${provider.toLowerCase()}%`);
|
|
}
|
|
}
|
|
if (options.includeSources?.length) {
|
|
conditions.push(`COALESCE(source, '') IN (${options.includeSources.map(() => "?").join(", ")})`);
|
|
values.push(...options.includeSources);
|
|
}
|
|
if (options.sizeLessThan !== undefined) {
|
|
conditions.push("COALESCE(file_bytes, size) < ?");
|
|
values.push(options.sizeLessThan);
|
|
}
|
|
if (options.sizeGreaterThan !== undefined) {
|
|
conditions.push("COALESCE(file_bytes, size) > ?");
|
|
values.push(options.sizeGreaterThan);
|
|
}
|
|
if (options.lengthLessThan !== undefined) {
|
|
conditions.push("duration_secs < ?");
|
|
values.push(options.lengthLessThan);
|
|
}
|
|
if (options.lengthGreaterThan !== undefined) {
|
|
conditions.push("duration_secs > ?");
|
|
values.push(options.lengthGreaterThan);
|
|
}
|
|
if (options.titleQuery?.trim()) {
|
|
const query = normalizeTitleQuery(options.titleQuery);
|
|
if (options.titleExact) {
|
|
// One phrase, substring of the title; `*` still acts as a wildcard.
|
|
conditions.push("title_key LIKE ? ESCAPE '\\'");
|
|
values.push(`%${titleLikePattern(query)}%`);
|
|
} else {
|
|
for (const token of query.split(" ").filter(Boolean)) {
|
|
const pattern = titleLikePattern(token);
|
|
if (token.startsWith("*")) {
|
|
// Leading `*`: anywhere in a word, not just the start.
|
|
conditions.push("COALESCE(title_key, '') LIKE ? ESCAPE '\\'");
|
|
values.push(`%${pattern}%`);
|
|
} else {
|
|
conditions.push("(' ' || COALESCE(title_key, '')) LIKE ? ESCAPE '\\'");
|
|
values.push(`% ${pattern}%`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (options.after) {
|
|
conditions.push("posted_at >= ?");
|
|
values.push(options.after);
|
|
}
|
|
if (options.before) {
|
|
conditions.push("posted_at <= ?");
|
|
values.push(options.before);
|
|
}
|
|
|
|
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
|
|
if (aspect) conditions.push(aspect);
|
|
|
|
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
|
|
if (resolution) conditions.push(resolution);
|
|
|
|
return { where: conditions.length ? `WHERE ${conditions.join(" AND ")}` : "", values };
|
|
}
|
|
|
|
queryMedia(options: MediaFilterOptions = {}): MediaRow[] {
|
|
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
|
const offset = Math.max(0, options.offset ?? 0);
|
|
const { where, values } = this.mediaWhere(options);
|
|
|
|
const rows = this.db
|
|
.prepare(
|
|
`SELECT media.*,
|
|
(SELECT message_id FROM media_sources WHERE media_id = media.id LIMIT 1) AS message_id
|
|
FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`
|
|
)
|
|
.all(...values, limit, offset);
|
|
|
|
return rows as unknown as MediaRow[];
|
|
}
|
|
|
|
}
|