Add the media scrape engine, store, and puppet walker
This commit is contained in:
Fisto
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Per-scrape storage: one directory, one SQLite index, one media folder.
|
||||
* Layout and rules: docs/media.md.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { extractMedia } from "./extract.js";
|
||||
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
||||
|
||||
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
||||
export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url));
|
||||
|
||||
export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT;
|
||||
}
|
||||
|
||||
export function scrapeDir(root: string, id: string): string {
|
||||
return join(root, id);
|
||||
}
|
||||
|
||||
export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void {
|
||||
writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n");
|
||||
}
|
||||
|
||||
export function readScrapeMeta(dir: string): ScrapeMeta | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Every scrape on disk, newest first. */
|
||||
export function listScrapes(root: string): ScrapeMeta[] {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(root);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const metas: ScrapeMeta[] = [];
|
||||
for (const entry of entries) {
|
||||
const meta = readScrapeMeta(join(root, entry));
|
||||
if (meta) metas.push(meta);
|
||||
}
|
||||
|
||||
return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
|
||||
}
|
||||
|
||||
/** Combined size of a directory tree, used by the storage limit. */
|
||||
export function dirSizeBytes(dir: string): number {
|
||||
let entries;
|
||||
try {
|
||||
entries = readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
for (const entry of entries) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
total += dirSizeBytes(path);
|
||||
} else {
|
||||
try {
|
||||
total += statSync(path).size;
|
||||
} catch {
|
||||
// A file that vanished mid-walk counts as zero.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
export function createScrapeDirs(root: string, id: string): string {
|
||||
const dir = scrapeDir(root, id);
|
||||
mkdirSync(join(dir, "media"), { recursive: true });
|
||||
mkdirSync(join(dir, "thumbs"), { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** `<channelId>-<YYYYMMDDHHMMSS>`, with a numeric suffix on a same-second collision. */
|
||||
export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string {
|
||||
const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, "");
|
||||
const base = `${channelId}-${stamp}`;
|
||||
|
||||
let id = base;
|
||||
let suffix = 2;
|
||||
while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`;
|
||||
return id;
|
||||
}
|
||||
|
||||
export interface MediaRow {
|
||||
id: number;
|
||||
kind: MediaKind;
|
||||
url: string;
|
||||
provider: string | null;
|
||||
content_type: string | null;
|
||||
size: number | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
duration_secs: number | null;
|
||||
downloaded: number;
|
||||
preview_url: string | null;
|
||||
file_hash: string | null;
|
||||
preview_hash: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
author_id TEXT NOT NULL,
|
||||
author_name TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
content TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kind TEXT NOT NULL,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
provider TEXT,
|
||||
content_type TEXT,
|
||||
size INTEGER,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
duration_secs REAL,
|
||||
downloaded INTEGER NOT NULL DEFAULT 0,
|
||||
preview_url TEXT,
|
||||
file_hash TEXT,
|
||||
preview_hash TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_sources (
|
||||
media_id INTEGER NOT NULL REFERENCES media(id),
|
||||
message_id TEXT NOT NULL,
|
||||
PRIMARY KEY (media_id, message_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
||||
`;
|
||||
|
||||
/** One open SQLite index. The caller closes it. */
|
||||
export class ScrapeStore {
|
||||
private readonly db: DatabaseSync;
|
||||
|
||||
constructor(readonly dir: string) {
|
||||
this.db = new DatabaseSync(join(dir, "index.db"));
|
||||
this.db.exec(SCHEMA);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
/** Write one page of messages in a transaction. Returns the newly inserted media per kind. */
|
||||
insertPage(messages: CoreMessage[]): Record<MediaKind, number> {
|
||||
const added = emptyMediaCounts();
|
||||
const insertMessage = this.db.prepare(
|
||||
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
||||
);
|
||||
const insertMedia = this.db.prepare(
|
||||
"INSERT OR IGNORE INTO media (kind, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
);
|
||||
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
||||
const linkSource = this.db.prepare(
|
||||
"INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)"
|
||||
);
|
||||
const createdAt = new Date().toISOString();
|
||||
|
||||
this.db.exec("BEGIN");
|
||||
try {
|
||||
for (const message of messages) {
|
||||
insertMessage.run(
|
||||
message.id,
|
||||
message.author.id,
|
||||
message.author.username,
|
||||
message.timestamp,
|
||||
message.content
|
||||
);
|
||||
|
||||
for (const item of extractMedia(message)) {
|
||||
const result = insertMedia.run(
|
||||
item.kind,
|
||||
item.url,
|
||||
item.provider,
|
||||
item.contentType,
|
||||
item.size,
|
||||
item.width,
|
||||
item.height,
|
||||
item.durationSecs,
|
||||
item.downloaded ? 1 : 0,
|
||||
item.previewUrl,
|
||||
createdAt
|
||||
);
|
||||
if (result.changes > 0) added[item.kind]++;
|
||||
|
||||
const row = findMedia.get(item.url) as { id: number } | undefined;
|
||||
if (row) linkSource.run(row.id, message.id);
|
||||
}
|
||||
}
|
||||
this.db.exec("COMMIT");
|
||||
} catch (error) {
|
||||
this.db.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
counts(): Record<MediaKind, number> {
|
||||
const rows = this.db
|
||||
.prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")
|
||||
.all() as unknown as Array<{ kind: string; n: number }>;
|
||||
|
||||
const result = emptyMediaCounts();
|
||||
for (const row of rows) {
|
||||
if (row.kind in result) result[row.kind as MediaKind] = row.n;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
queryMedia(options: { kind?: MediaKind; limit?: number; offset?: number } = {}): MediaRow[] {
|
||||
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
||||
const offset = Math.max(0, options.offset ?? 0);
|
||||
|
||||
const statement = options.kind
|
||||
? this.db.prepare("SELECT * FROM media WHERE kind = ? ORDER BY id DESC LIMIT ? OFFSET ?")
|
||||
: this.db.prepare("SELECT * FROM media ORDER BY id DESC LIMIT ? OFFSET ?");
|
||||
|
||||
const rows = options.kind
|
||||
? statement.all(options.kind, limit, offset)
|
||||
: statement.all(limit, offset);
|
||||
|
||||
return rows as unknown as MediaRow[];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user