Add the media scrape engine, store, and puppet walker
This commit is contained in:
Fisto
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
* The scrape engine: queue, walk, pacing, budgets, backoff, resume.
|
||||
* Rules: docs/media-scraper.md.
|
||||
*/
|
||||
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createScrapeDirs,
|
||||
listScrapes,
|
||||
readScrapeMeta,
|
||||
ScrapeStore,
|
||||
uniqueScrapeId,
|
||||
writeScrapeMeta
|
||||
} from "./store.js";
|
||||
import {
|
||||
emptyMediaCounts,
|
||||
MEDIA_KINDS,
|
||||
type BackendName,
|
||||
type CoreMessage,
|
||||
type ScrapeMeta,
|
||||
type ScrapeMode
|
||||
} from "./types.js";
|
||||
|
||||
export interface FetchPage {
|
||||
messages: CoreMessage[];
|
||||
/** Set by a backend that saw a rate-limit hint. The engine sleeps before the next page. */
|
||||
rateLimitedMs?: number;
|
||||
}
|
||||
|
||||
export interface ScrapeBackend {
|
||||
readonly name: BackendName;
|
||||
fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage>;
|
||||
}
|
||||
|
||||
export interface ScrapeEngineOptions {
|
||||
rootDir: string;
|
||||
pageSize?: number;
|
||||
pageMinMs?: number;
|
||||
pageMaxMs?: number;
|
||||
pagesPerHour?: number;
|
||||
pagesPerDay?: number;
|
||||
maxRetries?: number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
random?: () => number;
|
||||
now?: () => number;
|
||||
log?: (line: string) => void;
|
||||
}
|
||||
|
||||
interface QueuedJob {
|
||||
meta: ScrapeMeta;
|
||||
dir: string;
|
||||
backend: ScrapeBackend;
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms));
|
||||
|
||||
function oldestId(messages: CoreMessage[]): string | null {
|
||||
let oldest: string | null = null;
|
||||
for (const message of messages) {
|
||||
if (oldest === null || BigInt(message.id) < BigInt(oldest)) oldest = message.id;
|
||||
}
|
||||
return oldest;
|
||||
}
|
||||
|
||||
export class ScrapeEngine {
|
||||
private readonly queue: QueuedJob[] = [];
|
||||
private readonly active = new Map<string, ScrapeMeta>();
|
||||
private readonly stopRequested = new Set<string>();
|
||||
private readonly pageTimes: number[] = [];
|
||||
private processingPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(private readonly options: ScrapeEngineOptions) {}
|
||||
|
||||
/** Create a scrape and queue the walk. Returns immediately. */
|
||||
async start(input: {
|
||||
channelId: string;
|
||||
guildId: string | null;
|
||||
mode: ScrapeMode;
|
||||
backend: ScrapeBackend;
|
||||
}): Promise<ScrapeMeta> {
|
||||
const now = this.now();
|
||||
const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
|
||||
const dir = createScrapeDirs(this.options.rootDir, id);
|
||||
const timestamp = new Date(now).toISOString();
|
||||
|
||||
const meta: ScrapeMeta = {
|
||||
id,
|
||||
channelId: input.channelId,
|
||||
guildId: input.guildId,
|
||||
backend: input.backend.name,
|
||||
mode: input.mode,
|
||||
state: "queued",
|
||||
cursor: null,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
pages: 0,
|
||||
messages: 0,
|
||||
media: emptyMediaCounts(),
|
||||
errors: 0,
|
||||
lastError: null
|
||||
};
|
||||
|
||||
writeScrapeMeta(dir, meta);
|
||||
this.active.set(id, meta);
|
||||
this.queue.push({ meta, dir, backend: input.backend });
|
||||
void this.processQueue();
|
||||
|
||||
return { ...meta };
|
||||
}
|
||||
|
||||
/** Run queued jobs until the queue is empty. Tests await this; the daemon lets it run. */
|
||||
processQueue(): Promise<void> {
|
||||
if (!this.processingPromise) {
|
||||
this.processingPromise = this.drainQueue().finally(() => {
|
||||
this.processingPromise = null;
|
||||
});
|
||||
}
|
||||
return this.processingPromise;
|
||||
}
|
||||
|
||||
private async drainQueue(): Promise<void> {
|
||||
while (this.queue.length > 0) {
|
||||
const job = this.queue.shift();
|
||||
if (!job) break;
|
||||
await this.runJob(job);
|
||||
}
|
||||
}
|
||||
|
||||
get(id: string): ScrapeMeta | null {
|
||||
const active = this.active.get(id);
|
||||
if (active) return { ...active };
|
||||
return readScrapeMeta(join(this.options.rootDir, id));
|
||||
}
|
||||
|
||||
/** Every scrape on disk, merged with the jobs this engine still holds in memory. */
|
||||
list(): ScrapeMeta[] {
|
||||
const merged = new Map<string, ScrapeMeta>();
|
||||
for (const meta of listScrapes(this.options.rootDir)) merged.set(meta.id, meta);
|
||||
for (const [id, meta] of this.active) merged.set(id, meta);
|
||||
|
||||
return [...merged.values()].sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
|
||||
}
|
||||
|
||||
/** Ask a running or queued scrape to stop. It pauses after the current page. */
|
||||
stop(id: string): void {
|
||||
this.stopRequested.add(id);
|
||||
}
|
||||
|
||||
private async runJob(job: QueuedJob): Promise<void> {
|
||||
const { meta, dir, backend } = job;
|
||||
const store = new ScrapeStore(dir);
|
||||
const pageSize = this.options.pageSize ?? 100;
|
||||
|
||||
try {
|
||||
if (this.stopRequested.has(meta.id)) {
|
||||
meta.state = "paused";
|
||||
return;
|
||||
}
|
||||
|
||||
meta.state = "running";
|
||||
this.persist(dir, meta);
|
||||
|
||||
for (;;) {
|
||||
if (this.stopRequested.has(meta.id)) {
|
||||
meta.state = "paused";
|
||||
break;
|
||||
}
|
||||
|
||||
if (!this.allowPage()) {
|
||||
meta.state = "paused";
|
||||
this.log(`${meta.id}: page budget reached, paused`);
|
||||
break;
|
||||
}
|
||||
|
||||
const page = await this.fetchWithBackoff(meta, backend);
|
||||
if (page.rateLimitedMs) await this.sleep(page.rateLimitedMs);
|
||||
|
||||
if (page.messages.length === 0) {
|
||||
meta.state = "done";
|
||||
break;
|
||||
}
|
||||
|
||||
const added = store.insertPage(page.messages);
|
||||
for (const kind of MEDIA_KINDS) meta.media[kind] += added[kind];
|
||||
meta.pages++;
|
||||
meta.messages += page.messages.length;
|
||||
meta.cursor = oldestId(page.messages) ?? meta.cursor;
|
||||
this.pageTimes.push(this.now());
|
||||
this.persist(dir, meta);
|
||||
|
||||
if (page.messages.length < pageSize) {
|
||||
meta.state = "done";
|
||||
break;
|
||||
}
|
||||
|
||||
await this.sleep(this.gapMs());
|
||||
}
|
||||
} catch (error) {
|
||||
meta.state = "failed";
|
||||
meta.lastError = error instanceof Error ? error.message : String(error);
|
||||
this.log(`${meta.id}: failed: ${meta.lastError}`);
|
||||
} finally {
|
||||
this.persist(dir, meta);
|
||||
store.close();
|
||||
this.active.set(meta.id, meta);
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise<FetchPage> {
|
||||
const pageSize = this.options.pageSize ?? 100;
|
||||
const maxRetries = this.options.maxRetries ?? 8;
|
||||
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
try {
|
||||
return await backend.fetchPage(meta.channelId, meta.cursor, pageSize);
|
||||
} catch (error) {
|
||||
meta.errors++;
|
||||
if (attempt >= maxRetries) throw error;
|
||||
|
||||
const retryAfter = (error as { retryAfterMs?: number }).retryAfterMs;
|
||||
const delay = retryAfter ?? Math.pow(2, attempt) * 1000 + 1000;
|
||||
this.log(`${meta.id}: page failed, retrying in ${delay} ms`);
|
||||
await this.sleep(delay);
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private allowPage(): boolean {
|
||||
const now = this.now();
|
||||
while (this.pageTimes.length > 0 && this.pageTimes[0] < now - 86_400_000) {
|
||||
this.pageTimes.shift();
|
||||
}
|
||||
|
||||
const lastHour = this.pageTimes.filter(time => time > now - 3_600_000).length;
|
||||
if (lastHour >= (this.options.pagesPerHour ?? 300)) return false;
|
||||
if (this.pageTimes.length >= (this.options.pagesPerDay ?? 3000)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private gapMs(): number {
|
||||
const min = this.options.pageMinMs ?? 1200;
|
||||
const max = Math.max(min, this.options.pageMaxMs ?? 3500);
|
||||
return min + Math.floor(this.random() * (max - min + 1));
|
||||
}
|
||||
|
||||
private persist(dir: string, meta: ScrapeMeta): void {
|
||||
meta.updatedAt = new Date(this.now()).toISOString();
|
||||
writeScrapeMeta(dir, meta);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
return (this.options.now ?? Date.now)();
|
||||
}
|
||||
|
||||
private random(): number {
|
||||
return (this.options.random ?? Math.random)();
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return (this.options.sleep ?? defaultSleep)(ms);
|
||||
}
|
||||
|
||||
private log(line: string): void {
|
||||
this.options.log?.(line);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/*
|
||||
* Media extraction: classify one core message into media items.
|
||||
* Rules: docs/media-scraper.md. Third-party embeds are never downloaded.
|
||||
*/
|
||||
|
||||
import type { CoreAttachment, CoreEmbed, CoreSticker, CoreMessage, MediaKind, MediaMessage } from "./types.js";
|
||||
|
||||
const IMAGE_EXT = new Set(["png", "jpg", "jpeg", "webp", "bmp", "avif"]);
|
||||
const GIF_EXT = new Set(["gif"]);
|
||||
const VIDEO_EXT = new Set(["mp4", "webm", "mov", "mkv"]);
|
||||
const AUDIO_EXT = new Set(["mp3", "m4a", "ogg", "wav", "flac"]);
|
||||
|
||||
/** Sticker formatType to file extension. 3 is Lottie JSON; not archived. */
|
||||
const STICKER_EXT: Record<number, string | undefined> = { 1: "png", 2: "apng", 4: "gif" };
|
||||
|
||||
/** A media item extracted from a message. `url` is the original source link. */
|
||||
export interface ExtractedMedia {
|
||||
kind: MediaKind;
|
||||
url: string;
|
||||
provider: string | null;
|
||||
contentType: string | null;
|
||||
size: number | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
durationSecs: number | null;
|
||||
/** False for every embed. Downloads are a later step, never for third-party embeds. */
|
||||
downloaded: boolean;
|
||||
previewUrl: string | null;
|
||||
}
|
||||
|
||||
function extension(filename: string | null | undefined): string {
|
||||
const match = /\.([a-z0-9]+)$/i.exec(filename ?? "");
|
||||
return match ? match[1].toLowerCase() : "";
|
||||
}
|
||||
|
||||
function classifyAttachment(attachment: CoreAttachment, voiceMessage: boolean): MediaKind | null {
|
||||
const ext = extension(attachment.filename);
|
||||
const type = (attachment.contentType ?? "").toLowerCase();
|
||||
|
||||
// A voice-message flag wins unless the file is clearly an image or a video.
|
||||
if (
|
||||
voiceMessage &&
|
||||
!IMAGE_EXT.has(ext) &&
|
||||
!GIF_EXT.has(ext) &&
|
||||
!VIDEO_EXT.has(ext) &&
|
||||
!type.startsWith("image/") &&
|
||||
!type.startsWith("video/")
|
||||
) {
|
||||
return "audio";
|
||||
}
|
||||
|
||||
if (AUDIO_EXT.has(ext) || type.startsWith("audio/")) return "audio";
|
||||
if (VIDEO_EXT.has(ext) || type.startsWith("video/")) return "video";
|
||||
if (GIF_EXT.has(ext) || type === "image/gif") return "gif";
|
||||
if (IMAGE_EXT.has(ext) || type.startsWith("image/")) return "image";
|
||||
return null;
|
||||
}
|
||||
|
||||
function hostname(url: string | null): string | null {
|
||||
if (!url) return null;
|
||||
try {
|
||||
return new URL(url).hostname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
|
||||
const provider = embed.provider ?? hostname(embed.url);
|
||||
const common = {
|
||||
provider,
|
||||
downloaded: false,
|
||||
size: null,
|
||||
width: null,
|
||||
height: null,
|
||||
durationSecs: null
|
||||
};
|
||||
|
||||
if (embed.videoUrl) {
|
||||
return {
|
||||
...common,
|
||||
kind: "video",
|
||||
url: embed.url ?? embed.videoUrl,
|
||||
contentType: null,
|
||||
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
|
||||
};
|
||||
}
|
||||
|
||||
const providerKey = (provider ?? "").toLowerCase();
|
||||
if (embed.type === "gifv" || providerKey.includes("tenor") || providerKey.includes("giphy")) {
|
||||
return {
|
||||
...common,
|
||||
kind: "gif",
|
||||
url: embed.url ?? embed.imageUrl ?? "",
|
||||
contentType: null,
|
||||
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
|
||||
};
|
||||
}
|
||||
|
||||
if (embed.imageUrl) {
|
||||
return {
|
||||
...common,
|
||||
kind: "image",
|
||||
url: embed.url ?? embed.imageUrl,
|
||||
contentType: null,
|
||||
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractSticker(sticker: CoreSticker): ExtractedMedia | null {
|
||||
if (!sticker.id) return null;
|
||||
const ext = STICKER_EXT[sticker.formatType ?? 1];
|
||||
if (!ext) return null;
|
||||
|
||||
return {
|
||||
kind: ext === "gif" ? "gif" : "image",
|
||||
url: `https://cdn.discordapp.com/stickers/${sticker.id}.${ext}`,
|
||||
provider: "discord",
|
||||
contentType: null,
|
||||
size: null,
|
||||
width: null,
|
||||
height: null,
|
||||
durationSecs: null,
|
||||
downloaded: false,
|
||||
previewUrl: null
|
||||
};
|
||||
}
|
||||
|
||||
function extractFrom(message: MediaMessage): ExtractedMedia[] {
|
||||
const items: ExtractedMedia[] = [];
|
||||
|
||||
for (const attachment of message.attachments ?? []) {
|
||||
if (!attachment.url) continue;
|
||||
const kind = classifyAttachment(attachment, message.voiceMessage);
|
||||
if (!kind) continue;
|
||||
items.push({
|
||||
kind,
|
||||
url: attachment.url,
|
||||
provider: "discord",
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
width: attachment.width,
|
||||
height: attachment.height,
|
||||
durationSecs: attachment.durationSecs,
|
||||
downloaded: false,
|
||||
previewUrl: null
|
||||
});
|
||||
}
|
||||
|
||||
for (const embed of message.embeds ?? []) {
|
||||
const item = extractEmbed(embed);
|
||||
if (item?.url) items.push(item);
|
||||
}
|
||||
|
||||
for (const sticker of message.stickerItems ?? []) {
|
||||
const item = extractSticker(sticker);
|
||||
if (item) items.push(item);
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/** Extract every media item from a message, including forwarded snapshots. Deduped by URL. */
|
||||
export function extractMedia(message: CoreMessage): ExtractedMedia[] {
|
||||
const items = extractFrom(message);
|
||||
|
||||
for (const snapshot of message.forwarded ?? []) {
|
||||
items.push(...extractFrom(snapshot));
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
return items.filter(item => {
|
||||
if (seen.has(item.url)) return false;
|
||||
seen.add(item.url);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Media archive entry point. The daemon mounts this module at checkpoint C5;
|
||||
* nothing imports it before that.
|
||||
*/
|
||||
|
||||
export { extractMedia, type ExtractedMedia } from "./extract.js";
|
||||
export {
|
||||
ScrapeEngine,
|
||||
type FetchPage,
|
||||
type ScrapeBackend,
|
||||
type ScrapeEngineOptions
|
||||
} from "./engine.js";
|
||||
export {
|
||||
DEFAULT_DB_ROOT,
|
||||
dirSizeBytes,
|
||||
listScrapes,
|
||||
readScrapeMeta,
|
||||
resolveDbRoot,
|
||||
ScrapeStore,
|
||||
scrapeDir,
|
||||
uniqueScrapeId,
|
||||
writeScrapeMeta,
|
||||
type MediaRow
|
||||
} from "./store.js";
|
||||
export * from "./types.js";
|
||||
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Per-scrape storage: one directory, one SQLite index, one media folder.
|
||||
* Layout and rules: docs/media.md.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { extractMedia } from "./extract.js";
|
||||
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
||||
|
||||
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
||||
export const DEFAULT_DB_ROOT = fileURLToPath(new URL("../db", import.meta.url));
|
||||
|
||||
export function resolveDbRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env.VESKSLOP_MEDIA_DB_DIR ?? DEFAULT_DB_ROOT;
|
||||
}
|
||||
|
||||
export function scrapeDir(root: string, id: string): string {
|
||||
return join(root, id);
|
||||
}
|
||||
|
||||
export function writeScrapeMeta(dir: string, meta: ScrapeMeta): void {
|
||||
writeFileSync(join(dir, "scrape.json"), JSON.stringify(meta, null, 2) + "\n");
|
||||
}
|
||||
|
||||
export function readScrapeMeta(dir: string): ScrapeMeta | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(join(dir, "scrape.json"), "utf8")) as ScrapeMeta;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Every scrape on disk, newest first. */
|
||||
export function listScrapes(root: string): ScrapeMeta[] {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(root);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const metas: ScrapeMeta[] = [];
|
||||
for (const entry of entries) {
|
||||
const meta = readScrapeMeta(join(root, entry));
|
||||
if (meta) metas.push(meta);
|
||||
}
|
||||
|
||||
return metas.sort((a, b) => (a.createdAt < b.createdAt ? 1 : -1));
|
||||
}
|
||||
|
||||
/** Combined size of a directory tree, used by the storage limit. */
|
||||
export function dirSizeBytes(dir: string): number {
|
||||
let entries;
|
||||
try {
|
||||
entries = readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
for (const entry of entries) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
total += dirSizeBytes(path);
|
||||
} else {
|
||||
try {
|
||||
total += statSync(path).size;
|
||||
} catch {
|
||||
// A file that vanished mid-walk counts as zero.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
export function createScrapeDirs(root: string, id: string): string {
|
||||
const dir = scrapeDir(root, id);
|
||||
mkdirSync(join(dir, "media"), { recursive: true });
|
||||
mkdirSync(join(dir, "thumbs"), { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** `<channelId>-<YYYYMMDDHHMMSS>`, with a numeric suffix on a same-second collision. */
|
||||
export function uniqueScrapeId(root: string, channelId: string, now = new Date()): string {
|
||||
const stamp = now.toISOString().replace(/\.\d+Z$/, "").replace(/[-:T]/g, "");
|
||||
const base = `${channelId}-${stamp}`;
|
||||
|
||||
let id = base;
|
||||
let suffix = 2;
|
||||
while (readScrapeMeta(join(root, id))) id = `${base}-${suffix++}`;
|
||||
return id;
|
||||
}
|
||||
|
||||
export interface MediaRow {
|
||||
id: number;
|
||||
kind: MediaKind;
|
||||
url: string;
|
||||
provider: string | null;
|
||||
content_type: string | null;
|
||||
size: number | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
duration_secs: number | null;
|
||||
downloaded: number;
|
||||
preview_url: string | null;
|
||||
file_hash: string | null;
|
||||
preview_hash: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
author_id TEXT NOT NULL,
|
||||
author_name TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
content TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kind TEXT NOT NULL,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
provider TEXT,
|
||||
content_type TEXT,
|
||||
size INTEGER,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
duration_secs REAL,
|
||||
downloaded INTEGER NOT NULL DEFAULT 0,
|
||||
preview_url TEXT,
|
||||
file_hash TEXT,
|
||||
preview_hash TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_sources (
|
||||
media_id INTEGER NOT NULL REFERENCES media(id),
|
||||
message_id TEXT NOT NULL,
|
||||
PRIMARY KEY (media_id, message_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
||||
`;
|
||||
|
||||
/** One open SQLite index. The caller closes it. */
|
||||
export class ScrapeStore {
|
||||
private readonly db: DatabaseSync;
|
||||
|
||||
constructor(readonly dir: string) {
|
||||
this.db = new DatabaseSync(join(dir, "index.db"));
|
||||
this.db.exec(SCHEMA);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
/** Write one page of messages in a transaction. Returns the newly inserted media per kind. */
|
||||
insertPage(messages: CoreMessage[]): Record<MediaKind, number> {
|
||||
const added = emptyMediaCounts();
|
||||
const insertMessage = this.db.prepare(
|
||||
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
||||
);
|
||||
const insertMedia = this.db.prepare(
|
||||
"INSERT OR IGNORE INTO media (kind, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
);
|
||||
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
||||
const linkSource = this.db.prepare(
|
||||
"INSERT OR IGNORE INTO media_sources (media_id, message_id) VALUES (?, ?)"
|
||||
);
|
||||
const createdAt = new Date().toISOString();
|
||||
|
||||
this.db.exec("BEGIN");
|
||||
try {
|
||||
for (const message of messages) {
|
||||
insertMessage.run(
|
||||
message.id,
|
||||
message.author.id,
|
||||
message.author.username,
|
||||
message.timestamp,
|
||||
message.content
|
||||
);
|
||||
|
||||
for (const item of extractMedia(message)) {
|
||||
const result = insertMedia.run(
|
||||
item.kind,
|
||||
item.url,
|
||||
item.provider,
|
||||
item.contentType,
|
||||
item.size,
|
||||
item.width,
|
||||
item.height,
|
||||
item.durationSecs,
|
||||
item.downloaded ? 1 : 0,
|
||||
item.previewUrl,
|
||||
createdAt
|
||||
);
|
||||
if (result.changes > 0) added[item.kind]++;
|
||||
|
||||
const row = findMedia.get(item.url) as { id: number } | undefined;
|
||||
if (row) linkSource.run(row.id, message.id);
|
||||
}
|
||||
}
|
||||
this.db.exec("COMMIT");
|
||||
} catch (error) {
|
||||
this.db.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
counts(): Record<MediaKind, number> {
|
||||
const rows = this.db
|
||||
.prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")
|
||||
.all() as unknown as Array<{ kind: string; n: number }>;
|
||||
|
||||
const result = emptyMediaCounts();
|
||||
for (const row of rows) {
|
||||
if (row.kind in result) result[row.kind as MediaKind] = row.n;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
queryMedia(options: { kind?: MediaKind; limit?: number; offset?: number } = {}): MediaRow[] {
|
||||
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
||||
const offset = Math.max(0, options.offset ?? 0);
|
||||
|
||||
const statement = options.kind
|
||||
? this.db.prepare("SELECT * FROM media WHERE kind = ? ORDER BY id DESC LIMIT ? OFFSET ?")
|
||||
: this.db.prepare("SELECT * FROM media ORDER BY id DESC LIMIT ? OFFSET ?");
|
||||
|
||||
const rows = options.kind
|
||||
? statement.all(options.kind, limit, offset)
|
||||
: statement.all(limit, offset);
|
||||
|
||||
return rows as unknown as MediaRow[];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Shared types for the media archive. See docs/media.md and docs/media-scraper.md.
|
||||
*/
|
||||
|
||||
export type MediaKind = "image" | "gif" | "video" | "audio";
|
||||
|
||||
export const MEDIA_KINDS: MediaKind[] = ["image", "gif", "video", "audio"];
|
||||
|
||||
export type ScrapeState =
|
||||
| "queued"
|
||||
| "running"
|
||||
| "paused"
|
||||
| "done"
|
||||
| "failed"
|
||||
| "pending-delete"
|
||||
| "deleting"
|
||||
| "deleted";
|
||||
|
||||
export type ScrapeMode = "links" | "download";
|
||||
|
||||
export type BackendName = "puppet" | "slopbot";
|
||||
|
||||
export interface CoreAttachment {
|
||||
id: string | null;
|
||||
filename: string;
|
||||
url: string | null;
|
||||
contentType: string | null;
|
||||
size: number | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
durationSecs: number | null;
|
||||
}
|
||||
|
||||
export interface CoreEmbed {
|
||||
type: string | null;
|
||||
url: string | null;
|
||||
title: string | null;
|
||||
provider: string | null;
|
||||
videoUrl: string | null;
|
||||
imageUrl: string | null;
|
||||
thumbnailUrl: string | null;
|
||||
}
|
||||
|
||||
export interface CoreSticker {
|
||||
id: string | null;
|
||||
name: string | null;
|
||||
formatType: number | null;
|
||||
}
|
||||
|
||||
/** The part of a core message the extractor reads. Forwarded snapshots use the same shape. */
|
||||
export interface MediaMessage {
|
||||
attachments: CoreAttachment[];
|
||||
embeds: CoreEmbed[];
|
||||
stickerItems: CoreSticker[];
|
||||
voiceMessage: boolean;
|
||||
}
|
||||
|
||||
/** A message as shaped by the core bridge (`message.fetch`, `message.read`). */
|
||||
export interface CoreMessage extends MediaMessage {
|
||||
id: string;
|
||||
author: { id: string; username: string; avatar: string | null };
|
||||
content: string;
|
||||
timestamp: string;
|
||||
editedAt: string | null;
|
||||
replyTo: { id: string; authorId: string | null; authorName: string | null } | null;
|
||||
forwarded: Array<MediaMessage & { content: string; timestamp: string | null }>;
|
||||
}
|
||||
|
||||
/** `scrape.json`: the state of one scrape. The directory is the scrape's identity. */
|
||||
export interface ScrapeMeta {
|
||||
id: string;
|
||||
channelId: string;
|
||||
guildId: string | null;
|
||||
backend: BackendName;
|
||||
mode: ScrapeMode;
|
||||
state: ScrapeState;
|
||||
/** The oldest message ID seen. The next page is fetched before it. */
|
||||
cursor: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
pages: number;
|
||||
messages: number;
|
||||
media: Record<MediaKind, number>;
|
||||
errors: number;
|
||||
lastError: string | null;
|
||||
}
|
||||
|
||||
export function emptyMediaCounts(): Record<MediaKind, number> {
|
||||
return { image: 0, gif: 0, video: 0, audio: 0 };
|
||||
}
|
||||
Reference in New Issue
Block a user