/* * Media archive service: the daemon side of media-scrape/. See docs/media.md. * * Owns the scrape engine, the slopbot client, and the read path the Web UI and * (later) MCP tools call. */ import { rmSync } from "node:fs"; import { join } from "node:path"; import { dirSizeBytes, isNearLimit, planRetention, ScrapeEngine, ScrapeStore, scrapeDir, resolveDbRoot, type MediaKind, type MediaRow, type ScrapeBackend, type ScrapeMeta } from "../../media-scrape/src/index.js"; import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js"; import { loadSlopbotConfig, slopbotFetcher, slopbotStatus, SlopbotBackend, startSlopbot, type SlopbotFetcher, type SlopbotStatus } from "../../media-scrape/slopbot/index.js"; import type { CoreClient } from "./core-client.js"; import type { MediaPreferences } from "./preferences.js"; type BotClient = Awaited>; export interface MediaServiceDeps { getCore: () => CoreClient | undefined; isCoreConnected: () => boolean; getMediaPrefs: () => MediaPreferences; log?: (line: string) => void; } export interface StartScrapeInput { channelId: string; guildId?: string | null; mode?: "links" | "download"; backend?: "auto" | "slopbot" | "puppet"; /** Set by a re-run of the command after the near-limit warning. */ confirm?: boolean; } export interface ScrapeBytes { id: string; channelId: string; state: string; createdAt: string; bytes: number; } export interface MediaStorageStatus { usedBytes: number; limitBytes: number; warnPercent: number; overrunPercent: number; nearLimit: boolean; byKind: Record; scrapes: ScrapeBytes[]; } /** Thrown when a scrape starts near the storage limit without confirmation. */ export class MediaLimitError extends Error { readonly code = "MEDIA_LIMIT"; constructor( message: string, readonly usedBytes: number, readonly limitBytes: number ) { super(message); } } function formatBytes(bytes: number): string { const gib = bytes / 1024 ** 3; return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`; } export interface MediaItemResult { scrapeId: string; itemId: number; channelId: string; kind: MediaKind; url: string; localUrl: string | null; provider: string | null; downloaded: boolean; createdAt: string; } const FILE_CONTENT_TYPES: Record = { png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", webp: "image/webp", avif: "image/avif", mp4: "video/mp4", webm: "video/webm", mov: "video/quicktime", mkv: "video/x-matroska", mp3: "audio/mpeg", ogg: "audio/ogg", wav: "audio/wav", m4a: "audio/mp4", flac: "audio/flac" }; export interface MediaStatus { slopbot: SlopbotStatus; scrapes: ScrapeMeta[]; } export class MediaService { private readonly rootDir = resolveDbRoot(); private readonly engine: ScrapeEngine; private bot: BotClient | null = null; private botFetcher: SlopbotFetcher | null = null; private botStarting: Promise | null = null; constructor(private readonly deps: MediaServiceDeps) { const number = (value: string | undefined, fallback: number) => { const parsed = Number(value); return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; }; this.engine = new ScrapeEngine({ rootDir: this.rootDir, log: line => this.deps.log?.(line), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000), maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000) } }); } /** Log slopbot in on first use. False when it is disabled or the login failed. */ private async ensureSlopbot(): Promise { if (this.bot) return true; const config = loadSlopbotConfig(); if (!config.enabled) return false; if (!this.botStarting) { this.botStarting = startSlopbot(config) .then(client => { this.bot = client; this.botFetcher = slopbotFetcher(client); this.deps.log?.("slopbot connected"); }) .catch(error => { this.deps.log?.(`slopbot login failed: ${String(error)}`); }); } await this.botStarting; return this.bot !== null; } private puppetBackend(): ScrapeBackend { return new PuppetBackend(async (channelId, limit, options) => { const core = this.deps.getCore(); if (!core || !this.deps.isCoreConnected()) { throw new Error("the core bridge is not connected"); } return (await core.fetchMessages(channelId, limit, options)) as { messages?: unknown[] }; }); } /** Combined size of every scrape directory, with the limit settings applied. */ storageStatus(): MediaStorageStatus { const prefs = this.deps.getMediaPrefs(); const byKind: Record = { image: 0, gif: 0, video: 0, audio: 0 }; const scrapes = this.engine.list().map(meta => { const dir = scrapeDir(this.rootDir, meta.id); const bytes = dirSizeBytes(dir); try { const store = new ScrapeStore(dir); try { const sums = store.storedBytesByKind(); for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind]; } finally { store.close(); } } catch { // an unreadable index counts as zero bytes } return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes }; }); const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0); return { usedBytes, limitBytes: prefs.limitBytes, warnPercent: prefs.warnPercent, overrunPercent: prefs.overrunPercent, nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent), byKind, scrapes }; } /** Mark the oldest completed scrapes when the usage is at or over the limit. */ refreshRetention(): MediaStorageStatus { const status = this.storageStatus(); const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent); for (const id of plan.mark) this.engine.setState(id, "pending-delete"); return this.storageStatus(); } private applyRetention(): void { const status = this.storageStatus(); const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent); for (const id of plan.mark) this.engine.setState(id, "pending-delete"); for (const id of plan.delete) this.deleteScrape(id); } deleteScrape(id: string): boolean { try { rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true }); this.engine.forget(id); this.deps.log?.(`deleted scrape ${id}`); return true; } catch (error) { this.deps.log?.(`delete failed for ${id}: ${String(error)}`); return false; } } /** Delete the given scrapes, or every scrape when no IDs are given. */ cleanMedia(ids?: string[]): number { const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id); let deleted = 0; for (const id of targets) { if (this.deleteScrape(id)) deleted++; } return deleted; } async startScrape(input: StartScrapeInput): Promise { this.applyRetention(); const status = this.storageStatus(); if (status.nearLimit && !input.confirm) { throw new MediaLimitError( `media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`, status.usedBytes, status.limitBytes ); } const mode = input.mode ?? "links"; let backend: ScrapeBackend; if (input.backend === "puppet") { backend = this.puppetBackend(); } else { const available = await this.ensureSlopbot(); const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false); if (input.backend === "slopbot" && (!available || !inGuild)) { throw new Error("slopbot is not connected to that guild"); } const fetcher = this.botFetcher; backend = available && inGuild && fetcher ? new SlopbotBackend(fetcher) : this.puppetBackend(); } return this.engine.start({ channelId: input.channelId, guildId: input.guildId ?? null, mode, backend }); } stopScrape(id: string): void { this.engine.stop(id); } listScrapes(): ScrapeMeta[] { return this.engine.list(); } getScrape(id: string): ScrapeMeta | null { return this.engine.get(id); } status(): MediaStatus { return { slopbot: slopbotStatus(this.bot), scrapes: this.engine.list() }; } /** Absolute path and content type for a downloaded file, or null. */ getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null { const item = this.getMediaItem(scrapeId, itemId); if (!item?.file_path) return null; const extension = item.file_path.split(".").pop()?.toLowerCase() ?? ""; return { path: join(this.rootDir, scrapeId, item.file_path), contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream" }; } /** One media row inside a scrape, with its scrape and channel for context. */ getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { const meta = this.engine.get(scrapeId); if (!meta) return null; const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); try { const row = store.getMedia(itemId); return row ? { ...row, scrapeId, channelId: meta.channelId } : null; } finally { store.close(); } } /** Read media rows across every scrape, newest scrape first. */ queryMedia( options: { kind?: MediaKind; scrapeId?: string; limit?: number; offset?: number } = {} ): MediaItemResult[] { const limit = Math.max(1, Math.min(options.limit ?? 60, 500)); const offset = Math.max(0, options.offset ?? 0); const results: MediaItemResult[] = []; let skipped = 0; for (const meta of this.engine.list()) { if (meta.state === "deleted") continue; if (options.scrapeId && meta.id !== options.scrapeId) continue; const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { const rows = store.queryMedia({ kind: options.kind, limit: limit + offset }); for (const row of rows) { if (skipped++ < offset) continue; if (results.length >= limit) break; results.push({ scrapeId: meta.id, itemId: row.id, channelId: meta.channelId, kind: row.kind, url: row.url, localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null, provider: row.provider, downloaded: row.downloaded === 1, createdAt: row.created_at }); } } finally { store.close(); } if (results.length >= limit) break; } return results; } }