/* * Media archive service: the daemon side of media-scrape/. See docs/media.md. * * Owns the scrape engine, the slopbot client, and the read path the Web UI and * (later) MCP tools call. */ import { rmSync } from "node:fs"; import { isAbsolute, join } from "node:path"; import { BackfillQueue, betterCandidate, messageUrlFor, sortCandidates, dirSizeBytes, isNearLimit, planRetention, ScrapeEngine, ScrapeStore, scrapeDir, TitleLane, type TitleLaneStatus, snowflakeFromIso, resolveDbRoot, type MediaKind, type MediaRow, PROVIDER_FILTERS, type BackfillSettings, type BackfillStatus, type MediaSort, type ScrapeBackend, type ScrapeMeta } from "../../media-scrape/src/index.js"; import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js"; import { countMessages, loadSlopbotConfig, slopbotFetcher, slopbotStatus, SlopbotBackend, startSlopbot, type SlopbotFetcher, type SlopbotRest, type SlopbotStatus } from "../../media-scrape/slopbot/index.js"; import type { CoreClient } from "./core-client.js"; import type { MediaPreferences } from "./preferences.js"; type BotClient = Awaited>; export interface MediaServiceDeps { getCore: () => CoreClient | undefined; isCoreConnected: () => boolean; getMediaPrefs: () => MediaPreferences; log?: (line: string) => void; } export interface StartScrapeInput { channelId: string; guildId?: string | null; mode?: "links" | "download"; backend?: "auto" | "slopbot" | "puppet"; /** Set by a re-run of the command after the near-limit warning. */ confirm?: boolean; /** Absolute destination for downloaded files. Null keeps them under the scrape dir. */ downloadDir?: string | null; /** From-date as ISO; the walk stops below it. */ after?: string | null; /** To-date as ISO; the walk starts just below it. */ before?: string | null; } export interface MediaErrorEntry { at: string; source: string; message: string; } export interface ScrapeBytes { id: string; channelId: string; state: string; createdAt: string; bytes: number; } export interface MediaStorageStatus { usedBytes: number; limitBytes: number; warnPercent: number; overrunPercent: number; nearLimit: boolean; byKind: Record; scrapes: ScrapeBytes[]; } /** Thrown when a scrape starts near the storage limit without confirmation. */ export class MediaLimitError extends Error { readonly code = "MEDIA_LIMIT"; constructor( message: string, readonly usedBytes: number, readonly limitBytes: number ) { super(message); } } function formatBytes(bytes: number): string { const gib = bytes / 1024 ** 3; return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`; } export interface MediaItemResult { scrapeId: string; itemId: number; channelId: string; guildId: string | null; /** The message that carried the media; the permanent link target. */ messageId: string | null; messageUrl: string | null; kind: MediaKind; source: string | null; authorId: string | null; authorName: string | null; postedAt: string | null; width: number | null; height: number | null; aspectRatio: number | null; contentType: string | null; durationSecs: number | null; url: string; playUrl: string | null; /** The remote thumbnail URL from Discord's embed data (may be an animated GIF). */ remotePreviewUrl: string | null; /** Why no preview exists, when a fetch failed (for example "http 404"). */ previewError: string | null; /** Why no title was resolved (provider oEmbed and Discord's embed both failed). */ titleError: string | null; /** Local animated GIF for hover playback, when the original was cached. */ animUrl: string | null; localUrl: string | null; thumbUrl: string | null; provider: string | null; downloaded: boolean; /** Embed title (YouTube and friends), when Discord provided one. */ title: string | null; /** YouTube channel name kept from oEmbed, even if the video later disappears. */ titleAuthor: string | null; createdAt: string; } const FILE_CONTENT_TYPES: Record = { png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", webp: "image/webp", avif: "image/avif", mp4: "video/mp4", webm: "video/webm", mov: "video/quicktime", mkv: "video/x-matroska", mp3: "audio/mpeg", ogg: "audio/ogg", wav: "audio/wav", m4a: "audio/mp4", flac: "audio/flac" }; export interface MediaStatus { slopbot: SlopbotStatus; scrapes: ScrapeMeta[]; /** ISO time while a Discord rate limit holds media traffic. */ limitedUntil: string | null; /** The common providers, for the "Do not include" shortcuts in the Advanced drawer. */ providerFilters: string[]; } export class MediaService { private readonly rootDir = resolveDbRoot(); private readonly engine: ScrapeEngine; private readonly backfill: BackfillQueue; private readonly titles: TitleLane; private bot: BotClient | null = null; private botFetcher: SlopbotFetcher | null = null; private botStarting: Promise | null = null; /** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */ private readonly errors: MediaErrorEntry[] = []; /** One-shot maintenance: re-fetch previews for Discord rows closed out with HTTP errors. */ private readonly recover = { state: "idle" as "idle" | "running" | "paused" | "done", processed: 0, recovered: 0, deferred: 0, gone: 0, startedAt: null as number | null, finishedAt: null as number | null, currentScrapeId: null as string | null }; /** Finished maintenance tasks, newest first. */ private readonly taskHistory: Array<{ name: string; at: string; detail: string }> = []; constructor(private readonly deps: MediaServiceDeps) { const tunables = this.deps.getMediaPrefs().tunables; this.engine = new ScrapeEngine({ rootDir: this.rootDir, log: line => this.deps.log?.(line), onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`), walkCyclePages: tunables.scrape.walkCyclePages, rateLimitPauseMs: tunables.timers.rateLimitPauseMs, pageMinMs: tunables.timers.pageMinMs, pageMaxMs: tunables.timers.pageMaxMs, maxRetries: tunables.scrape.maxRetries, previewMaxAttempts: tunables.backfill.maxAttempts, suspectRetryMs: 50_000, embedMeta: async (scrapeId, media) => this.embedMetaFor(scrapeId, media), refreshUrl: async (scrapeId, media) => { const meta = this.engine.get(scrapeId); if (!meta) return null; const item = this.getMediaItem(scrapeId, media.id); if (!item) return null; return this.refreshUrls(meta, item); }, download: { maxFileBytes: tunables.scrape.maxFileBytes, previewMaxBytes: tunables.backfill.maxBytes, previewMinGapMs: tunables.backfill.previewMinGapMs, previewMaxGapMs: tunables.backfill.previewMaxGapMs, previewBatchSize: tunables.backfill.batchSize, previewBatchPauseMs: tunables.backfill.batchPauseMs, minGapMs: tunables.scrape.downloadMinGapMs, maxGapMs: tunables.scrape.downloadMaxGapMs } }); this.backfill = new BackfillQueue(this.engine, { rootDir: this.rootDir, getSettings: () => this.backfillSettings(), getGaps: () => { const live = this.deps.getMediaPrefs().tunables; return { minGapMs: live.backfill.previewMinGapMs, maxGapMs: live.backfill.previewMaxGapMs }; }, log: line => this.deps.log?.(line) }); // The title catch-up lane talks to YouTube only, so it runs beside the backfill queue. // It auto-starts because it is a one-time catch-up: with nothing pending it exits at once. this.titles = new TitleLane(this.engine, { rootDir: this.rootDir, log: line => this.deps.log?.(line) }); this.titles.start(); const stale = this.engine.reconcileStaleJobs(); if (stale.length) { this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`); } // Preview passes only run when a walk ends; the queue fills in scrapes that paused early. this.backfill.autoStart(); } private backfillSettings(): BackfillSettings { const tunables = this.deps.getMediaPrefs().tunables; return { batchSize: tunables.backfill.batchSize, batchPauseMs: tunables.backfill.batchPauseMs, auto: tunables.backfill.auto }; } /** Push preference changes into the live engine and backfill queue. */ applyTunables(): void { const tunables = this.deps.getMediaPrefs().tunables; this.engine.updateOptions({ walkCyclePages: tunables.scrape.walkCyclePages, rateLimitPauseMs: tunables.timers.rateLimitPauseMs, pageMinMs: tunables.timers.pageMinMs, pageMaxMs: tunables.timers.pageMaxMs, maxRetries: tunables.scrape.maxRetries, previewMaxAttempts: tunables.backfill.maxAttempts, download: { maxFileBytes: tunables.scrape.maxFileBytes, previewMaxBytes: tunables.backfill.maxBytes, previewMinGapMs: tunables.backfill.previewMinGapMs, previewMaxGapMs: tunables.backfill.previewMaxGapMs, previewBatchSize: tunables.backfill.batchSize, previewBatchPauseMs: tunables.backfill.batchPauseMs, minGapMs: tunables.scrape.downloadMinGapMs, maxGapMs: tunables.scrape.downloadMaxGapMs } }); } /** Backfill jobs, totals, and pacing, plus the engine's live cooldown. */ backfillStatus(): BackfillStatus { const status = this.backfill.list(); const limit = this.engine.rateLimitStatus(); return { ...status, limitedUntil: limit.active ? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString() : status.limitedUntil }; } titleStatus(): TitleLaneStatus { return this.titles.status(); } startTitles(): TitleLaneStatus { this.titles.start(); return this.titles.status(); } pauseTitles(): TitleLaneStatus { this.titles.pause(); return this.titles.status(); } /** The recovery task with its live remaining count and the task history. */ recoverStatus(): { state: string; total: number; processed: number; recovered: number; deferred: number; gone: number; startedAt: string | null; finishedAt: string | null; currentScrapeId: string | null; history: Array<{ name: string; at: string; detail: string }>; } { let total = 0; for (const meta of this.engine.list()) { if (meta.state === "deleted" || meta.state === "deleting") continue; try { const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { total += store.discordPreviewFailureCount(); } finally { store.close(); } } catch { // an unreadable index has nothing to recover } } return { state: this.recover.state, total, processed: this.recover.processed, recovered: this.recover.recovered, deferred: this.recover.deferred, gone: this.recover.gone, startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null, finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null, currentScrapeId: this.recover.currentScrapeId, history: [...this.taskHistory] }; } startRecover(): void { if (this.recover.state === "running") return; this.recover.state = "running"; this.recover.startedAt = this.recover.startedAt ?? Date.now(); this.recover.processed = 0; this.recover.recovered = 0; this.recover.deferred = 0; this.recover.gone = 0; this.deps.log?.("recover: retrying Discord-CDN previews marked gone"); // Guild scrapes refresh fastest through the bot REST; without it the puppet is tried. if (this.engine.list().some(meta => meta.backend === "slopbot")) { void this.ensureSlopbot().then(ok => { if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet"); }); } void this.runRecover(); } pauseRecover(): void { if (this.recover.state !== "running") return; this.recover.state = "paused"; this.deps.log?.("recover: pausing"); } /** * Walk every scrape's Discord-CDN preview failures, refresh the signed URL from the source * message (slopbot REST for guild scrapes, the puppet for DMs and group chats), then let the * engine fetch the preview. Rows whose URL cannot be refreshed are counted as gone. */ private async runRecover(): Promise { const scrapeIds = this.engine .list() .filter(meta => meta.state !== "deleted" && meta.state !== "deleting") .map(meta => meta.id); // Each row is tried once per run; rows whose URL cannot be refreshed keep their state // instead of being re-processed forever. const attempted = new Set(); while (this.recover.state === "running") { let acted = false; for (const scrapeId of scrapeIds) { if (this.recover.state !== "running") break; const meta = this.engine.get(scrapeId); if (!meta) continue; let store: ScrapeStore | null = null; try { store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); } catch { continue; } try { const rows = store .discordPreviewFailures(200) .filter(row => !attempted.has(`${scrapeId}:${row.id}`)) .slice(0, 10); if (!rows.length) continue; acted = true; this.recover.currentScrapeId = scrapeId; const cleared: MediaRow[] = []; for (const row of rows) { if (this.recover.state !== "running") break; attempted.add(`${scrapeId}:${row.id}`); this.recover.processed++; const fresh = await this.refreshUrls(meta, { ...row, scrapeId, channelId: meta.channelId } as MediaRow & { scrapeId: string; channelId: string }); if (fresh) { store.clearPreviewError(row.id); store.clearSuspect(row.id); cleared.push(row); } else { // No fresh URL after a full run: unavailable. Mark it so later runs skip it. store.markUnavailableAfterRecovery(row.id); this.recover.gone++; } } if (cleared.length && this.recover.state === "running") { await this.engine.previewBatch(scrapeId, cleared.length); for (const row of cleared) { const after = store.getMedia(row.id); if (after?.preview_hash) { this.recover.recovered++; } else { // Refreshed URL plus a fetch still failed: the file is gone. Mark it // so later recovery runs do not retry it again. if (after?.preview_suspect) store.markUnavailableAfterRecovery(row.id); this.recover.gone++; } } } } finally { store?.close(); } if (acted) break; } if (!acted) break; } if (this.recover.state === "running") this.recover.state = "done"; this.recover.finishedAt = Date.now(); this.recover.currentScrapeId = null; this.taskHistory.unshift({ name: "Recover Discord previews", at: new Date().toISOString(), detail: `${this.recover.recovered} recovered, ` + `${this.recover.deferred} deferred, ${this.recover.gone} unavailable` }); if (this.taskHistory.length > 10) this.taskHistory.length = 10; this.deps.log?.( `recover: done, ${this.recover.recovered} recovered, ` + `${this.recover.deferred} deferred, ${this.recover.gone} unavailable` ); } startBackfill(chatId?: string): BackfillStatus { this.backfill.start(chatId); return this.backfillStatus(); } pauseBackfill(chatId?: string): BackfillStatus { this.backfill.pause(chatId); return this.backfillStatus(); } /** Log slopbot in on first use. False when it is disabled or the login failed. */ private async ensureSlopbot(): Promise { if (this.bot) return true; const config = loadSlopbotConfig(); if (!config.enabled) return false; if (!this.botStarting) { this.botStarting = startSlopbot(config) .then(client => { this.bot = client; this.botFetcher = slopbotFetcher(client); this.deps.log?.("slopbot connected"); }) .catch(error => { const message = `slopbot login failed: ${String(error)}`; this.deps.log?.(message); this.recordError("slopbot", message); }); } await this.botStarting; return this.bot !== null; } private puppetBackend(): ScrapeBackend { return new PuppetBackend(async (channelId, limit, options) => { const core = this.deps.getCore(); if (!core || !this.deps.isCoreConnected()) { throw new Error("the core bridge is not connected"); } return (await core.fetchMessages(channelId, limit, options)) as { messages?: unknown[] }; }); } /** Combined size of every scrape directory, with the limit settings applied. */ storageStatus(): MediaStorageStatus { const prefs = this.deps.getMediaPrefs(); const byKind: Record = { image: 0, gif: 0, video: 0, audio: 0 }; const scrapes = this.engine.list().map(meta => { const dir = scrapeDir(this.rootDir, meta.id); const bytes = dirSizeBytes(dir); try { const store = new ScrapeStore(dir); try { const sums = store.storedBytesByKind(); for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind]; } finally { store.close(); } } catch { // an unreadable index counts as zero bytes } return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes }; }); const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0); return { usedBytes, limitBytes: prefs.limitBytes, warnPercent: prefs.warnPercent, overrunPercent: prefs.overrunPercent, nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent), byKind, scrapes }; } /** Mark the oldest completed scrapes when the usage is at or over the limit. */ refreshRetention(): MediaStorageStatus { const status = this.storageStatus(); const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent); for (const id of plan.mark) this.engine.setState(id, "pending-delete"); return this.storageStatus(); } private applyRetention(): void { const status = this.storageStatus(); const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent); for (const id of plan.mark) this.engine.setState(id, "pending-delete"); for (const id of plan.delete) this.deleteScrape(id); } deleteScrape(id: string): boolean { try { rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true }); this.engine.forget(id); this.deps.log?.(`deleted scrape ${id}`); return true; } catch (error) { this.deps.log?.(`delete failed for ${id}: ${String(error)}`); return false; } } /** Delete the given scrapes, or every scrape when no IDs are given. */ cleanMedia(ids?: string[]): number { const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id); let deleted = 0; for (const id of targets) { if (this.deleteScrape(id)) deleted++; } return deleted; } async startScrape(input: StartScrapeInput): Promise { if (input.downloadDir && !isAbsolute(input.downloadDir)) { throw new Error("downloadDir must be an absolute path"); } this.applyRetention(); const status = this.storageStatus(); if (status.nearLimit && !input.confirm) { throw new MediaLimitError( `media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`, status.usedBytes, status.limitBytes ); } try { const mode = input.mode ?? "links"; const afterId = input.after ? snowflakeFromIso(input.after) : null; const beforeId = input.before ? snowflakeFromIso(input.before) : null; let backend: ScrapeBackend; if (input.backend === "puppet") { backend = this.puppetBackend(); } else { const available = await this.ensureSlopbot(); const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false); if (input.backend === "slopbot" && (!available || !inGuild)) { throw new Error("slopbot is not connected to that guild"); } const fetcher = this.botFetcher; backend = available && inGuild && fetcher ? new SlopbotBackend(fetcher) : this.puppetBackend(); } // Best effort: one search request gives the total for the range. let estimatedMessages: number | null = null; if (backend.name === "slopbot" && input.guildId && this.bot) { estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, { channelId: input.channelId, guildId: input.guildId, afterId, beforeId }); } return this.engine.start({ channelId: input.channelId, guildId: input.guildId ?? null, mode, backend, downloadDir: input.downloadDir ?? null, afterId, beforeId, fromDate: input.after ?? null, toDate: input.before ?? null, estimatedMessages }); } catch (error) { this.recordError("scrape", error instanceof Error ? error.message : String(error)); throw error; } } stopScrape(id: string): void { this.engine.stop(id); } /** * Title and channel from the source message's Discord embed. Discord keeps the embed data * after a video is deleted, so this is the only title source for gone YouTube videos. */ private async embedMetaFor( scrapeId: string, media: { id: number; url: string; playUrl: string | null } ): Promise<{ title: string; author: string | null } | null> { const meta = this.engine.get(scrapeId); if (!meta) return null; try { const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); let messageId: string | null = null; try { messageId = store.sourceMessageId(media.id); } finally { store.close(); } if (!messageId) return null; const message = await this.fetchSourceMessage(meta, messageId); if (!message) return null; const path = (url: string) => url.split("?")[0]; const embeds: any[] = message.embeds ?? []; const embed = embeds.find( candidate => (candidate?.url && path(candidate.url) === path(media.url)) || (media.playUrl && candidate?.videoUrl && path(candidate.videoUrl) === path(media.playUrl)) || (candidate?.video?.url && media.playUrl && path(candidate.video.url) === path(media.playUrl)) ); const title = embed?.title ?? null; if (!title) return null; const author = embed?.authorName ?? embed?.author?.name ?? null; return { title, author }; } catch (error) { this.deps.log?.(`embed meta failed for ${scrapeId}:${media.id}: ${String(error)}`); return null; } } /** The source message through slopbot REST (guild scrapes) or the puppet bridge. */ private async fetchSourceMessage(meta: ScrapeMeta, messageId: string): Promise { if (this.bot && meta.guildId && meta.backend === "slopbot") { const channel = await this.bot.channels.fetch(meta.channelId); if (channel?.isTextBased()) return await channel.messages.fetch(messageId); return null; } const core = this.deps.getCore(); if (!core || !this.deps.isCoreConnected()) return null; const result = (await core.fetchMessages(meta.channelId, 1, { before: (BigInt(messageId) + 1n).toString() })) as { messages?: Array> }; return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null; } /** Continue a paused scrape from its cursor with the same backend family. */ async resumeScrape(scrapeId: string): Promise { const meta = this.engine.get(scrapeId); if (!meta) return null; let backend: ScrapeBackend; if (meta.backend === "slopbot" && meta.guildId) { const available = await this.ensureSlopbot(); const inGuild = this.bot?.guilds.cache.has(meta.guildId) ?? false; const fetcher = this.botFetcher; if (!available || !inGuild || !fetcher) { throw new Error("slopbot is not connected to that guild"); } backend = new SlopbotBackend(fetcher); } else { backend = this.puppetBackend(); } this.deps.log?.(`resuming scrape ${scrapeId} from cursor ${meta.cursor ?? "start"}`); return this.engine.resume(scrapeId, backend); } listScrapes(): ScrapeMeta[] { return this.engine.list(); } getScrape(id: string): ScrapeMeta | null { return this.engine.get(id); } status(): MediaStatus { const limit = this.engine.rateLimitStatus(); return { slopbot: slopbotStatus(this.bot), scrapes: this.engine.list(), limitedUntil: limit.active ? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString() : null, providerFilters: [...PROVIDER_FILTERS] }; } /** * A servable media source: the local file, or a remote URL refreshed from the source * message (signed CDN links expire). Null when nothing can be resolved. */ async resolveMediaSource( scrapeId: string, itemId: number ): Promise< | { kind: "file"; path: string; contentType: string } | { kind: "remote"; url: string } | null > { const file = this.getMediaFile(scrapeId, itemId); if (file) return { kind: "file", ...file }; const item = this.getMediaItem(scrapeId, itemId); if (!item) return null; return { kind: "remote", url: item.play_url ?? item.url }; } /** Re-fetch the source message for a fresh signed URL. Null when not possible. */ async refreshMediaUrl(scrapeId: string, itemId: number): Promise { const meta = this.engine.get(scrapeId); const item = this.getMediaItem(scrapeId, itemId); if (!meta || !item) return null; return this.refreshUrls(meta, item); } /** * Re-fetch the source message and update expired URLs. Returns the fresh playable URL. * Slopbot scrapes use the bot REST; DM and group-DM scrapes go through the core bridge. */ private async refreshUrls(meta: ScrapeMeta, item: MediaRow): Promise { try { const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); let messageId: string | null = null; try { messageId = store.sourceMessageId(item.id); } finally { store.close(); } if (!messageId) return null; const path = (url: string) => url.split("?")[0]; const saveAndReturn = (update: { url?: string; playUrl?: string | null }, fresh: string) => { const writer = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { writer.updateUrls(item.id, update); } finally { writer.close(); } return fresh; }; if (this.bot && meta.guildId && meta.backend === "slopbot") { const channel = await this.bot.channels.fetch(meta.channelId); if (channel?.isTextBased()) { const message = await channel.messages.fetch(messageId); const attachment = [...message.attachments.values()].find( candidate => path(candidate.url) === path(item.url) ); if (attachment) return saveAndReturn({ url: attachment.url }, attachment.url); const embed = message.embeds.find( candidate => (item.play_url && candidate.video?.url && path(candidate.video.url) === path(item.play_url)) || (candidate.video?.url && path(candidate.url ?? "") === path(item.url)) ); if (embed?.video?.url) { // The direct media link is always fresh; a bare embed.url can be stale. return saveAndReturn( { playUrl: embed.video.url, url: embed.url ?? embed.video.url }, embed.video.url ); } } } // The puppet path: the newest message below id+1 is exactly the target message. const core = this.deps.getCore(); if (core && this.deps.isCoreConnected()) { const result = (await core.fetchMessages(meta.channelId, 1, { before: (BigInt(messageId) + 1n).toString() })) as { messages?: Array> }; const message = result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0]; if (message) { const attachment = (message.attachments ?? []).find( (candidate: any) => candidate?.url && path(candidate.url) === path(item.url) ); if (attachment?.url) return saveAndReturn({ url: attachment.url }, attachment.url); const embed = (message.embeds ?? []).find( (candidate: any) => (item.play_url && candidate?.videoUrl && path(candidate.videoUrl) === path(item.play_url)) || (candidate?.videoUrl && path(candidate.url ?? "") === path(item.url)) ); if (embed?.videoUrl) { // Same as the bot path: keep the canonical url fresh too. return saveAndReturn( { playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl }, embed.videoUrl ); } } } return null; } catch (error) { this.deps.log?.(`refresh failed for ${meta.id}:${item.id}: ${String(error)}`); return null; } } /** Absolute path and content type for a downloaded file, or null. */ getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null { // A custom destination is the user's filesystem, outside the archive: not served. const meta = this.engine.get(scrapeId); if (meta?.downloadDir) return null; const item = this.getMediaItem(scrapeId, itemId); if (!item?.file_path) return null; const extension = item.file_path.split(".").pop()?.toLowerCase() ?? ""; return { path: join(this.rootDir, scrapeId, item.file_path), contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream" }; } /** Absolute path for a cached animated GIF, or null. */ getMediaAnim(scrapeId: string, itemId: number): { path: string; contentType: string } | null { const item = this.getMediaItem(scrapeId, itemId); if (!item?.anim_hash) return null; return { path: join(this.rootDir, scrapeId, "thumbs", `${item.anim_hash}.gif`), contentType: "image/gif" }; } /** Absolute path for a cached preview, or null. */ getMediaThumb(scrapeId: string, itemId: number): { path: string; contentType: string } | null { const item = this.getMediaItem(scrapeId, itemId); if (!item?.preview_hash) return null; const extension = item.preview_ext ?? "webp"; return { path: join(this.rootDir, scrapeId, "thumbs", `${item.preview_hash}.${extension}`), contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream" }; } /** Discord search total for a channel or date range. Slopbot (guild) channels only. */ async countMessages(input: { channelId: string; guildId?: string | null; after?: string | null; before?: string | null; }): Promise<{ count: number | null; reason?: string }> { if (!input.guildId) { return { count: null, reason: "not available for DMs or group chats" }; } const available = await this.ensureSlopbot(); if (!available || !this.bot) { return { count: null, reason: "slopbot is not connected" }; } const count = await countMessages(this.bot.rest as SlopbotRest, { channelId: input.channelId, guildId: input.guildId, afterId: input.after ? snowflakeFromIso(input.after) : null, beforeId: input.before ? snowflakeFromIso(input.before) : null }); return count === null ? { count: null, reason: "search unavailable or rate limited" } : { count }; } private recordError(source: string, message: string): void { this.errors.unshift({ at: new Date().toISOString(), source, message }); if (this.errors.length > 50) this.errors.length = 50; } /** Bot and scrape errors, newest first. */ listErrors(): MediaErrorEntry[] { return [...this.errors]; } clearErrors(): number { const count = this.errors.length; this.errors.length = 0; return count; } /** Item counts for the status row, under the same filters as queryMedia. */ stats(options: { kind?: MediaKind; source?: string; channelId?: string; authorId?: string; authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; excludeExts?: string[]; excludeProviders?: string[]; excludeSources?: string[]; includeProviders?: string[]; includeSources?: string[]; titleQuery?: string; titleExact?: boolean; sizeLessThan?: number; sizeGreaterThan?: number; lengthLessThan?: number; lengthGreaterThan?: number; sort?: MediaSort; aspect?: string; resolution?: string; after?: string; before?: string; } = {}): Record { // Same richest-per-URL dedupe as queryMedia. const candidates = this.collectCandidates(options); const totals: Record< MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number } > = { image: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }, gif: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }, video: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }, audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 } }; for (const { row } of candidates) { const bucket = totals[row.kind]; bucket.total++; if (row.downloaded === 1) { bucket.downloaded++; } else { bucket.notDownloaded++; if (row.preview_hash !== null) bucket.cached++; } } return totals; } /** Preview failures across scrapes with their reasons, newest scrape first. */ previewFailures( limit = 50 ): Array<{ scrapeId: string; itemId: number; kind: MediaKind; provider: string | null; url: string; error: string; }> { const results: Array<{ scrapeId: string; itemId: number; kind: MediaKind; provider: string | null; url: string; error: string; }> = []; for (const meta of this.engine.list()) { if (results.length >= limit) break; const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { for (const row of store.previewFailures(limit - results.length)) { results.push({ scrapeId: meta.id, itemId: row.id, ...row }); } } finally { store.close(); } } return results; } /** Distinct posters across every scrape, biggest first. */ listAuthors(): Array<{ id: string | null; name: string | null; count: number }> { const totals = new Map(); for (const meta of this.engine.list()) { if (meta.state === "deleted") continue; const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { for (const author of store.authors()) { const key = author.id ?? author.name ?? "unknown"; const current = totals.get(key); if (current) current.count += author.count; else totals.set(key, { ...author }); } } finally { store.close(); } } return [...totals.values()].sort((a, b) => b.count - a.count); } /** One media row inside a scrape, with its scrape and channel for context. */ getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { const meta = this.engine.get(scrapeId); if (!meta) return null; const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); try { const row = store.getMedia(itemId); return row ? { ...row, scrapeId, channelId: meta.channelId } : null; } finally { store.close(); } } /** * One entry per URL across every scrape. Scrapes are visited newest first, so a newer row * wins by default; an older row replaces it only when it is richer (downloaded, or with a * cached preview). Without this, a stopped or partial scrape hides usable previews. */ private collectCandidates(options: { kind?: MediaKind; source?: string; scrapeId?: string; channelId?: string; authorId?: string; authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; excludeExts?: string[]; excludeProviders?: string[]; excludeSources?: string[]; includeProviders?: string[]; includeSources?: string[]; titleQuery?: string; titleExact?: boolean; sizeLessThan?: number; sizeGreaterThan?: number; lengthLessThan?: number; lengthGreaterThan?: number; aspect?: string; resolution?: string; after?: string; before?: string; sort?: MediaSort; }): Array<{ meta: ScrapeMeta; row: MediaRow }> { const candidates = new Map(); for (const meta of this.engine.list()) { if (meta.state === "deleted") continue; if (options.scrapeId && meta.id !== options.scrapeId) continue; if (options.channelId && meta.channelId !== options.channelId) continue; if (options.origin === "dm" && meta.guildId) continue; if (options.origin === "guild" && !meta.guildId) continue; const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { let storeOffset = 0; for (;;) { const rows = store.queryMedia({ kind: options.kind, source: options.source, authorId: options.authorId, authorIds: options.authorIds, cachedOnly: options.cachedOnly, exts: options.exts, excludeHosts: options.excludeHosts, excludeExts: options.excludeExts, excludeProviders: options.excludeProviders, excludeSources: options.excludeSources, includeProviders: options.includeProviders, includeSources: options.includeSources, titleQuery: options.titleQuery, titleExact: options.titleExact, sizeLessThan: options.sizeLessThan, sizeGreaterThan: options.sizeGreaterThan, lengthLessThan: options.lengthLessThan, lengthGreaterThan: options.lengthGreaterThan, aspect: options.aspect, resolution: options.resolution, after: options.after, before: options.before, limit: 500, offset: storeOffset }); if (!rows.length) break; storeOffset += rows.length; for (const row of rows) { const current = candidates.get(row.url); if (!current) { candidates.set(row.url, { meta, row }); continue; } if ( betterCandidate( { downloaded: current.row.downloaded === 1, hasPreview: current.row.preview_hash !== null }, { downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null } ) ) { candidates.set(row.url, { meta, row }); } } if (rows.length < 500) break; } } finally { store.close(); } } return sortCandidates([...candidates.values()], options.sort ?? "newest"); } /** Read media rows across every scrape, richest per URL, newest scrape first. */ queryMedia( options: { kind?: MediaKind; source?: string; scrapeId?: string; channelId?: string; authorId?: string; authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; excludeExts?: string[]; excludeProviders?: string[]; excludeSources?: string[]; includeProviders?: string[]; includeSources?: string[]; titleQuery?: string; titleExact?: boolean; sizeLessThan?: number; sizeGreaterThan?: number; lengthLessThan?: number; lengthGreaterThan?: number; sort?: MediaSort; aspect?: string; resolution?: string; after?: string; before?: string; limit?: number; offset?: number; } = {} ): MediaItemResult[] { const limit = Math.max(1, Math.min(options.limit ?? 60, 500)); const offset = Math.max(0, options.offset ?? 0); return this.collectCandidates(options) .slice(offset, offset + limit) .map(({ meta, row }) => ({ scrapeId: meta.id, itemId: row.id, channelId: meta.channelId, guildId: meta.guildId, messageId: row.message_id, messageUrl: messageUrlFor(meta.guildId, meta.channelId, row.message_id), kind: row.kind, source: row.source, authorId: row.author_id, authorName: row.author_name, postedAt: row.posted_at, width: row.width, height: row.height, aspectRatio: row.aspect_ratio, contentType: row.content_type, durationSecs: row.duration_secs, url: row.url, playUrl: row.play_url, remotePreviewUrl: row.preview_url, previewError: row.preview_error, titleError: row.title_error, animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null, localUrl: row.file_path && !meta.downloadDir ? `/api/media/${meta.id}/${row.id}/file` : null, thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, provider: row.provider, downloaded: row.downloaded === 1, title: row.title, titleAuthor: row.title_author, createdAt: row.created_at })); } }