/* * Media archive unit tests. No Discord, no network: a fake backend and a temp DB root. * Run with: pnpm test:media (from poc/). */ import assert from "node:assert/strict"; import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js"; import type { CoreAttachment, CoreMessage } from "../src/types.js"; let failures = 0; async function test(name: string, fn: () => void | Promise): Promise { try { await fn(); console.log(`ok - ${name}`); } catch (error) { failures++; console.error(`not ok - ${name}`); console.error(error); } } function message(id: string, extra: Partial = {}): CoreMessage { return { id, author: { id: "1", username: "tester", avatar: null }, content: "", timestamp: "2026-09-16T00:00:00.000Z", editedAt: null, replyTo: null, attachments: [], embeds: [], stickerItems: [], forwarded: [], voiceMessage: false, ...extra }; } function attachment(extra: Partial = {}): CoreAttachment { const filename = extra.filename ?? "file.bin"; return { id: null, filename, url: `https://cdn.discordapp.com/attachments/1/2/${filename}`, contentType: null, size: null, width: null, height: null, durationSecs: null, ...extra }; } class FakeBackend implements ScrapeBackend { readonly name = "puppet" as const; readonly calls: Array<{ before: string | null; limit: number }> = []; constructor(private readonly pages: Array) {} async fetchPage(_channelId: string, before: string | null, limit: number): Promise { this.calls.push({ before, limit }); const page = this.pages.shift(); if (!page) return { messages: [] }; if (page instanceof Error) throw page; return page; } } function engineFor(root: string, backend: ScrapeBackend, extra: Record = {}) { return new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, ...extra }); } void (async () => { const root = mkdtempSync(join(tmpdir(), "veskslop-media-")); await test("extract classifies attachments, embeds, and stickers", () => { const items = extractMedia( message("100", { attachments: [ attachment({ filename: "cat.png", contentType: "image/png" }), attachment({ filename: "loop.gif" }), attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234 }), attachment({ filename: "note.ogg", durationSecs: 3.2 }) ], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: "x", provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" }, { type: "gifv", url: "https://tenor.com/view/x", title: null, provider: "Tenor", videoUrl: null, imageUrl: null, thumbnailUrl: "https://media.tenor.com/x.gif" } ], stickerItems: [ { id: "55", name: "wave", formatType: 4 }, { id: "56", name: "lottie", formatType: 3 } ] }) ); const kinds = items.map(item => item.kind).sort(); assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]); const youtube = items.find(item => item.url.includes("youtube.com/watch")); assert.equal(youtube?.kind, "video"); assert.equal(youtube?.provider, "YouTube"); assert.equal(youtube?.downloaded, false); assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif"); assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif"); assert.equal(items.some(item => item.url.includes("lottie")), false); assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2); }); await test("extract finds media in forwarded snapshots", () => { const items = extractMedia( message("101", { forwarded: [ { content: "", timestamp: null, attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })], embeds: [], stickerItems: [], voiceMessage: false } ] }) ); assert.equal(items.length, 1); assert.equal(items[0].kind, "video"); }); await test("extract treats a voice message as audio", () => { const items = extractMedia( message("102", { voiceMessage: true, attachments: [attachment({ filename: "voice-message.ogg" })] }) ); assert.equal(items.length, 1); assert.equal(items[0].kind, "audio"); }); await test("engine walks pages, persists meta, and stores media", async () => { const backend = new FakeBackend([ { messages: [ message("100", { attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })] }), message("101") ] }, { messages: [message("99")] } ]); const engine = engineFor(root, backend); const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.pages, 2); assert.equal(meta?.messages, 3); assert.equal(meta?.cursor, "99"); assert.equal(meta?.media.image, 1); assert.equal(backend.calls[0].limit, 2); assert.equal(backend.calls[1].before, "100"); const dir = join(root, started.id); assert.equal(existsSync(join(dir, "scrape.json")), true); assert.equal(existsSync(join(dir, "index.db")), true); assert.ok(dirSizeBytes(dir) > 0); const stored = new ScrapeStore(dir); assert.equal(stored.counts().image, 1); assert.equal(stored.queryMedia({ kind: "image" }).length, 1); assert.equal(stored.queryMedia({ kind: "video" }).length, 0); assert.equal(stored.storedBytesByKind().image, 0); stored.close(); assert.equal(listScrapes(root).length, 1); }); await test("engine backs off and retries with retryAfterMs", async () => { const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 }); const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]); const sleeps: number[] = []; const engine = engineFor(root, backend, { maxRetries: 3, sleep: async (ms: number) => { sleeps.push(ms); } }); const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.errors, 2); assert.deepEqual(sleeps, [250, 3000]); }); await test("engine pauses when the page budget is reached", async () => { const backend = new FakeBackend([ { messages: [message("200"), message("201")] }, { messages: [message("199"), message("198")] } ]); const engine = engineFor(root, backend, { pagesPerDay: 1 }); const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "paused"); assert.equal(meta?.pages, 1); assert.equal(backend.calls.length, 1); }); await test("engine stop pauses after the current page", async () => { const backend = new FakeBackend([ { messages: [message("300"), message("301")] }, { messages: [message("299")] } ]); const engine = engineFor(root, backend); const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend }); engine.stop(started.id); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "paused"); assert.equal(meta?.pages, 1); }); await test("puppet backend passes before and limit to the core", async () => { const calls: Array> = []; const backend = new PuppetBackend(async (channelId, limit, options) => { calls.push({ channelId, limit, ...options }); return { messages: [message("50")] }; }); const page = await backend.fetchPage("123", "60", 5); assert.equal(page.messages.length, 1); assert.equal(page.messages[0].id, "50"); assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" }); }); await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, { id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 }, { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 } ]; const plan = planRetention(scrapes, 1000, 100); assert.deepEqual(plan.mark, ["old", "new"]); assert.deepEqual(plan.delete, []); }); await test("retention deletes marked scrapes past the overrun allowance", () => { const scrapes = [ { id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, { id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 }, { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 } ]; const plan = planRetention(scrapes, 1000, 5); assert.deepEqual(plan.delete, ["old", "new"]); assert.deepEqual(plan.mark, []); }); await test("retention leaves usage under the limit alone", () => { const plan = planRetention( [{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }], 1000, 5 ); assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 }); assert.equal(isNearLimit(900, 1000, 90), true); assert.equal(isNearLimit(899, 1000, 90), false); }); rmSync(root, { recursive: true, force: true }); if (failures > 0) { console.error(`${failures} test(s) failed`); process.exit(1); } console.log("all media tests passed"); })();