/* * Media archive unit tests. No Discord, no network: a fake backend and a temp DB root. * Run with: pnpm test:media (from poc/). */ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js"; import type { CoreAttachment, CoreMessage } from "../src/types.js"; let failures = 0; async function test(name: string, fn: () => void | Promise): Promise { try { await fn(); console.log(`ok - ${name}`); } catch (error) { failures++; console.error(`not ok - ${name}`); console.error(error); } } function message(id: string, extra: Partial = {}): CoreMessage { return { id, author: { id: "1", username: "tester", avatar: null }, content: "", timestamp: "2026-09-16T00:00:00.000Z", editedAt: null, replyTo: null, attachments: [], embeds: [], stickerItems: [], forwarded: [], voiceMessage: false, ...extra }; } function attachment(extra: Partial = {}): CoreAttachment { const filename = extra.filename ?? "file.bin"; return { id: null, filename, url: `https://cdn.discordapp.com/attachments/1/2/${filename}`, contentType: null, size: null, width: null, height: null, durationSecs: null, ...extra }; } class FakeBackend implements ScrapeBackend { readonly name = "puppet" as const; readonly calls: Array<{ before: string | null; limit: number }> = []; constructor(private readonly pages: Array) {} async fetchPage(_channelId: string, before: string | null, limit: number): Promise { this.calls.push({ before, limit }); const page = this.pages.shift(); if (!page) return { messages: [] }; if (page instanceof Error) throw page; return page; } } function engineFor(root: string, backend: ScrapeBackend, extra: Record = {}) { return new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: false, ...extra }); } void (async () => { const root = mkdtempSync(join(tmpdir(), "veskslop-media-")); await test("extract classifies attachments, embeds, and stickers", () => { const items = extractMedia( message("100", { attachments: [ attachment({ filename: "cat.png", contentType: "image/png" }), attachment({ filename: "loop.gif" }), attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234 }), attachment({ filename: "note.ogg", durationSecs: 3.2 }) ], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: "x", provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" }, { type: "gifv", url: "https://tenor.com/view/x", title: null, provider: "Tenor", videoUrl: null, imageUrl: null, thumbnailUrl: "https://media.tenor.com/x.gif" } ], stickerItems: [ { id: "55", name: "wave", formatType: 4 }, { id: "56", name: "lottie", formatType: 3 } ] }) ); const kinds = items.map(item => item.kind).sort(); assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]); const youtube = items.find(item => item.url.includes("youtube.com/watch")); assert.equal(youtube?.kind, "video"); assert.equal(youtube?.provider, "YouTube"); assert.equal(youtube?.downloaded, false); assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif"); assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif"); assert.equal(items.some(item => item.url.includes("lottie")), false); assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2); }); await test("extract finds media in forwarded snapshots", () => { const items = extractMedia( message("101", { forwarded: [ { content: "", timestamp: null, attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })], embeds: [], stickerItems: [], voiceMessage: false } ] }) ); assert.equal(items.length, 1); assert.equal(items[0].kind, "video"); }); await test("extract treats a voice message as audio", () => { const items = extractMedia( message("102", { voiceMessage: true, attachments: [attachment({ filename: "voice-message.ogg" })] }) ); assert.equal(items.length, 1); assert.equal(items[0].kind, "audio"); }); await test("engine walks pages, persists meta, and stores media", async () => { const backend = new FakeBackend([ { messages: [ message("100", { attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })] }), message("101") ] }, { messages: [message("99")] } ]); const engine = engineFor(root, backend); const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.pages, 2); assert.equal(meta?.messages, 3); assert.equal(meta?.cursor, "99"); assert.equal(meta?.media.image, 1); assert.ok(meta?.finishedAt); assert.equal(backend.calls[0].limit, 2); assert.equal(backend.calls[1].before, "100"); const dir = join(root, started.id); assert.equal(existsSync(join(dir, "scrape.json")), true); assert.equal(existsSync(join(dir, "index.db")), true); assert.ok(dirSizeBytes(dir) > 0); const stored = new ScrapeStore(dir); assert.equal(stored.counts().image, 1); assert.equal(stored.queryMedia({ kind: "image" }).length, 1); assert.equal(stored.queryMedia({ kind: "video" }).length, 0); assert.equal(stored.storedBytesByKind().image, 0); const row = stored.queryMedia({ kind: "image" })[0]; assert.equal(row.author_name, "tester"); assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z"); stored.close(); assert.equal(listScrapes(root).length, 1); }); await test("engine backs off and retries with retryAfterMs", async () => { const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 }); const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]); const sleeps: number[] = []; const engine = engineFor(root, backend, { maxRetries: 3, sleep: async (ms: number) => { sleeps.push(ms); } }); const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.errors, 2); assert.deepEqual(sleeps, [250, 3000]); }); await test("engine pauses when the page budget is reached", async () => { const backend = new FakeBackend([ { messages: [message("200"), message("201")] }, { messages: [message("199"), message("198")] } ]); const engine = engineFor(root, backend, { pagesPerDay: 1 }); const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "paused"); assert.equal(meta?.pages, 1); assert.equal(backend.calls.length, 1); }); await test("engine stop pauses after the current page", async () => { const backend = new FakeBackend([ { messages: [message("300"), message("301")] }, { messages: [message("299")] } ]); const engine = engineFor(root, backend); const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend }); engine.stop(started.id); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "paused"); assert.equal(meta?.pages, 1); }); await test("puppet backend passes before and limit to the core", async () => { const calls: Array> = []; const backend = new PuppetBackend(async (channelId, limit, options) => { calls.push({ channelId, limit, ...options }); return { messages: [message("50")] }; }); const page = await backend.fetchPage("123", "60", 5); assert.equal(page.messages.length, 1); assert.equal(page.messages[0].id, "50"); assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" }); }); await test("download stores bytes by hash and skips expired URLs", async () => { const dir = join(root, "download-test"); mkdirSync(dir, { recursive: true }); const ok = await downloadOne( { id: 1, url: "https://cdn.discordapp.com/attachments/1/2/note.txt", kind: "image" }, dir, { fetchImpl: async () => new Response("hello", { status: 200, headers: { "content-type": "text/plain" } }) } ); assert.equal(ok.state, "downloaded"); assert.equal(ok.hash, createHash("sha256").update("hello").digest("hex")); assert.equal(existsSync(join(dir, `${ok.hash}.txt`)), true); const expired = await downloadOne( { id: 2, url: "https://cdn.discordapp.com/attachments/1/2/gone.png", kind: "image" }, dir, { fetchImpl: async () => new Response("no", { status: 404 }) } ); assert.equal(expired.state, "skipped"); const large = await downloadOne( { id: 3, url: "https://cdn.discordapp.com/attachments/1/2/big.mp4", kind: "video" }, dir, { fetchImpl: async () => new Response("x", { status: 200, headers: { "content-length": "999999999" } }) } ); assert.equal(large.state, "skipped"); }); await test("download retries a 5xx and then succeeds", async () => { const dir = join(root, "download-retry"); mkdirSync(dir, { recursive: true }); let calls = 0; const outcome = await downloadOne( { id: 4, url: "https://cdn.discordapp.com/attachments/1/2/flaky.png", kind: "image" }, dir, { maxRetries: 2, sleep: async () => {}, fetchImpl: async () => { calls++; return calls === 1 ? new Response("boom", { status: 500 }) : new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }); } } ); assert.equal(outcome.state, "downloaded"); assert.equal(calls, 2); }); await test("engine downloads attachments in download mode", async () => { const backend = new FakeBackend([ { messages: [ message("500", { attachments: [attachment({ filename: "pic.png", contentType: "image/png" })], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: null, provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" } ] }) ] } ]); const fetched: string[] = []; const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: true, download: { fetchImpl: async (input: any) => { fetched.push(String(input)); return new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }); }, minGapMs: 0, maxGapMs: 0, sleep: async () => {} } }); const started = await engine.start({ channelId: "127", guildId: null, mode: "download", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.downloaded, 1); assert.equal(meta?.previewed, 2); assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false); assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true); const store = new ScrapeStore(join(root, started.id)); const row = store.queryMedia({ kind: "image" })[0]; assert.equal(row.downloaded, 1); assert.ok(row.file_path); assert.ok((row.file_bytes ?? 0) > 0); const embed = store.queryMedia({ kind: "video" })[0]; assert.equal(embed.downloaded, 0); assert.equal(embed.file_path, null); store.close(); assert.equal(existsSync(join(root, started.id, row.file_path)), true); }); await test("aspect and resolution presets have fixed buckets", () => { assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95"); assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05"); assert.equal(aspectCondition("nope"), null); assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260"); assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)"); assert.equal(resolutionCondition("nope"), null); }); await test("store filters by source, kind, author, ratio, resolution, and dates", () => { const dir = join(root, "store-source"); mkdirSync(dir, { recursive: true }); const store = new ScrapeStore(dir); store.insertPage([ message("700", { attachments: [ attachment({ filename: "pic.png", contentType: "image/png", width: 640, height: 360 }) ], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: null, provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: null } ] }) ]); assert.equal(store.queryMedia({ source: "attachment" }).length, 1); assert.equal(store.queryMedia({ source: "embed" }).length, 1); assert.equal(store.queryMedia({ source: "sticker" }).length, 0); assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1); assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0); assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1); assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0); assert.equal(store.queryMedia({ resolution: "sd" }).length, 1); assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0); assert.equal(store.queryMedia({ authorId: "1" }).length, 2); assert.equal(store.queryMedia({ authorId: "999" }).length, 0); assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2); assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0); const [wide] = store.queryMedia({ kind: "image" }); assert.equal(wide.aspect_ratio, 1.778); store.close(); }); await test("engine probes dimensions when the metadata lacks them", async () => { const backend = new FakeBackend([ { messages: [ message("701", { attachments: [attachment({ filename: "probe.png", contentType: "image/png" })] }) ] } ]); const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: false, probe: async () => ({ width: 1080, height: 1920 }), download: { fetchImpl: async () => new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }), minGapMs: 0, maxGapMs: 0, sleep: async () => {} } }); const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend }); await engine.processQueue(); const store = new ScrapeStore(join(root, started.id)); const row = store.queryMedia({ kind: "image" })[0]; assert.equal(row.width, 1080); assert.equal(row.height, 1920); assert.equal(row.aspect_ratio, 0.563); store.close(); }); await test("preview urls resize Discord and pass embed thumbnails through", () => { assert.equal( previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }), "https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp" ); assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null); assert.equal( previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video", previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" }), "https://i.ytimg.com/vi/x/hqdefault.jpg" ); }); await test("preview downloads are written by hash", async () => { const dir = join(root, "preview-test"); mkdirSync(dir, { recursive: true }); const outcome = await downloadPreview( { id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image", source: "attachment", preview_url: null }, dir, { fetchImpl: async () => new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }) } ); assert.equal(outcome.state, "downloaded"); assert.equal(outcome.path, `${outcome.hash}.webp`); assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true); }); await test("embed thumbnails download only when enabled", async () => { const dir = join(root, "preview-embed"); mkdirSync(dir, { recursive: true }); const media = { id: 11, url: "https://www.youtube.com/watch?v=x", kind: "video" as const, source: "embed", preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg" }; const calls: string[] = []; const fetchImpl = async (input: any) => { calls.push(String(input)); return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } }); }; const off = await downloadPreview(media, dir, { fetchImpl }); assert.equal(off.state, "skipped"); assert.equal(calls.length, 0); const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true }); assert.equal(on.state, "downloaded"); assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg"); assert.equal(calls.some(url => url.includes("youtube.com/watch")), false); }); await test("engine caches previews in links mode, including embed thumbnails", async () => { const backend = new FakeBackend([ { messages: [ message("600", { attachments: [attachment({ filename: "prev.png", contentType: "image/png" })], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: null, provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" } ] }) ] } ]); const fetched: string[] = []; const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: true, download: { fetchImpl: async (input: any) => { fetched.push(String(input)); return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }); }, minGapMs: 0, maxGapMs: 0, sleep: async () => {} } }); const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.previewed, 2); assert.equal(fetched.some(url => url.includes("ytimg.com")), true); assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false); const store = new ScrapeStore(join(root, started.id)); const row = store.queryMedia({ kind: "image" })[0]; assert.ok(row.preview_hash); assert.ok((row.preview_bytes ?? 0) > 0); store.close(); assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true); }); await test("custom download destination keeps files off-archive and copies embed previews", async () => { const dest = join(root, "custom-dest"); mkdirSync(dest, { recursive: true }); const backend = new FakeBackend([ { messages: [ message("702", { attachments: [attachment({ filename: "offsite.png", contentType: "image/png" })], embeds: [ { type: "video", url: "https://www.youtube.com/watch?v=x", title: null, provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" } ] }) ] } ]); const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: true, download: { fetchImpl: async () => new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }), minGapMs: 0, maxGapMs: 0, sleep: async () => {} } }); const started = await engine.start({ channelId: "130", guildId: null, mode: "download", backend, downloadDir: dest }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.downloadDir, dest); assert.equal(meta?.downloaded, 1); const store = new ScrapeStore(join(root, started.id)); const file = store.queryMedia({ kind: "image" })[0]; const filePath = file.file_path; assert.ok(filePath && filePath.startsWith(dest)); assert.equal(existsSync(filePath), true); assert.equal(existsSync(join(root, started.id, "media", filePath.split("/").pop() ?? "")), false); const embed = store.queryMedia({ kind: "video" })[0]; assert.ok(embed.preview_hash); assert.equal( existsSync(join(dest, started.id, "thumbnails", `${embed.preview_hash}.webp`)), true ); assert.equal(existsSync(join(root, started.id, "thumbs", `${embed.preview_hash}.webp`)), true); store.close(); }); await test("snowflake helpers round-trip dates", () => { const id = snowflakeFromIso("2026-09-16T00:00:00.000Z"); assert.ok(id); assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z"); assert.equal(snowflakeFromIso("not a date"), null); assert.equal(isoFromSnowflake("nope"), null); }); await test("engine honors a from-date boundary", async () => { const backend = new FakeBackend([ { messages: [message("1000"), message("900"), message("800")] } ]); const engine = new ScrapeEngine({ rootDir: root, pageSize: 10, pageMinMs: 0, pageMaxMs: 0, sleep: async () => {}, random: () => 0, log: () => {}, previews: false }); const started = await engine.start({ channelId: "131", guildId: null, mode: "links", backend, afterId: "900" }); await engine.processQueue(); const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.messages, 1); assert.equal(meta?.afterId, "900"); assert.equal(backend.calls.length, 1); const store = new ScrapeStore(join(root, started.id)); assert.equal(store.queryMedia({}).length, 0); store.close(); }); await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, { id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 }, { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 } ]; const plan = planRetention(scrapes, 1000, 100); assert.deepEqual(plan.mark, ["old", "new"]); assert.deepEqual(plan.delete, []); }); await test("retention deletes marked scrapes past the overrun allowance", () => { const scrapes = [ { id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, { id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 }, { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 } ]; const plan = planRetention(scrapes, 1000, 5); assert.deepEqual(plan.delete, ["old", "new"]); assert.deepEqual(plan.mark, []); }); await test("retention leaves usage under the limit alone", () => { const plan = planRetention( [{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }], 1000, 5 ); assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 }); assert.equal(isNearLimit(900, 1000, 90), true); assert.equal(isNearLimit(899, 1000, 90), false); }); rmSync(root, { recursive: true, force: true }); if (failures > 0) { console.error(`${failures} test(s) failed`); process.exit(1); } console.log("all media tests passed"); })();