1154 lines
43 KiB
TypeScript
1154 lines
43 KiB
TypeScript
/*
|
|
* Media archive unit tests. No Discord, no network: a fake backend and a temp DB root.
|
|
* Run with: pnpm test:media (from poc/).
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import {
|
|
downloadOne,
|
|
downloadPreview,
|
|
gifPoster,
|
|
previewUrlFor,
|
|
remoteFrame,
|
|
videoPoster
|
|
} from "../src/download.js";
|
|
import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
|
|
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
|
import { extractMedia } from "../src/extract.js";
|
|
import { PuppetBackend } from "../src/puppet-backend.js";
|
|
import { betterCandidate } from "../src/dedupe.js";
|
|
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
|
|
import { isNearLimit, planRetention } from "../src/retention.js";
|
|
import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
|
|
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
|
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
|
|
|
let failures = 0;
|
|
|
|
async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
|
|
try {
|
|
await fn();
|
|
console.log(`ok - ${name}`);
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(`not ok - ${name}`);
|
|
console.error(error);
|
|
}
|
|
}
|
|
|
|
function message(id: string, extra: Partial<CoreMessage> = {}): CoreMessage {
|
|
return {
|
|
id,
|
|
author: { id: "1", username: "tester", avatar: null },
|
|
content: "",
|
|
timestamp: "2026-09-16T00:00:00.000Z",
|
|
editedAt: null,
|
|
replyTo: null,
|
|
attachments: [],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
forwarded: [],
|
|
voiceMessage: false,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
function attachment(extra: Partial<CoreAttachment> = {}): CoreAttachment {
|
|
const filename = extra.filename ?? "file.bin";
|
|
return {
|
|
id: null,
|
|
filename,
|
|
url: `https://cdn.discordapp.com/attachments/1/2/${filename}`,
|
|
contentType: null,
|
|
size: null,
|
|
width: null,
|
|
height: null,
|
|
durationSecs: null,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
class FakeBackend implements ScrapeBackend {
|
|
readonly name = "puppet" as const;
|
|
readonly calls: Array<{ before: string | null; limit: number }> = [];
|
|
|
|
constructor(private readonly pages: Array<FetchPage | Error>) {}
|
|
|
|
async fetchPage(_channelId: string, before: string | null, limit: number): Promise<FetchPage> {
|
|
this.calls.push({ before, limit });
|
|
const page = this.pages.shift();
|
|
if (!page) return { messages: [] };
|
|
if (page instanceof Error) throw page;
|
|
return page;
|
|
}
|
|
}
|
|
|
|
function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, unknown> = {}) {
|
|
return new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
...extra
|
|
});
|
|
}
|
|
|
|
void (async () => {
|
|
const root = mkdtempSync(join(tmpdir(), "veskslop-media-"));
|
|
|
|
await test("extract classifies attachments, embeds, and stickers", () => {
|
|
const items = extractMedia(
|
|
message("100", {
|
|
attachments: [
|
|
attachment({ filename: "cat.png", contentType: "image/png" }),
|
|
attachment({ filename: "loop.gif" }),
|
|
attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234, durationSecs: 0 }),
|
|
attachment({ filename: "note.ogg", durationSecs: 3.2 })
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: "x",
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
},
|
|
{
|
|
type: "gifv",
|
|
url: "https://tenor.com/view/x",
|
|
title: null,
|
|
provider: "Tenor",
|
|
videoUrl: null,
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://media.tenor.com/x.gif"
|
|
}
|
|
],
|
|
stickerItems: [
|
|
{ id: "55", name: "wave", formatType: 4 },
|
|
{ id: "56", name: "lottie", formatType: 3 }
|
|
]
|
|
})
|
|
);
|
|
|
|
const kinds = items.map(item => item.kind).sort();
|
|
assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]);
|
|
const youtube = items.find(item => item.url.includes("youtube.com/watch"));
|
|
assert.equal(youtube?.kind, "video");
|
|
assert.equal(youtube?.provider, "YouTube");
|
|
assert.equal(youtube?.downloaded, false);
|
|
assert.equal(youtube?.playUrl, "https://www.youtube.com/embed/x");
|
|
assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif");
|
|
assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
|
|
assert.equal(items.some(item => item.url.includes("lottie")), false);
|
|
assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2);
|
|
assert.equal(items.find(item => item.url.endsWith("clip.mp4"))?.durationSecs, null);
|
|
});
|
|
|
|
await test("extract finds media in forwarded snapshots", () => {
|
|
const items = extractMedia(
|
|
message("101", {
|
|
forwarded: [
|
|
{
|
|
content: "",
|
|
timestamp: null,
|
|
attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
voiceMessage: false
|
|
}
|
|
]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "video");
|
|
});
|
|
|
|
await test("extract treats a voice message as audio", () => {
|
|
const items = extractMedia(
|
|
message("102", {
|
|
voiceMessage: true,
|
|
attachments: [attachment({ filename: "voice-message.ogg" })]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "audio");
|
|
});
|
|
|
|
await test("engine walks pages, persists meta, and stores media", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("100", {
|
|
attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })]
|
|
}),
|
|
message("101")
|
|
]
|
|
},
|
|
{ messages: [message("99")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.pages, 2);
|
|
assert.equal(meta?.messages, 3);
|
|
assert.equal(meta?.cursor, "99");
|
|
assert.equal(meta?.media.image, 1);
|
|
assert.ok(meta?.finishedAt);
|
|
assert.equal(backend.calls[0].limit, 2);
|
|
assert.equal(backend.calls[1].before, "100");
|
|
|
|
const dir = join(root, started.id);
|
|
assert.equal(existsSync(join(dir, "scrape.json")), true);
|
|
assert.equal(existsSync(join(dir, "index.db")), true);
|
|
assert.ok(dirSizeBytes(dir) > 0);
|
|
|
|
const stored = new ScrapeStore(dir);
|
|
assert.equal(stored.counts().image, 1);
|
|
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
|
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
|
assert.equal(stored.storedBytesByKind().image, 0);
|
|
const row = stored.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.author_name, "tester");
|
|
assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
|
|
stored.close();
|
|
|
|
assert.equal(listScrapes(root).length, 1);
|
|
});
|
|
|
|
await test("engine backs off and retries with retryAfterMs", async () => {
|
|
const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 });
|
|
const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]);
|
|
const sleeps: number[] = [];
|
|
|
|
const engine = engineFor(root, backend, {
|
|
maxRetries: 3,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
}
|
|
});
|
|
const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.errors, 2);
|
|
assert.deepEqual(sleeps, [250, 3000]);
|
|
});
|
|
|
|
await test("engine pauses when the page budget is reached", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("200"), message("201")] },
|
|
{ messages: [message("199"), message("198")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend, { pagesPerDay: 1 });
|
|
const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
assert.equal(backend.calls.length, 1);
|
|
});
|
|
|
|
await test("engine stop pauses after the current page", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("300"), message("301")] },
|
|
{ messages: [message("299")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend });
|
|
engine.stop(started.id);
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
});
|
|
|
|
await test("puppet backend passes before and limit to the core", async () => {
|
|
const calls: Array<Record<string, unknown>> = [];
|
|
const backend = new PuppetBackend(async (channelId, limit, options) => {
|
|
calls.push({ channelId, limit, ...options });
|
|
return { messages: [message("50")] };
|
|
});
|
|
|
|
const page = await backend.fetchPage("123", "60", 5);
|
|
assert.equal(page.messages.length, 1);
|
|
assert.equal(page.messages[0].id, "50");
|
|
assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
|
|
});
|
|
|
|
await test("download stores bytes by hash and skips expired URLs", async () => {
|
|
const dir = join(root, "download-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const ok = await downloadOne(
|
|
{ id: 1, url: "https://cdn.discordapp.com/attachments/1/2/note.txt", kind: "image" },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("hello", { status: 200, headers: { "content-type": "text/plain" } })
|
|
}
|
|
);
|
|
assert.equal(ok.state, "downloaded");
|
|
assert.equal(ok.hash, createHash("sha256").update("hello").digest("hex"));
|
|
assert.equal(existsSync(join(dir, `${ok.hash}.txt`)), true);
|
|
|
|
const expired = await downloadOne(
|
|
{ id: 2, url: "https://cdn.discordapp.com/attachments/1/2/gone.png", kind: "image" },
|
|
dir,
|
|
{ fetchImpl: async () => new Response("no", { status: 404 }) }
|
|
);
|
|
assert.equal(expired.state, "skipped");
|
|
|
|
const large = await downloadOne(
|
|
{ id: 3, url: "https://cdn.discordapp.com/attachments/1/2/big.mp4", kind: "video" },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("x", { status: 200, headers: { "content-length": "999999999" } })
|
|
}
|
|
);
|
|
assert.equal(large.state, "skipped");
|
|
});
|
|
|
|
await test("download retries a 5xx and then succeeds", async () => {
|
|
const dir = join(root, "download-retry");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
let calls = 0;
|
|
const outcome = await downloadOne(
|
|
{ id: 4, url: "https://cdn.discordapp.com/attachments/1/2/flaky.png", kind: "image" },
|
|
dir,
|
|
{
|
|
maxRetries: 2,
|
|
sleep: async () => {},
|
|
fetchImpl: async () => {
|
|
calls++;
|
|
return calls === 1
|
|
? new Response("boom", { status: 500 })
|
|
: new Response("bytes", { status: 200, headers: { "content-type": "image/png" } });
|
|
}
|
|
}
|
|
);
|
|
|
|
assert.equal(outcome.state, "downloaded");
|
|
assert.equal(calls, 2);
|
|
});
|
|
|
|
await test("engine downloads attachments in download mode", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("500", {
|
|
attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const fetched: string[] = [];
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async (input: any) => {
|
|
fetched.push(String(input));
|
|
return new Response("png-bytes", {
|
|
status: 200,
|
|
headers: { "content-type": "image/png" }
|
|
});
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "127", guildId: null, mode: "download", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.downloaded, 1);
|
|
assert.equal(meta?.previewed, 2);
|
|
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
|
assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.downloaded, 1);
|
|
assert.ok(row.file_path);
|
|
assert.ok((row.file_bytes ?? 0) > 0);
|
|
const embed = store.queryMedia({ kind: "video" })[0];
|
|
assert.equal(embed.downloaded, 0);
|
|
assert.equal(embed.file_path, null);
|
|
store.close();
|
|
assert.equal(existsSync(join(root, started.id, row.file_path)), true);
|
|
});
|
|
|
|
await test("aspect and resolution presets have fixed buckets", () => {
|
|
assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
|
|
assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
|
|
assert.equal(aspectCondition("nope"), null);
|
|
assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
|
|
assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
|
|
assert.equal(resolutionCondition("nope"), null);
|
|
});
|
|
|
|
await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
|
|
const dir = join(root, "store-source");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("700", {
|
|
attachments: [
|
|
attachment({
|
|
filename: "pic.png",
|
|
contentType: "image/png",
|
|
width: 640,
|
|
height: 360
|
|
})
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
})
|
|
]);
|
|
|
|
assert.equal(store.queryMedia({ source: "attachment" }).length, 1);
|
|
assert.equal(store.queryMedia({ source: "embed" }).length, 1);
|
|
assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
|
|
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
|
|
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
|
|
|
|
assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
|
|
assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
|
|
assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
|
|
assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
|
|
assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
|
|
assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
|
|
assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
|
|
assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
|
|
|
|
const [wide] = store.queryMedia({ kind: "image" });
|
|
assert.equal(wide.aspect_ratio, 1.778);
|
|
store.close();
|
|
});
|
|
|
|
await test("engine probes dimensions when the metadata lacks them", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("701", {
|
|
attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
probe: async () => ({ width: 1080, height: 1920, durationSecs: 12.3 }),
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
|
|
await engine.processQueue();
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.width, 1080);
|
|
assert.equal(row.height, 1920);
|
|
assert.equal(row.aspect_ratio, 0.563);
|
|
assert.equal(row.duration_secs, 12.3);
|
|
store.close();
|
|
});
|
|
|
|
await test("preview urls resize Discord and pass embed thumbnails through", () => {
|
|
assert.equal(
|
|
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
|
|
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
|
|
);
|
|
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
|
|
assert.equal(
|
|
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4?ex=abc", kind: "video" }),
|
|
"https://media.discordapp.net/attachments/1/2/clip.mp4?ex=abc&width=320&height=320&format=webp"
|
|
);
|
|
assert.equal(
|
|
previewUrlFor({
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
kind: "video",
|
|
previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}),
|
|
"https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
);
|
|
});
|
|
|
|
await test("preview downloads are written by hash", async () => {
|
|
const dir = join(root, "preview-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const outcome = await downloadPreview(
|
|
{
|
|
id: 9,
|
|
url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
|
|
kind: "image",
|
|
source: "attachment",
|
|
preview_url: null
|
|
},
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } })
|
|
}
|
|
);
|
|
|
|
assert.equal(outcome.state, "downloaded");
|
|
assert.equal(outcome.path, `${outcome.hash}.webp`);
|
|
assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
|
|
|
|
const jpeg = await downloadPreview(
|
|
{ id: 10, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image", source: "attachment", preview_url: null },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("jpeg-bytes", { status: 200, headers: { "content-type": "image/jpeg" } })
|
|
}
|
|
);
|
|
assert.equal(jpeg.path, `${jpeg.hash}.jpg`);
|
|
assert.equal(existsSync(join(dir, `${jpeg.hash}.jpg`)), true);
|
|
});
|
|
|
|
await test("embed thumbnails download only when enabled", async () => {
|
|
const dir = join(root, "preview-embed");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const media = {
|
|
id: 11,
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
kind: "video" as const,
|
|
source: "embed",
|
|
preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
};
|
|
const calls: string[] = [];
|
|
const fetchImpl = async (input: any) => {
|
|
calls.push(String(input));
|
|
return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
|
|
};
|
|
|
|
const off = await downloadPreview(media, dir, { fetchImpl });
|
|
assert.equal(off.state, "skipped");
|
|
assert.equal(calls.length, 0);
|
|
|
|
const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
|
|
assert.equal(on.state, "downloaded");
|
|
assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
|
|
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
|
});
|
|
|
|
await test("engine caches previews in links mode, including embed thumbnails", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("600", {
|
|
attachments: [
|
|
attachment({ filename: "prev.png", contentType: "image/png" }),
|
|
attachment({ filename: "clip.mp4", contentType: "video/mp4" })
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const fetched: string[] = [];
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async (input: any) => {
|
|
fetched.push(String(input));
|
|
return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.previewed, 3);
|
|
assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
|
|
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
|
assert.equal(fetched.some(url => url.includes("clip.mp4") && url.includes("format=webp")), true);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.ok(row.preview_hash);
|
|
assert.ok((row.preview_bytes ?? 0) > 0);
|
|
assert.equal(row.preview_ext, "webp");
|
|
store.close();
|
|
assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true);
|
|
});
|
|
|
|
await test("custom download destination keeps files off-archive and copies embed previews", async () => {
|
|
const dest = join(root, "custom-dest");
|
|
mkdirSync(dest, { recursive: true });
|
|
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("702", {
|
|
attachments: [attachment({ filename: "offsite.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({
|
|
channelId: "130",
|
|
guildId: null,
|
|
mode: "download",
|
|
backend,
|
|
downloadDir: dest
|
|
});
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.downloadDir, dest);
|
|
assert.equal(meta?.downloaded, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const file = store.queryMedia({ kind: "image" })[0];
|
|
const filePath = file.file_path;
|
|
assert.ok(filePath && filePath.startsWith(dest));
|
|
assert.equal(existsSync(filePath), true);
|
|
assert.equal(existsSync(join(root, started.id, "media", filePath.split("/").pop() ?? "")), false);
|
|
|
|
const embed = store.queryMedia({ kind: "video" })[0];
|
|
assert.ok(embed.preview_hash);
|
|
const previewFile = `${embed.preview_hash}.${embed.preview_ext ?? "webp"}`;
|
|
assert.equal(existsSync(join(dest, started.id, "thumbnails", previewFile)), true);
|
|
assert.equal(existsSync(join(root, started.id, "thumbs", previewFile)), true);
|
|
store.close();
|
|
});
|
|
|
|
await test("embed previews resolve from providers and og:image", async () => {
|
|
const fetchImpl = async (input: any) => {
|
|
const url = String(input);
|
|
if (url.includes("youtube.com/oembed")) {
|
|
return new Response(JSON.stringify({ thumbnail_url: "https://i.ytimg.com/ok.jpg" }), {
|
|
status: 200
|
|
});
|
|
}
|
|
if (url.includes("tenor.com/oembed")) {
|
|
return new Response(JSON.stringify({ thumbnail_url: "https://media.tenor.com/ok.png" }), {
|
|
status: 200
|
|
});
|
|
}
|
|
if (url.includes("api.fxtwitter.com")) {
|
|
return new Response(
|
|
JSON.stringify({ tweet: { media: { videos: [{ thumbnail_url: "https://pbs.twimg.com/ok.jpg" }] } } }),
|
|
{ status: 200 }
|
|
);
|
|
}
|
|
if (url.includes("vxreddit.com")) {
|
|
return new Response(
|
|
'<html><head><meta property="og:image" content="https://i.redd.it/ok.jpg"></head></html>',
|
|
{ status: 200 }
|
|
);
|
|
}
|
|
return new Response("no", { status: 404 });
|
|
};
|
|
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://youtu.be/abc", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://i.ytimg.com/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://fxtwitter.com/u/status/12345", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://pbs.twimg.com/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://tenor.com/view/abc-123", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://media.tenor.com/ok.png"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://vxreddit.com/r/x/comments/1", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://i.redd.it/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://x.com/u/status/1", source: "embed", preview_url: "https://stored/ok.jpg" },
|
|
{ fetchImpl }
|
|
),
|
|
"https://stored/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://cdn.discordapp.com/x.png", source: "attachment", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
null
|
|
);
|
|
|
|
assert.equal(
|
|
ogImageFromHtml('<meta content="https://a/1.jpg" property="og:image">'),
|
|
"https://a/1.jpg"
|
|
);
|
|
assert.equal(
|
|
ogImageFromHtml('<meta name="twitter:image" content="https://a/2.jpg">'),
|
|
"https://a/2.jpg"
|
|
);
|
|
assert.equal(ogImageFromHtml("<html></html>"), null);
|
|
});
|
|
|
|
await test("videoPoster extracts a cached frame", async () => {
|
|
const dir = join(root, "poster-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const videoPath = join(dir, "clip.mp4");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc=duration=1:size=64x64:rate=5",
|
|
videoPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - videoPoster skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const poster = await videoPoster(videoPath, dir);
|
|
assert.ok(poster?.hash);
|
|
assert.ok((poster?.bytes ?? 0) > 0);
|
|
assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("gifPoster freezes an animated gif", async () => {
|
|
const dir = join(root, "gif-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const gifPath = join(dir, "anim.gif");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc=duration=1:size=64x64:rate=2",
|
|
gifPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - gifPoster skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const poster = await gifPoster(gifPath, dir);
|
|
assert.ok(poster?.hash);
|
|
assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("engine resumes a paused scrape from its cursor", async () => {
|
|
const first = new FakeBackend([
|
|
{ messages: [message("500"), message("499")] },
|
|
{ messages: [message("498")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, first);
|
|
const started = await engine.start({ channelId: "140", guildId: null, mode: "links", backend: first });
|
|
engine.stop(started.id);
|
|
await engine.processQueue();
|
|
|
|
let meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.cursor, "499");
|
|
assert.equal(meta?.messages, 2);
|
|
|
|
const second = new FakeBackend([{ messages: [message("498")] }]);
|
|
await engine.resume(started.id, second);
|
|
await engine.processQueue();
|
|
|
|
meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.messages, 3);
|
|
assert.equal(meta?.cursor, "498");
|
|
assert.equal(second.calls[0].before, "499");
|
|
});
|
|
|
|
await test("backfillPreviews fills rows left by a preview-less walk", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("800", {
|
|
attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
assert.equal(engine.get(started.id)?.previewed, undefined);
|
|
|
|
const added = await engine.backfillPreviews(started.id);
|
|
assert.equal(added, 2);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
|
|
assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
|
|
store.close();
|
|
});
|
|
|
|
await test("dedupe prefers the richer row for a URL", () => {
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: false },
|
|
{ downloaded: false, hasPreview: true }
|
|
),
|
|
true
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: true },
|
|
{ downloaded: false, hasPreview: false }
|
|
),
|
|
false
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: true, hasPreview: false },
|
|
{ downloaded: false, hasPreview: true }
|
|
),
|
|
false
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: true },
|
|
{ downloaded: true, hasPreview: false }
|
|
),
|
|
true
|
|
);
|
|
});
|
|
|
|
await test("previews pause between batches", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("900", {
|
|
attachments: [
|
|
attachment({ filename: "a.png", contentType: "image/png" }),
|
|
attachment({ filename: "b.png", contentType: "image/png" }),
|
|
attachment({ filename: "c.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const sleeps: number[] = [];
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 10,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
previewMinGapMs: 0,
|
|
previewMaxGapMs: 0,
|
|
previewBatchSize: 2,
|
|
previewBatchPauseMs: 5000,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
assert.equal(engine.get(started.id)?.previewed, 3);
|
|
assert.equal(sleeps.filter(ms => ms === 5000).length, 1);
|
|
});
|
|
|
|
await test("remoteFrame renders an audio waveform", async () => {
|
|
const dir = join(root, "waveform-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const wavPath = join(dir, "tone.wav");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"sine=frequency=440:duration=1",
|
|
wavPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - remoteFrame skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const frame = await remoteFrame(wavPath, dir, { waveform: true });
|
|
assert.ok(frame?.hash);
|
|
assert.ok((frame?.bytes ?? 0) > 0);
|
|
assert.equal(existsSync(join(dir, `${frame?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("snowflake helpers round-trip dates", () => {
|
|
const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
|
|
assert.ok(id);
|
|
assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z");
|
|
assert.equal(snowflakeFromIso("not a date"), null);
|
|
assert.equal(isoFromSnowflake("nope"), null);
|
|
});
|
|
|
|
await test("engine honors a from-date boundary", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("1000"), message("900"), message("800")] }
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 10,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false
|
|
});
|
|
|
|
const started = await engine.start({
|
|
channelId: "131",
|
|
guildId: null,
|
|
mode: "links",
|
|
backend,
|
|
afterId: "900"
|
|
});
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.messages, 1);
|
|
assert.equal(meta?.afterId, "900");
|
|
assert.equal(backend.calls.length, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.equal(store.queryMedia({}).length, 0);
|
|
store.close();
|
|
});
|
|
|
|
await test("retention marks the oldest completed scrapes up to the limit", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 100);
|
|
assert.deepEqual(plan.mark, ["old", "new"]);
|
|
assert.deepEqual(plan.delete, []);
|
|
});
|
|
|
|
await test("retention deletes marked scrapes past the overrun allowance", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 5);
|
|
assert.deepEqual(plan.delete, ["old", "new"]);
|
|
assert.deepEqual(plan.mark, []);
|
|
});
|
|
|
|
await test("retention leaves usage under the limit alone", () => {
|
|
const plan = planRetention(
|
|
[{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }],
|
|
1000,
|
|
5
|
|
);
|
|
|
|
assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 });
|
|
assert.equal(isNearLimit(900, 1000, 90), true);
|
|
assert.equal(isNearLimit(899, 1000, 90), false);
|
|
});
|
|
|
|
rmSync(root, { recursive: true, force: true });
|
|
|
|
if (failures > 0) {
|
|
console.error(`${failures} test(s) failed`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("all media tests passed");
|
|
})();
|