2428 lines
91 KiB
TypeScript
2428 lines
91 KiB
TypeScript
/*
|
|
* Media archive unit tests. No Discord, no network: a fake backend and a temp DB root.
|
|
* Run with: pnpm test:media (from poc/).
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import {
|
|
downloadOne,
|
|
downloadPreview,
|
|
gifPoster,
|
|
previewUrlFor,
|
|
remoteFrame,
|
|
videoPoster
|
|
} from "../src/download.js";
|
|
import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
|
|
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
|
import { BackfillQueue } from "../src/backfill.js";
|
|
import { TitleLane } from "../src/titles.js";
|
|
import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
|
|
import { PuppetBackend } from "../src/puppet-backend.js";
|
|
import { messageUrlFor } from "../src/discord-links.js";
|
|
import { betterCandidate, sortCandidates } from "../src/dedupe.js";
|
|
import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
|
|
import { isNearLimit, planRetention } from "../src/retention.js";
|
|
import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
|
|
import {
|
|
dirSizeBytes,
|
|
listScrapes,
|
|
normalizeTitleKey,
|
|
normalizeTitleQuery,
|
|
readScrapeMeta,
|
|
ScrapeStore,
|
|
writeScrapeMeta
|
|
} from "../src/store.js";
|
|
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
|
|
|
let failures = 0;
|
|
|
|
async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
|
|
try {
|
|
await fn();
|
|
console.log(`ok - ${name}`);
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(`not ok - ${name}`);
|
|
console.error(error);
|
|
}
|
|
}
|
|
|
|
function message(id: string, extra: Partial<CoreMessage> = {}): CoreMessage {
|
|
return {
|
|
id,
|
|
author: { id: "1", username: "tester", avatar: null },
|
|
content: "",
|
|
timestamp: "2026-09-16T00:00:00.000Z",
|
|
editedAt: null,
|
|
replyTo: null,
|
|
attachments: [],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
forwarded: [],
|
|
voiceMessage: false,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
function attachment(extra: Partial<CoreAttachment> = {}): CoreAttachment {
|
|
const filename = extra.filename ?? "file.bin";
|
|
return {
|
|
id: null,
|
|
filename,
|
|
url: `https://cdn.discordapp.com/attachments/1/2/${filename}`,
|
|
contentType: null,
|
|
size: null,
|
|
width: null,
|
|
height: null,
|
|
durationSecs: null,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
class FakeBackend implements ScrapeBackend {
|
|
readonly name = "puppet" as const;
|
|
readonly calls: Array<{ before: string | null; limit: number }> = [];
|
|
|
|
constructor(private readonly pages: Array<FetchPage | Error>) {}
|
|
|
|
async fetchPage(_channelId: string, before: string | null, limit: number): Promise<FetchPage> {
|
|
this.calls.push({ before, limit });
|
|
const page = this.pages.shift();
|
|
if (!page) return { messages: [] };
|
|
if (page instanceof Error) throw page;
|
|
return page;
|
|
}
|
|
}
|
|
|
|
/** A backend whose first page waits for `release()`, so a test can stop the walk mid-page. */
|
|
function gatedBackend(page: FetchPage): { backend: ScrapeBackend; release: () => void } {
|
|
let release!: () => void;
|
|
const gate = new Promise<void>(resolve => (release = resolve));
|
|
let first = true;
|
|
|
|
const backend: ScrapeBackend = {
|
|
name: "puppet",
|
|
async fetchPage() {
|
|
if (first) {
|
|
first = false;
|
|
await gate;
|
|
}
|
|
return page;
|
|
}
|
|
};
|
|
|
|
return { backend, release };
|
|
}
|
|
|
|
function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, unknown> = {}) {
|
|
return new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
...extra
|
|
});
|
|
}
|
|
|
|
void (async () => {
|
|
const root = mkdtempSync(join(tmpdir(), "veskslop-media-"));
|
|
|
|
await test("extract classifies attachments, embeds, and stickers", () => {
|
|
const items = extractMedia(
|
|
message("100", {
|
|
attachments: [
|
|
attachment({ filename: "cat.png", contentType: "image/png" }),
|
|
attachment({ filename: "loop.gif" }),
|
|
attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234, durationSecs: 0 }),
|
|
attachment({ filename: "note.ogg", durationSecs: 3.2 })
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: "x",
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
},
|
|
{
|
|
type: "gifv",
|
|
url: "https://tenor.com/view/x",
|
|
title: null,
|
|
provider: "Tenor",
|
|
videoUrl: null,
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://media.tenor.com/x.gif"
|
|
}
|
|
],
|
|
stickerItems: [
|
|
{ id: "55", name: "wave", formatType: 4 },
|
|
{ id: "56", name: "lottie", formatType: 3 }
|
|
]
|
|
})
|
|
);
|
|
|
|
const kinds = items.map(item => item.kind).sort();
|
|
assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]);
|
|
const youtube = items.find(item => item.url.includes("youtube.com/watch"));
|
|
assert.equal(youtube?.kind, "video");
|
|
assert.equal(youtube?.provider, "YouTube");
|
|
assert.equal(youtube?.downloaded, false);
|
|
assert.equal(youtube?.playUrl, "https://www.youtube.com/embed/x");
|
|
assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif");
|
|
assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
|
|
assert.equal(items.some(item => item.url.includes("lottie")), false);
|
|
assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2);
|
|
assert.equal(items.find(item => item.url.endsWith("clip.mp4"))?.durationSecs, null);
|
|
});
|
|
|
|
await test("extract finds media in forwarded snapshots", () => {
|
|
const items = extractMedia(
|
|
message("101", {
|
|
forwarded: [
|
|
{
|
|
content: "",
|
|
timestamp: null,
|
|
attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
voiceMessage: false
|
|
}
|
|
]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "video");
|
|
});
|
|
|
|
await test("extract treats a voice message as audio", () => {
|
|
const items = extractMedia(
|
|
message("102", {
|
|
voiceMessage: true,
|
|
attachments: [attachment({ filename: "voice-message.ogg" })]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "audio");
|
|
});
|
|
|
|
await test("engine walks pages, persists meta, and stores media", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("100", {
|
|
attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })]
|
|
}),
|
|
message("101")
|
|
]
|
|
},
|
|
{ messages: [message("99")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.pages, 2);
|
|
assert.equal(meta?.messages, 3);
|
|
assert.equal(meta?.cursor, "99");
|
|
assert.equal(meta?.media.image, 1);
|
|
assert.ok(meta?.finishedAt);
|
|
assert.equal(backend.calls[0].limit, 2);
|
|
assert.equal(backend.calls[1].before, "100");
|
|
|
|
const dir = join(root, started.id);
|
|
assert.equal(existsSync(join(dir, "scrape.json")), true);
|
|
assert.equal(existsSync(join(dir, "index.db")), true);
|
|
assert.ok(dirSizeBytes(dir) > 0);
|
|
|
|
const stored = new ScrapeStore(dir);
|
|
assert.equal(stored.counts().image, 1);
|
|
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
|
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
|
assert.equal(stored.storedBytesByKind().image, 0);
|
|
const row = stored.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.author_name, "tester");
|
|
assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
|
|
stored.close();
|
|
|
|
assert.equal(listScrapes(root).length, 1);
|
|
});
|
|
|
|
await test("engine backs off and retries with retryAfterMs", async () => {
|
|
const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 });
|
|
const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]);
|
|
const sleeps: number[] = [];
|
|
|
|
const engine = engineFor(root, backend, {
|
|
maxRetries: 3,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
}
|
|
});
|
|
const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.errors, 2);
|
|
assert.deepEqual(sleeps, [250, 3000]);
|
|
});
|
|
|
|
await test("engine pauses when the page budget is reached", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("200"), message("201")] },
|
|
{ messages: [message("199"), message("198")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend, { pagesPerDay: 1 });
|
|
const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
assert.equal(backend.calls.length, 1);
|
|
});
|
|
|
|
await test("engine stop pauses after the current page", async () => {
|
|
const { backend, release } = gatedBackend({ messages: [message("300"), message("301")] });
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend });
|
|
|
|
// Stop while the first page is in flight; the page lands, then the walk pauses.
|
|
await new Promise(resolve => setImmediate(resolve));
|
|
engine.stop(started.id);
|
|
release();
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
});
|
|
|
|
await test("an immediate stop fetches nothing", async () => {
|
|
const backend = new FakeBackend([{ messages: [message("310")] }]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
|
|
engine.stop(started.id);
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 0);
|
|
assert.equal(backend.calls.length, 0);
|
|
});
|
|
|
|
await test("puppet backend passes before and limit to the core", async () => {
|
|
const calls: Array<Record<string, unknown>> = [];
|
|
const backend = new PuppetBackend(async (channelId, limit, options) => {
|
|
calls.push({ channelId, limit, ...options });
|
|
return { messages: [message("50")] };
|
|
});
|
|
|
|
const page = await backend.fetchPage("123", "60", 5);
|
|
assert.equal(page.messages.length, 1);
|
|
assert.equal(page.messages[0].id, "50");
|
|
assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
|
|
});
|
|
|
|
await test("download stores bytes by hash and skips expired URLs", async () => {
|
|
const dir = join(root, "download-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const ok = await downloadOne(
|
|
{ id: 1, url: "https://cdn.discordapp.com/attachments/1/2/note.txt", kind: "image" },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("hello", { status: 200, headers: { "content-type": "text/plain" } })
|
|
}
|
|
);
|
|
assert.equal(ok.state, "downloaded");
|
|
assert.equal(ok.hash, createHash("sha256").update("hello").digest("hex"));
|
|
assert.equal(existsSync(join(dir, `${ok.hash}.txt`)), true);
|
|
|
|
const expired = await downloadOne(
|
|
{ id: 2, url: "https://cdn.discordapp.com/attachments/1/2/gone.png", kind: "image" },
|
|
dir,
|
|
{ fetchImpl: async () => new Response("no", { status: 404 }) }
|
|
);
|
|
assert.equal(expired.state, "skipped");
|
|
|
|
const large = await downloadOne(
|
|
{ id: 3, url: "https://cdn.discordapp.com/attachments/1/2/big.mp4", kind: "video" },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("x", { status: 200, headers: { "content-length": "999999999" } })
|
|
}
|
|
);
|
|
assert.equal(large.state, "skipped");
|
|
});
|
|
|
|
await test("download retries a 5xx and then succeeds", async () => {
|
|
const dir = join(root, "download-retry");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
let calls = 0;
|
|
const outcome = await downloadOne(
|
|
{ id: 4, url: "https://cdn.discordapp.com/attachments/1/2/flaky.png", kind: "image" },
|
|
dir,
|
|
{
|
|
maxRetries: 2,
|
|
sleep: async () => {},
|
|
fetchImpl: async () => {
|
|
calls++;
|
|
return calls === 1
|
|
? new Response("boom", { status: 500 })
|
|
: new Response("bytes", { status: 200, headers: { "content-type": "image/png" } });
|
|
}
|
|
}
|
|
);
|
|
|
|
assert.equal(outcome.state, "downloaded");
|
|
assert.equal(calls, 2);
|
|
});
|
|
|
|
await test("engine downloads attachments in download mode", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("500", {
|
|
attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const fetched: string[] = [];
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async (input: any) => {
|
|
fetched.push(String(input));
|
|
return new Response("png-bytes", {
|
|
status: 200,
|
|
headers: { "content-type": "image/png" }
|
|
});
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "127", guildId: null, mode: "download", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.downloaded, 1);
|
|
assert.equal(meta?.previewed, 2);
|
|
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
|
assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.downloaded, 1);
|
|
assert.ok(row.file_path);
|
|
assert.ok((row.file_bytes ?? 0) > 0);
|
|
const embed = store.queryMedia({ kind: "video" })[0];
|
|
assert.equal(embed.downloaded, 0);
|
|
assert.equal(embed.file_path, null);
|
|
store.close();
|
|
assert.equal(existsSync(join(root, started.id, row.file_path)), true);
|
|
});
|
|
|
|
await test("aspect and resolution presets have fixed buckets", () => {
|
|
assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
|
|
assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
|
|
assert.equal(aspectCondition("nope"), null);
|
|
assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
|
|
assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
|
|
assert.equal(resolutionCondition("nope"), null);
|
|
});
|
|
|
|
await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
|
|
const dir = join(root, "store-source");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("700", {
|
|
attachments: [
|
|
attachment({
|
|
filename: "pic.png",
|
|
contentType: "image/png",
|
|
width: 640,
|
|
height: 360
|
|
})
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
})
|
|
]);
|
|
|
|
assert.equal(store.queryMedia({ source: "attachment" }).length, 1);
|
|
assert.equal(store.queryMedia({ source: "embed" }).length, 1);
|
|
assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
|
|
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
|
|
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
|
|
|
|
assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
|
|
assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
|
|
assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
|
|
assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
|
|
assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
|
|
assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
|
|
assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
|
|
assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
|
|
|
|
const [wide] = store.queryMedia({ kind: "image" });
|
|
assert.equal(wide.aspect_ratio, 1.778);
|
|
store.close();
|
|
});
|
|
|
|
await test("engine probes dimensions when the metadata lacks them", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("701", {
|
|
attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
probe: async () => ({ width: 1080, height: 1920, durationSecs: 12.3 }),
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
|
|
await engine.processQueue();
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.equal(row.width, 1080);
|
|
assert.equal(row.height, 1920);
|
|
assert.equal(row.aspect_ratio, 0.563);
|
|
assert.equal(row.duration_secs, 12.3);
|
|
store.close();
|
|
});
|
|
|
|
await test("preview urls resize Discord and pass embed thumbnails through", () => {
|
|
assert.equal(
|
|
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
|
|
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
|
|
);
|
|
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
|
|
assert.equal(
|
|
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4?ex=abc", kind: "video" }),
|
|
"https://media.discordapp.net/attachments/1/2/clip.mp4?ex=abc&width=320&height=320&format=webp"
|
|
);
|
|
assert.equal(
|
|
previewUrlFor({
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
kind: "video",
|
|
previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}),
|
|
"https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
);
|
|
});
|
|
|
|
await test("preview downloads are written by hash", async () => {
|
|
const dir = join(root, "preview-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const outcome = await downloadPreview(
|
|
{
|
|
id: 9,
|
|
url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
|
|
kind: "image",
|
|
source: "attachment",
|
|
preview_url: null
|
|
},
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } })
|
|
}
|
|
);
|
|
|
|
assert.equal(outcome.state, "downloaded");
|
|
assert.equal(outcome.path, `${outcome.hash}.webp`);
|
|
assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
|
|
|
|
const jpeg = await downloadPreview(
|
|
{ id: 10, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image", source: "attachment", preview_url: null },
|
|
dir,
|
|
{
|
|
fetchImpl: async () =>
|
|
new Response("jpeg-bytes", { status: 200, headers: { "content-type": "image/jpeg" } })
|
|
}
|
|
);
|
|
assert.equal(jpeg.path, `${jpeg.hash}.jpg`);
|
|
assert.equal(existsSync(join(dir, `${jpeg.hash}.jpg`)), true);
|
|
});
|
|
|
|
await test("embed thumbnails download only when enabled", async () => {
|
|
const dir = join(root, "preview-embed");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const media = {
|
|
id: 11,
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
kind: "video" as const,
|
|
source: "embed",
|
|
preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
};
|
|
const calls: string[] = [];
|
|
const fetchImpl = async (input: any) => {
|
|
calls.push(String(input));
|
|
return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
|
|
};
|
|
|
|
const off = await downloadPreview(media, dir, { fetchImpl });
|
|
assert.equal(off.state, "skipped");
|
|
assert.equal(calls.length, 0);
|
|
|
|
const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
|
|
assert.equal(on.state, "downloaded");
|
|
assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
|
|
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
|
});
|
|
|
|
await test("pendingPreviews includes embeds without a stored thumbnail", () => {
|
|
const dir = join(root, "pending-embeds");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("670", {
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
|
|
title: null,
|
|
provider: "cdn.discordapp.com",
|
|
videoUrl: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
})
|
|
]);
|
|
|
|
assert.equal(store.pendingPreviews(10, false).length, 0);
|
|
assert.equal(store.pendingPreviews(10, true).length, 1);
|
|
store.close();
|
|
});
|
|
|
|
await test("CDN attachment embeds resolve through the Discord proxy", async () => {
|
|
const resolved = await resolveEmbedPreview({
|
|
url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
|
|
source: "embed",
|
|
preview_url: null
|
|
});
|
|
|
|
assert.ok(resolved);
|
|
const parsed = new URL(String(resolved));
|
|
assert.equal(parsed.hostname, "media.discordapp.net");
|
|
assert.equal(parsed.searchParams.get("width"), "320");
|
|
assert.equal(parsed.searchParams.get("format"), "webp");
|
|
});
|
|
|
|
await test("the og:image fallback never reads media bytes", async () => {
|
|
const seen: string[] = [];
|
|
const resolved = await resolveEmbedPreview(
|
|
{
|
|
url: "https://example.com/clip.mp4",
|
|
source: "embed",
|
|
preview_url: null
|
|
},
|
|
{
|
|
fetchImpl: async (input: any) => {
|
|
seen.push(String(input));
|
|
return new Response("binary", {
|
|
status: 200,
|
|
headers: { "content-type": "video/mp4" }
|
|
});
|
|
}
|
|
}
|
|
);
|
|
|
|
assert.equal(resolved, null);
|
|
assert.equal(seen.length, 1);
|
|
});
|
|
|
|
await test("engine caches previews in links mode, including embed thumbnails", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("600", {
|
|
attachments: [
|
|
attachment({ filename: "prev.png", contentType: "image/png" }),
|
|
attachment({ filename: "clip.mp4", contentType: "video/mp4" })
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const fetched: string[] = [];
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async (input: any) => {
|
|
fetched.push(String(input));
|
|
return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.previewed, 3);
|
|
assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
|
|
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
|
assert.equal(fetched.some(url => url.includes("clip.mp4") && url.includes("format=webp")), true);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.ok(row.preview_hash);
|
|
assert.ok((row.preview_bytes ?? 0) > 0);
|
|
assert.equal(row.preview_ext, "webp");
|
|
store.close();
|
|
assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true);
|
|
});
|
|
|
|
await test("custom download destination keeps files off-archive and copies embed previews", async () => {
|
|
const dest = join(root, "custom-dest");
|
|
mkdirSync(dest, { recursive: true });
|
|
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("702", {
|
|
attachments: [attachment({ filename: "offsite.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({
|
|
channelId: "130",
|
|
guildId: null,
|
|
mode: "download",
|
|
backend,
|
|
downloadDir: dest
|
|
});
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.downloadDir, dest);
|
|
assert.equal(meta?.downloaded, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const file = store.queryMedia({ kind: "image" })[0];
|
|
const filePath = file.file_path;
|
|
assert.ok(filePath && filePath.startsWith(dest));
|
|
assert.equal(existsSync(filePath), true);
|
|
assert.equal(existsSync(join(root, started.id, "media", filePath.split("/").pop() ?? "")), false);
|
|
|
|
const embed = store.queryMedia({ kind: "video" })[0];
|
|
assert.ok(embed.preview_hash);
|
|
const previewFile = `${embed.preview_hash}.${embed.preview_ext ?? "webp"}`;
|
|
assert.equal(existsSync(join(dest, started.id, "thumbnails", previewFile)), true);
|
|
assert.equal(existsSync(join(root, started.id, "thumbs", previewFile)), true);
|
|
store.close();
|
|
});
|
|
|
|
await test("embed previews resolve from providers and og:image", async () => {
|
|
const fetchImpl = async (input: any) => {
|
|
const url = String(input);
|
|
if (url.includes("youtube.com/oembed")) {
|
|
return new Response(JSON.stringify({ thumbnail_url: "https://i.ytimg.com/ok.jpg" }), {
|
|
status: 200
|
|
});
|
|
}
|
|
if (url.includes("tenor.com/oembed")) {
|
|
return new Response(JSON.stringify({ thumbnail_url: "https://media.tenor.com/ok.png" }), {
|
|
status: 200
|
|
});
|
|
}
|
|
if (url.includes("api.fxtwitter.com")) {
|
|
return new Response(
|
|
JSON.stringify({ tweet: { media: { videos: [{ thumbnail_url: "https://pbs.twimg.com/ok.jpg" }] } } }),
|
|
{ status: 200 }
|
|
);
|
|
}
|
|
if (url.includes("vxreddit.com")) {
|
|
return new Response(
|
|
'<html><head><meta property="og:image" content="https://i.redd.it/ok.jpg"></head></html>',
|
|
{ status: 200 }
|
|
);
|
|
}
|
|
return new Response("no", { status: 404 });
|
|
};
|
|
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://youtu.be/abc", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://i.ytimg.com/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://fxtwitter.com/u/status/12345", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://pbs.twimg.com/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://tenor.com/view/abc-123", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://media.tenor.com/ok.png"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://vxreddit.com/r/x/comments/1", source: "embed", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
"https://i.redd.it/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://x.com/u/status/1", source: "embed", preview_url: "https://stored/ok.jpg" },
|
|
{ fetchImpl }
|
|
),
|
|
"https://stored/ok.jpg"
|
|
);
|
|
assert.equal(
|
|
await resolveEmbedPreview(
|
|
{ url: "https://cdn.discordapp.com/x.png", source: "attachment", preview_url: null },
|
|
{ fetchImpl }
|
|
),
|
|
null
|
|
);
|
|
|
|
assert.equal(
|
|
ogImageFromHtml('<meta content="https://a/1.jpg" property="og:image">'),
|
|
"https://a/1.jpg"
|
|
);
|
|
assert.equal(
|
|
ogImageFromHtml('<meta name="twitter:image" content="https://a/2.jpg">'),
|
|
"https://a/2.jpg"
|
|
);
|
|
assert.equal(ogImageFromHtml("<html></html>"), null);
|
|
});
|
|
|
|
await test("videoPoster extracts a cached frame", async () => {
|
|
const dir = join(root, "poster-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const videoPath = join(dir, "clip.mp4");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc=duration=1:size=64x64:rate=5",
|
|
videoPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - videoPoster skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const poster = await videoPoster(videoPath, dir);
|
|
assert.ok(poster?.hash);
|
|
assert.ok((poster?.bytes ?? 0) > 0);
|
|
assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("gifPoster freezes an animated gif", async () => {
|
|
const dir = join(root, "gif-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const gifPath = join(dir, "anim.gif");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"testsrc=duration=1:size=64x64:rate=2",
|
|
gifPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - gifPoster skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const poster = await gifPoster(gifPath, dir);
|
|
assert.ok(poster?.hash);
|
|
assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("stale running scrapes are paused on reconcile", async () => {
|
|
const backend = new FakeBackend([{ messages: [message("1400")] }]);
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "170", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
// Simulate a daemon that died mid-job: the meta still says running.
|
|
const dir = join(root, started.id);
|
|
const meta = readScrapeMeta(dir);
|
|
assert.ok(meta);
|
|
meta.state = "running";
|
|
writeScrapeMeta(dir, meta);
|
|
|
|
const fresh = new ScrapeEngine({ rootDir: root, previews: false });
|
|
const stale = fresh.reconcileStaleJobs();
|
|
assert.deepEqual(stale, [started.id]);
|
|
assert.equal(readScrapeMeta(dir)?.state, "paused");
|
|
});
|
|
|
|
await test("markPreview stores the animated gif reference", () => {
|
|
const dir = join(root, "anim-store");
|
|
mkdirSync(dir, { recursive: true });
|
|
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("1200", { attachments: [attachment({ filename: "loop.gif" })] })
|
|
]);
|
|
|
|
const row = store.queryMedia({ kind: "gif" })[0];
|
|
store.markPreview(row.id, { hash: "static-hash", bytes: 10, ext: "webp", animHash: "anim-hash" });
|
|
|
|
const updated = store.getMedia(row.id);
|
|
assert.equal(updated?.preview_hash, "static-hash");
|
|
assert.equal(updated?.anim_hash, "anim-hash");
|
|
store.close();
|
|
});
|
|
|
|
await test("rate limits pause media traffic", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("1300", {
|
|
attachments: [
|
|
attachment({ filename: "a.png", contentType: "image/png" }),
|
|
attachment({ filename: "b.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const sleeps: number[] = [];
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 10,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
rateLimitPauseMs: 30_000,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("slow down", { status: 429, headers: { "retry-after": "10" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
previewMinGapMs: 0,
|
|
previewMaxGapMs: 0,
|
|
previewBatchSize: 100,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "160", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
for (const row of store.queryMedia({ kind: "image" })) {
|
|
assert.equal(row.preview_hash, null);
|
|
assert.equal(row.preview_error, null, "rate limited rows stay pending");
|
|
}
|
|
store.close();
|
|
|
|
assert.ok(
|
|
sleeps.some(ms => ms > 25_000 && ms <= 30_000),
|
|
"the engine paused after the rate limit"
|
|
);
|
|
});
|
|
|
|
await test("previews run in cycles during the walk", async () => {
|
|
const events: string[] = [];
|
|
const pages: Array<{ messages: CoreMessage[] }> = [
|
|
{
|
|
messages: [
|
|
message("1000", {
|
|
attachments: [attachment({ filename: "a.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
},
|
|
{
|
|
messages: [
|
|
message("999", {
|
|
attachments: [attachment({ filename: "b.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
];
|
|
|
|
const backend: ScrapeBackend = {
|
|
name: "puppet",
|
|
async fetchPage() {
|
|
events.push("walk");
|
|
return pages.shift() ?? { messages: [] };
|
|
}
|
|
};
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 1,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
walkCyclePages: 1,
|
|
download: {
|
|
fetchImpl: async () => {
|
|
events.push("preview");
|
|
return new Response("webp", { status: 200, headers: { "content-type": "image/webp" } });
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
previewMinGapMs: 0,
|
|
previewMaxGapMs: 0,
|
|
previewBatchSize: 100,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "150", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
assert.equal(engine.get(started.id)?.previewed, 2);
|
|
const firstPreview = events.indexOf("preview");
|
|
const secondWalk = events.indexOf("walk", 1);
|
|
assert.ok(firstPreview !== -1, "a preview happened during the walk");
|
|
assert.ok(secondWalk > firstPreview, "the walk continued after previews");
|
|
});
|
|
|
|
await test("engine resumes a paused scrape from its cursor", async () => {
|
|
const { backend: first, release } = gatedBackend({ messages: [message("500"), message("499")] });
|
|
|
|
const engine = engineFor(root, first);
|
|
const started = await engine.start({ channelId: "140", guildId: null, mode: "links", backend: first });
|
|
|
|
// Stop while the first page is in flight so the cursor is stored, then pauses.
|
|
await new Promise(resolve => setImmediate(resolve));
|
|
engine.stop(started.id);
|
|
release();
|
|
await engine.processQueue();
|
|
|
|
let meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.cursor, "499");
|
|
assert.equal(meta?.messages, 2);
|
|
|
|
const second = new FakeBackend([{ messages: [message("498")] }]);
|
|
await engine.resume(started.id, second);
|
|
await engine.processQueue();
|
|
|
|
meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.messages, 3);
|
|
assert.equal(meta?.cursor, "498");
|
|
assert.equal(second.calls[0].before, "499");
|
|
});
|
|
|
|
await test("transient preview failures give up after the attempt limit", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("650", {
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://gfycat.com/deadclip",
|
|
title: null,
|
|
provider: "Gfycat",
|
|
videoUrl: "https://gfycat.com/deadclip",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://thumbs.gfycat.com/dead.webp"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
previewMaxAttempts: 3,
|
|
download: {
|
|
fetchImpl: async () => {
|
|
throw new Error("fetch failed");
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const dir = join(root, started.id);
|
|
for (let pass = 0; pass < 3; pass++) await engine.backfillPreviews(started.id);
|
|
|
|
const store = new ScrapeStore(dir);
|
|
const row = store.queryMedia({ kind: "video" })[0];
|
|
assert.equal(row.preview_attempts, 3);
|
|
assert.match(String(row.preview_error), /^unreachable after 3 attempt/);
|
|
assert.equal(store.pendingPreviews(10, true).length, 0);
|
|
store.close();
|
|
});
|
|
|
|
await test("previewBatch respects the batch limit and reports the remainder", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("655", {
|
|
attachments: [
|
|
attachment({ filename: "one.png", contentType: "image/png" }),
|
|
attachment({ filename: "two.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "129", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const first = await engine.previewBatch(started.id, 1);
|
|
assert.equal(first.cached, 1);
|
|
assert.equal(first.remaining, 1);
|
|
|
|
const second = await engine.previewBatch(started.id, 5);
|
|
assert.equal(second.cached, 1);
|
|
assert.equal(second.remaining, 0);
|
|
});
|
|
|
|
await test("previewBatch registers a rate limit for the caller", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("656", {
|
|
attachments: [attachment({ filename: "slow.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
rateLimitPauseMs: 30_000,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("slow down", { status: 429, headers: { "retry-after": "5" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "130", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const result = await engine.previewBatch(started.id, 5);
|
|
assert.equal(result.ratelimited, 1);
|
|
assert.equal(result.remaining, 1);
|
|
assert.equal(engine.rateLimitStatus().active, true);
|
|
assert.ok(engine.rateLimitStatus().remainingMs > 0);
|
|
});
|
|
|
|
await test("updateOptions retunes the attempt cap live", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("657", {
|
|
attachments: [attachment({ filename: "dead.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
previewMaxAttempts: 5,
|
|
download: {
|
|
fetchImpl: async () => {
|
|
throw new Error("fetch failed");
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "131", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
engine.updateOptions({ previewMaxAttempts: 1 });
|
|
|
|
const result = await engine.previewBatch(started.id, 5);
|
|
assert.equal(result.skipped, 1);
|
|
assert.equal(result.remaining, 0);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "image" })[0];
|
|
assert.match(String(row.preview_error), /^unreachable after 1 attempt/);
|
|
store.close();
|
|
});
|
|
|
|
await test("youtube shorts links normalize to watch URLs", () => {
|
|
assert.equal(
|
|
normalizeYouTubeUrl("https://www.youtube.com/shorts/uAf18d7mrnU"),
|
|
"https://www.youtube.com/watch?v=uAf18d7mrnU"
|
|
);
|
|
assert.equal(
|
|
normalizeYouTubeUrl("https://m.youtube.com/shorts/abc123XYZ?feature=share"),
|
|
"https://www.youtube.com/watch?v=abc123XYZ"
|
|
);
|
|
assert.equal(
|
|
normalizeYouTubeUrl("https://www.youtube.com/watch?v=keep"),
|
|
"https://www.youtube.com/watch?v=keep"
|
|
);
|
|
assert.equal(normalizeYouTubeUrl("https://youtu.be/short"), "https://youtu.be/short");
|
|
});
|
|
|
|
await test("shorts store normalized and old rows migrate", () => {
|
|
const dir = join(root, "shorts");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
|
|
store.insertPage([
|
|
message("1200", {
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/shorts/uAf18d7mrnU",
|
|
title: "A short",
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/shorts/uAf18d7mrnU",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
})
|
|
]);
|
|
|
|
const row = store.queryMedia({ kind: "video" })[0];
|
|
assert.equal(row.url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
|
|
assert.equal(row.play_url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
|
|
store.close();
|
|
|
|
// An old database (user_version 0) gets rewritten on open.
|
|
const legacyDir = join(root, "shorts-legacy");
|
|
mkdirSync(legacyDir, { recursive: true });
|
|
new ScrapeStore(legacyDir).close();
|
|
|
|
const raw = new DatabaseSync(join(legacyDir, "index.db"));
|
|
const insert = raw.prepare(
|
|
"INSERT INTO media (kind, source, url, play_url, provider, downloaded, preview_hash, created_at) VALUES ('video', 'embed', ?, ?, 'YouTube', 0, ?, ?)"
|
|
);
|
|
insert.run(
|
|
"https://www.youtube.com/shorts/legacyId123",
|
|
"https://www.youtube.com/shorts/legacyId123",
|
|
null,
|
|
"2026-09-17T00:00:00.000Z"
|
|
);
|
|
// The same video already indexed under its watch URL: the shorts row folds in.
|
|
insert.run(
|
|
"https://www.youtube.com/watch?v=legacyId456",
|
|
null,
|
|
null,
|
|
"2026-09-17T00:00:00.000Z"
|
|
);
|
|
insert.run(
|
|
"https://www.youtube.com/shorts/legacyId456",
|
|
"https://www.youtube.com/shorts/legacyId456",
|
|
"previewhash",
|
|
"2026-09-17T00:00:00.000Z"
|
|
);
|
|
raw.exec("PRAGMA user_version = 0");
|
|
raw.close();
|
|
|
|
const reopened = new ScrapeStore(legacyDir);
|
|
const rows = reopened.queryMedia({ kind: "video", limit: 50 });
|
|
assert.equal(rows.length, 2);
|
|
const legacyRow = rows.find(row => row.url.includes("legacyId123"));
|
|
assert.ok(legacyRow);
|
|
assert.equal(legacyRow.play_url, "https://www.youtube.com/watch?v=legacyId123");
|
|
const merged = rows.find(row => row.url.includes("legacyId456"));
|
|
assert.ok(merged);
|
|
assert.equal(merged.preview_hash, "previewhash");
|
|
reopened.close();
|
|
});
|
|
|
|
await test("dbSummary reports the message span and newest id", () => {
|
|
const dir = join(root, "summary");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("1800", { timestamp: "2026-09-16T00:00:00.000Z" }),
|
|
message("1801", { timestamp: "2026-09-17T00:00:00.000Z" })
|
|
]);
|
|
|
|
const summary = store.dbSummary();
|
|
assert.equal(summary.messages, 2);
|
|
assert.equal(summary.oldest, "2026-09-16T00:00:00.000Z");
|
|
assert.equal(summary.newest, "2026-09-17T00:00:00.000Z");
|
|
assert.equal(summary.newestId, "1801");
|
|
store.close();
|
|
});
|
|
|
|
await test("message links build for guilds and DMs", () => {
|
|
assert.equal(messageUrlFor("123", "456", "789"), "https://discord.com/channels/123/456/789");
|
|
assert.equal(messageUrlFor(null, "456", "789"), "https://discord.com/channels/@me/456/789");
|
|
assert.equal(messageUrlFor("123", "456", null), null);
|
|
});
|
|
|
|
await test("suspect rows park, due, and confirm", () => {
|
|
const dir = join(root, "suspects");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
store.insertPage([
|
|
message("1700", {
|
|
attachments: [attachment({ filename: "gone.mp4", contentType: "video/mp4" })]
|
|
})
|
|
]);
|
|
|
|
const row = store.queryMedia({ kind: "video" })[0];
|
|
const now = Date.now();
|
|
store.markSuspect(row.id, "http 404", now + 10_000);
|
|
|
|
assert.equal(store.previewCounts().suspect, 1);
|
|
assert.equal(store.previewCounts().unverified, 0);
|
|
assert.equal(store.pendingPreviews(10, true).length, 0);
|
|
assert.equal(store.dueSuspects(10, now).length, 0);
|
|
assert.equal(store.dueSuspects(10, now + 20_000).length, 1);
|
|
|
|
store.confirmSuspect(row.id, "still http 404 after retry");
|
|
assert.equal(store.previewCounts().suspect, 0);
|
|
assert.equal(store.previewCounts().unverified, 1);
|
|
assert.equal(store.dueSuspects(10, now + 20_000).length, 0);
|
|
|
|
store.markSuspect(row.id, "http 503", now);
|
|
assert.equal(store.previewCounts().suspect, 1);
|
|
assert.equal(store.previewCounts().unverified, 0);
|
|
assert.equal(store.dueSuspects(10, now).length, 1);
|
|
store.close();
|
|
});
|
|
|
|
await test("discord 404s become suspects and the delayed retry resolves them", async () => {
|
|
let failing = true;
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("1701", {
|
|
attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
const engine = engineFor(root, backend, {
|
|
suspectRetryMs: 0,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
failing
|
|
? new Response("gone", { status: 404 })
|
|
: new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "138", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const first = await engine.previewBatch(started.id, 5);
|
|
assert.equal(first.suspect, 1);
|
|
assert.equal(first.skipped, 0);
|
|
|
|
// Normal passes leave suspects alone.
|
|
assert.equal((await engine.previewBatch(started.id, 5)).suspect, 0);
|
|
|
|
failing = false;
|
|
const retry = await engine.previewSuspects(started.id, 5);
|
|
assert.equal(retry.cached, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.equal(store.previewCounts().cached, 1);
|
|
assert.equal(store.previewCounts().suspect, 0);
|
|
store.close();
|
|
|
|
// A retry that fails again is confirmed unverified.
|
|
const dead = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("1702", {
|
|
attachments: [attachment({ filename: "dead.mp4", contentType: "video/mp4" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
const deadEngine = engineFor(root, dead, {
|
|
suspectRetryMs: 0,
|
|
download: {
|
|
fetchImpl: async () => new Response("gone", { status: 404 }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
const deadScrape = await deadEngine.start({ channelId: "139", guildId: null, mode: "links", backend: dead });
|
|
await deadEngine.processQueue();
|
|
assert.equal((await deadEngine.previewBatch(deadScrape.id, 5)).suspect, 1);
|
|
assert.equal((await deadEngine.previewSuspects(deadScrape.id, 5)).unverified, 1);
|
|
const deadStore = new ScrapeStore(join(root, deadScrape.id));
|
|
assert.equal(deadStore.previewCounts().unverified, 1);
|
|
assert.equal(deadStore.pendingPreviews(10, true).length, 0);
|
|
deadStore.close();
|
|
});
|
|
|
|
await test("pendingTitles selects YouTube rows without a title", () => {
|
|
const dir = join(root, "pending-titles");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
|
|
store.insertPage([
|
|
message("1100", {
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=withtitle",
|
|
title: "Has a title",
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/watch?v=withtitle",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
},
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=notitle",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/watch?v=notitle",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
},
|
|
{
|
|
type: "video",
|
|
url: "https://youtu.be/short",
|
|
title: null,
|
|
provider: "youtu.be",
|
|
videoUrl: "https://youtu.be/short",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
},
|
|
{
|
|
type: "video",
|
|
url: "https://clips.twitch.tv/notyoutube",
|
|
title: null,
|
|
provider: "Twitch",
|
|
videoUrl: "https://clips.twitch.tv/notyoutube",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
})
|
|
]);
|
|
|
|
const label = (url: string) => url.split("=")[1] ?? url.split("/").pop();
|
|
// A titled row is still pending once for its channel (author) lookup.
|
|
const pending = store.pendingTitles(10).map(row => label(row.url));
|
|
assert.deepEqual(pending.sort(), ["notitle", "short", "withtitle"].sort());
|
|
|
|
const target = store.pendingTitles(10).find(row => row.url.includes("notitle"));
|
|
const dead = store.pendingTitles(10).find(row => row.url.includes("youtu.be"));
|
|
const titled = store.pendingTitles(10).find(row => row.url.includes("withtitle"));
|
|
assert.ok(target);
|
|
assert.ok(dead);
|
|
assert.ok(titled);
|
|
|
|
store.markTitle(target.id, "Resolved Title");
|
|
assert.equal(store.getMedia(target.id)?.title, "Resolved Title");
|
|
assert.equal(store.pendingTitles(10).some(row => row.id === target.id), false);
|
|
|
|
store.markTitle(titled.id, "Has a title", "Some Channel");
|
|
assert.equal(store.getMedia(titled.id)?.title_author, "Some Channel");
|
|
assert.equal(store.pendingTitles(10).some(row => row.id === titled.id), false);
|
|
|
|
store.markTitleError(dead.id, "no title after 1 attempt(s)");
|
|
assert.equal(store.pendingTitles(10).some(row => row.id === dead.id), false);
|
|
assert.equal(store.previewCounts().titlePending, 0);
|
|
store.close();
|
|
});
|
|
|
|
await test("titleBatch resolves titles and gives up after the cap", async () => {
|
|
const embed = {
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=titlepass",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/watch?v=titlepass",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
};
|
|
|
|
const backend = new FakeBackend([{ messages: [message("1101", { embeds: [embed] })] }]);
|
|
const engine = engineFor(root, backend, {
|
|
previewMaxAttempts: 3,
|
|
download: {
|
|
fetchImpl: async (input: any) => {
|
|
if (String(input).includes("oembed")) {
|
|
return new Response(JSON.stringify({ title: "Real Title" }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" }
|
|
});
|
|
}
|
|
return new Response("no", { status: 404 });
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "135", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
const batch = await engine.titleBatch(started.id, 5);
|
|
assert.equal(batch.titled, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
const row = store.queryMedia({ kind: "video" })[0];
|
|
assert.equal(row.title, "Real Title");
|
|
assert.equal(row.title_key, "real title");
|
|
assert.equal(store.previewCounts().titlePending, 0);
|
|
store.close();
|
|
|
|
// A provider that never answers is closed out after the cap.
|
|
const deadBackend = new FakeBackend([
|
|
{ messages: [message("1102", { embeds: [{ ...embed, url: "https://youtu.be/deadtitle", videoUrl: "https://youtu.be/deadtitle" }] })] }
|
|
]);
|
|
const deadEngine = engineFor(root, deadBackend, {
|
|
previewMaxAttempts: 1,
|
|
download: {
|
|
fetchImpl: async () => {
|
|
throw new Error("fetch failed");
|
|
},
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const dead = await deadEngine.start({ channelId: "136", guildId: null, mode: "links", backend: deadBackend });
|
|
await deadEngine.processQueue();
|
|
const deadBatch = await deadEngine.titleBatch(dead.id, 5);
|
|
assert.equal(deadBatch.failed, 1);
|
|
|
|
const deadStore = new ScrapeStore(join(root, dead.id));
|
|
const deadRow = deadStore.queryMedia({ kind: "video" })[0];
|
|
assert.match(String(deadRow.title_error), /^no title after 1 attempt/);
|
|
assert.equal(deadStore.previewCounts().titlePending, 0);
|
|
deadStore.close();
|
|
});
|
|
|
|
await test("title lane runs a catch-up across scrapes", async () => {
|
|
const laneRoot = join(root, "lane-root");
|
|
mkdirSync(laneRoot, { recursive: true });
|
|
|
|
const embed = (id: string) => ({
|
|
type: "video",
|
|
url: `https://www.youtube.com/watch?v=${id}`,
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: `https://www.youtube.com/watch?v=${id}`,
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
});
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("1500", { embeds: [embed("lane1"), embed("lane2")] })] }
|
|
]);
|
|
const engine = engineFor(laneRoot, backend, {
|
|
download: {
|
|
fetchImpl: async (input: any) =>
|
|
new Response(JSON.stringify({ title: `T ${String(input).length}` }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" }
|
|
}),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
await engine.start({ channelId: "137", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const lane = new TitleLane(engine, {
|
|
rootDir: laneRoot,
|
|
batchSize: 1,
|
|
batchPauseMs: 0,
|
|
sleep: async () => {},
|
|
log: () => {}
|
|
});
|
|
lane.start();
|
|
await lane.idle();
|
|
|
|
const status = lane.status();
|
|
assert.equal(status.state, "done");
|
|
assert.equal(status.pending, 0);
|
|
assert.equal(status.resolved, 2);
|
|
assert.equal(status.failed, 0);
|
|
assert.ok(status.processed >= 2);
|
|
});
|
|
|
|
await test("title search matches word prefixes and exact phrases", () => {
|
|
const dir = join(root, "title-search");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
const embed = (url: string, title: string) => ({
|
|
type: "video",
|
|
url,
|
|
title,
|
|
provider: "YouTube",
|
|
videoUrl: url,
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
});
|
|
|
|
store.insertPage([
|
|
message("1000", {
|
|
attachments: [attachment({ filename: "no-title.png", contentType: "image/png" })],
|
|
embeds: [
|
|
embed("https://www.youtube.com/watch?v=a", "Man who jumps over car"),
|
|
embed("https://www.youtube.com/watch?v=b", "Human cannonball"),
|
|
embed("https://www.youtube.com/watch?v=c", "Woman's Day: 3 Things!")
|
|
]
|
|
})
|
|
]);
|
|
|
|
const ids = (options: Record<string, unknown>) =>
|
|
store
|
|
.queryMedia({ limit: 50, ...options })
|
|
.map(row => row.url.split("v=")[1])
|
|
.sort();
|
|
|
|
// Prefix tokens in any order, but only at word starts.
|
|
assert.deepEqual(ids({ titleQuery: "Ma Ca Ju" }), ["a"]);
|
|
assert.deepEqual(ids({ titleQuery: "man" }), ["a"]);
|
|
assert.deepEqual(ids({ titleQuery: "man car" }), ["a"]);
|
|
assert.deepEqual(ids({ titleQuery: "cannon" }), ["b"]);
|
|
assert.deepEqual(ids({ titleQuery: "hum" }), ["b"]);
|
|
assert.deepEqual(ids({ titleQuery: "an" }), []);
|
|
|
|
// Exact mode is one phrase.
|
|
assert.deepEqual(ids({ titleQuery: "man who jumps", titleExact: true }), ["a"]);
|
|
assert.deepEqual(ids({ titleQuery: "man car", titleExact: true }), []);
|
|
|
|
// Punctuation folds away.
|
|
assert.deepEqual(ids({ titleQuery: "woman's day" }), ["c"]);
|
|
assert.deepEqual(ids({ titleQuery: "3 things" }), ["c"]);
|
|
|
|
store.close();
|
|
});
|
|
|
|
await test("title wildcards match anywhere and stay literal for % and _", () => {
|
|
const dir = join(root, "title-wildcards");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
const embed = (id: string, title: string) => ({
|
|
type: "video",
|
|
url: `https://www.youtube.com/watch?v=${id}`,
|
|
title,
|
|
provider: "YouTube",
|
|
videoUrl: `https://www.youtube.com/watch?v=${id}`,
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
});
|
|
|
|
store.insertPage([
|
|
message("1300", {
|
|
embeds: [
|
|
embed("w1", "Scar Tissue"),
|
|
embed("w2", "Discard the rest"),
|
|
embed("w3", "Car wash"),
|
|
embed("w4", "Cartoon physics"),
|
|
embed("w5", "Carly and the r")
|
|
]
|
|
})
|
|
]);
|
|
|
|
const ids = (options: Record<string, unknown>) =>
|
|
store
|
|
.queryMedia({ limit: 50, ...options })
|
|
.map(row => row.url.split("v=")[1])
|
|
.sort();
|
|
|
|
// Word-prefix default: car starts a word.
|
|
assert.deepEqual(ids({ titleQuery: "car" }), ["w3", "w4", "w5"]);
|
|
// Leading *: anywhere, so scar and discard join in.
|
|
assert.deepEqual(ids({ titleQuery: "*car" }), ["w1", "w2", "w3", "w4", "w5"]);
|
|
// Trailing * behaves like the prefix form.
|
|
assert.deepEqual(ids({ titleQuery: "car*" }), ["w3", "w4", "w5"]);
|
|
// In-word wildcard after a word start.
|
|
assert.deepEqual(ids({ titleQuery: "c*r" }), ["w3", "w4", "w5"]);
|
|
assert.deepEqual(ids({ titleQuery: "c*r*" }), ["w3", "w4", "w5"]);
|
|
assert.deepEqual(ids({ titleQuery: "*c*r*" }), ["w1", "w2", "w3", "w4", "w5"]);
|
|
store.close();
|
|
});
|
|
|
|
await test("normalizeTitleQuery keeps stars and drops punctuation", () => {
|
|
assert.equal(normalizeTitleQuery(" *CaR, C*r* "), "*car c*r*");
|
|
assert.equal(normalizeTitleQuery("Boat"), "boat");
|
|
});
|
|
|
|
await test("normalizeTitleKey collapses punctuation and case", () => {
|
|
assert.equal(normalizeTitleKey("Woman's Day: 3 Things!"), "woman s day 3 things");
|
|
assert.equal(normalizeTitleKey(" A---B "), "a b");
|
|
});
|
|
|
|
await test("provider filters are the curated archive list", () => {
|
|
assert.ok(PROVIDER_FILTERS.length >= 12, `provider list has ${PROVIDER_FILTERS.length} entries`);
|
|
assert.equal(new Set(PROVIDER_FILTERS).size, PROVIDER_FILTERS.length);
|
|
|
|
for (const entry of PROVIDER_FILTERS) {
|
|
const values = entry.split(",");
|
|
assert.ok(values.length >= 1 && values.every(value => value.length > 0), entry);
|
|
for (const value of values) assert.equal(value, value.toLowerCase(), entry);
|
|
}
|
|
|
|
for (const required of [
|
|
"discord",
|
|
"youtube",
|
|
"twitch",
|
|
"twitter",
|
|
"reddit",
|
|
"instagram",
|
|
"tiktok",
|
|
"streamable",
|
|
"facebook",
|
|
"gyazo",
|
|
"funnyjunk",
|
|
"internet archive",
|
|
"2ch.hk"
|
|
]) {
|
|
assert.ok((PROVIDER_FILTERS as readonly string[]).includes(required), required);
|
|
}
|
|
|
|
assert.ok((PROVIDER_FILTERS as readonly string[]).includes("giphy,tenor"), "giphy/tenor merged");
|
|
});
|
|
|
|
await test("store exclusion, size, and length filters", () => {
|
|
const dir = join(root, "exclude-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const store = new ScrapeStore(dir);
|
|
|
|
store.insertPage([
|
|
message("900", {
|
|
attachments: [
|
|
attachment({ filename: "keep.png", contentType: "image/png", size: 1_000 }),
|
|
attachment({ filename: "pixel.jpg", contentType: "image/jpeg", size: 400 } )
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=abc",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/abc",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
},
|
|
{
|
|
type: "video",
|
|
url: "https://clips.twitch.tv/xyz",
|
|
title: null,
|
|
provider: "Twitch",
|
|
videoUrl: "https://clips.twitch.tv/xyz",
|
|
imageUrl: null,
|
|
thumbnailUrl: null
|
|
}
|
|
]
|
|
}),
|
|
message("901", {
|
|
attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 9_000, durationSecs: 42 })]
|
|
})
|
|
]);
|
|
|
|
// Insert order decides the ids: 1 keep.png, 2 pixel.jpg, 3 YouTube, 4 Twitch, 5 clip.mp4.
|
|
const ids = (options: Record<string, unknown>) =>
|
|
store
|
|
.queryMedia({ limit: 50, ...options })
|
|
.map(row => row.id)
|
|
.sort((a, b) => a - b);
|
|
|
|
assert.deepEqual(ids({}), [1, 2, 3, 4, 5]);
|
|
assert.deepEqual(ids({ excludeProviders: ["youtube"] }), [1, 2, 4, 5]);
|
|
assert.deepEqual(ids({ excludeSources: ["embed"] }), [1, 2, 5]);
|
|
assert.deepEqual(ids({ excludeExts: ["png"] }), [2, 3, 4, 5]);
|
|
assert.deepEqual(ids({ sizeGreaterThan: 9000 }), []);
|
|
assert.deepEqual(ids({ sizeLessThan: 600 }), [2]);
|
|
assert.deepEqual(ids({ lengthGreaterThan: 10 }), [5]);
|
|
// Unknown lengths drop out while a length range is set.
|
|
assert.deepEqual(ids({ lengthLessThan: 10 }), []);
|
|
assert.deepEqual(ids({ includeProviders: ["youtube"] }), [3]);
|
|
assert.deepEqual(ids({ includeProviders: ["youtube", "twitch"] }), [3, 4]);
|
|
assert.deepEqual(ids({ includeSources: ["embed"] }), [3, 4]);
|
|
assert.deepEqual(ids({ includeSources: ["attachment"] }), [1, 2, 5]);
|
|
store.close();
|
|
});
|
|
|
|
await test("sortCandidates orders newest, oldest, user, size, and length", () => {
|
|
const make = (name: string | null, posted: string, size: number | null, duration: number | null, id: number) => ({
|
|
meta: {} as never,
|
|
row: {
|
|
id,
|
|
posted_at: posted,
|
|
author_name: name,
|
|
file_bytes: null,
|
|
size,
|
|
duration_secs: duration
|
|
}
|
|
});
|
|
|
|
const items = [
|
|
make("zoe", "2026-09-01T00:00:00.000Z", 100, 5, 1),
|
|
make("amy", "2026-09-03T00:00:00.000Z", 300, 1, 2),
|
|
make(null, "2026-09-02T00:00:00.000Z", null, null, 3)
|
|
];
|
|
|
|
const ids = (sort: "newest" | "oldest" | "user" | "size" | "length") =>
|
|
sortCandidates([...items], sort).map(candidate => candidate.row.id);
|
|
|
|
assert.deepEqual(ids("newest"), [2, 3, 1]);
|
|
assert.deepEqual(ids("oldest"), [1, 3, 2]);
|
|
assert.deepEqual(ids("user"), [2, 1, 3]);
|
|
assert.deepEqual(ids("size"), [2, 1, 3]);
|
|
assert.deepEqual(ids("length"), [1, 2, 3]);
|
|
});
|
|
|
|
await test("backfill queue groups scrapes per chat and estimates an ETA", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("660", {
|
|
attachments: [
|
|
attachment({ filename: "a.png", contentType: "image/png" }),
|
|
attachment({ filename: "b.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
},
|
|
{
|
|
messages: [
|
|
message("661", {
|
|
attachments: [attachment({ filename: "c.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const first = await engine.start({ channelId: "200", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
const second = await engine.start({ channelId: "200", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const queue = new BackfillQueue(engine, {
|
|
rootDir: root,
|
|
getSettings: () => ({ batchSize: 2, batchPauseMs: 0, auto: false }),
|
|
getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
|
|
sleep: async () => {}
|
|
});
|
|
|
|
const small = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("662", {
|
|
attachments: [attachment({ filename: "small.png", contentType: "image/png" })]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
await engine.start({ channelId: "203", guildId: null, mode: "links", backend: small });
|
|
await engine.processQueue();
|
|
|
|
const status = queue.list();
|
|
const job = status.jobs.find(candidate => candidate.chatId === "200");
|
|
const smallJob = status.jobs.find(candidate => candidate.chatId === "203");
|
|
assert.ok(job);
|
|
assert.ok(smallJob);
|
|
assert.equal(job.members.length, 2);
|
|
assert.equal(job.pending, 3);
|
|
assert.ok((job.etaMs ?? 0) > 0);
|
|
|
|
// Start everything: smallest first, so 203 runs and 200 waits behind it.
|
|
queue.start();
|
|
const scheduled = queue.list();
|
|
const firstJob = scheduled.jobs.find(candidate => candidate.chatId === "203");
|
|
const secondJob = scheduled.jobs.find(candidate => candidate.chatId === "200");
|
|
assert.ok(firstJob);
|
|
assert.ok(secondJob);
|
|
assert.equal(firstJob.etaStartMs, 0);
|
|
assert.ok((firstJob.etaRunMs ?? 0) > 0);
|
|
assert.equal(secondJob.etaTotalMs, (secondJob.etaStartMs ?? 0) + (secondJob.etaRunMs ?? 0));
|
|
assert.ok((secondJob.etaStartMs ?? 0) >= (firstJob.etaRunMs ?? 0));
|
|
assert.ok(status.totals.byBackend.puppet.pending >= 3);
|
|
assert.equal(status.totals.byBackend.slopbot.pending, 0);
|
|
assert.ok(status.totals.total >= 3);
|
|
assert.equal(job.members.some(member => member.scrapeId === first.id), true);
|
|
assert.equal(job.members.some(member => member.scrapeId === second.id), true);
|
|
queue.pause();
|
|
});
|
|
|
|
await test("media traffic serializes walks and backfill batches", async () => {
|
|
const { backend, release } = gatedBackend({ messages: [message("320")] });
|
|
const engine = engineFor(root, backend);
|
|
await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
|
|
await new Promise(resolve => setImmediate(resolve));
|
|
|
|
assert.equal(engine.isWalking(), true);
|
|
|
|
const order: string[] = [];
|
|
const queued = engine.acquireTraffic().then(releaseSecond => {
|
|
order.push("batch");
|
|
releaseSecond();
|
|
});
|
|
|
|
// While the walk holds the lock, the batch cannot start.
|
|
await new Promise(resolve => setTimeout(resolve, 5));
|
|
assert.deepEqual(order, []);
|
|
|
|
release();
|
|
await engine.processQueue();
|
|
await queued;
|
|
|
|
assert.deepEqual(order, ["batch"]);
|
|
assert.equal(engine.isWalking(), false);
|
|
});
|
|
|
|
await test("backfill queue processes batches and finishes", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("662", {
|
|
attachments: [
|
|
attachment({ filename: "x.png", contentType: "image/png" }),
|
|
attachment({ filename: "y.png", contentType: "image/png" }),
|
|
attachment({ filename: "z.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = engineFor(root, backend, {
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "201", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const queue = new BackfillQueue(engine, {
|
|
rootDir: root,
|
|
getSettings: () => ({ batchSize: 1, batchPauseMs: 0, auto: false }),
|
|
getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
|
|
sleep: async () => {}
|
|
});
|
|
|
|
queue.start();
|
|
await queue.idle();
|
|
|
|
const status = queue.list();
|
|
const job = status.jobs.find(candidate => candidate.chatId === "201");
|
|
assert.ok(job);
|
|
assert.equal(job.state, "done");
|
|
assert.equal(job.pending, 0);
|
|
assert.equal(job.cached, 3);
|
|
assert.ok(job.processed >= 3);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.equal(store.previewCounts().cached, 3);
|
|
store.close();
|
|
});
|
|
|
|
await test("backfill queue pause leaves rows pending", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("663", {
|
|
attachments: [
|
|
attachment({ filename: "p.png", contentType: "image/png" }),
|
|
attachment({ filename: "q.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = engineFor(root, backend, {
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
await engine.start({ channelId: "202", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const queue = new BackfillQueue(engine, {
|
|
rootDir: root,
|
|
getSettings: () => ({ batchSize: 1, batchPauseMs: 0, auto: false }),
|
|
getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
|
|
sleep: async () => {}
|
|
});
|
|
|
|
queue.start();
|
|
queue.pause();
|
|
await queue.idle();
|
|
|
|
const status = queue.list();
|
|
const job = status.jobs.find(candidate => candidate.chatId === "202");
|
|
assert.ok(job);
|
|
assert.equal(job.state, "paused");
|
|
assert.equal(job.pending, 2);
|
|
});
|
|
|
|
await test("backfillPreviews fills rows left by a preview-less walk", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("800", {
|
|
attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: null,
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
}
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
assert.equal(engine.get(started.id)?.previewed, undefined);
|
|
|
|
const added = await engine.backfillPreviews(started.id);
|
|
assert.equal(added, 2);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
|
|
assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
|
|
store.close();
|
|
});
|
|
|
|
await test("dedupe prefers the richer row for a URL", () => {
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: false },
|
|
{ downloaded: false, hasPreview: true }
|
|
),
|
|
true
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: true },
|
|
{ downloaded: false, hasPreview: false }
|
|
),
|
|
false
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: true, hasPreview: false },
|
|
{ downloaded: false, hasPreview: true }
|
|
),
|
|
false
|
|
);
|
|
assert.equal(
|
|
betterCandidate(
|
|
{ downloaded: false, hasPreview: true },
|
|
{ downloaded: true, hasPreview: false }
|
|
),
|
|
true
|
|
);
|
|
});
|
|
|
|
await test("previews pause between batches", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("900", {
|
|
attachments: [
|
|
attachment({ filename: "a.png", contentType: "image/png" }),
|
|
attachment({ filename: "b.png", contentType: "image/png" }),
|
|
attachment({ filename: "c.png", contentType: "image/png" })
|
|
]
|
|
})
|
|
]
|
|
}
|
|
]);
|
|
|
|
const sleeps: number[] = [];
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 10,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: true,
|
|
download: {
|
|
fetchImpl: async () =>
|
|
new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
|
|
minGapMs: 0,
|
|
maxGapMs: 0,
|
|
previewMinGapMs: 0,
|
|
previewMaxGapMs: 0,
|
|
previewBatchSize: 2,
|
|
previewBatchPauseMs: 5000,
|
|
sleep: async () => {}
|
|
}
|
|
});
|
|
|
|
const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
assert.equal(engine.get(started.id)?.previewed, 3);
|
|
assert.equal(sleeps.filter(ms => ms === 5000).length, 1);
|
|
});
|
|
|
|
await test("remoteFrame renders an audio waveform", async () => {
|
|
const dir = join(root, "waveform-test");
|
|
mkdirSync(dir, { recursive: true });
|
|
const wavPath = join(dir, "tone.wav");
|
|
|
|
try {
|
|
execFileSync("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"sine=frequency=440:duration=1",
|
|
wavPath
|
|
]);
|
|
} catch {
|
|
console.log("ok - remoteFrame skipped (no ffmpeg)");
|
|
return;
|
|
}
|
|
|
|
const frame = await remoteFrame(wavPath, dir, { waveform: true });
|
|
assert.ok(frame?.hash);
|
|
assert.ok((frame?.bytes ?? 0) > 0);
|
|
assert.equal(existsSync(join(dir, `${frame?.hash}.webp`)), true);
|
|
});
|
|
|
|
await test("snowflake helpers round-trip dates", () => {
|
|
const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
|
|
assert.ok(id);
|
|
assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z");
|
|
assert.equal(snowflakeFromIso("not a date"), null);
|
|
assert.equal(isoFromSnowflake("nope"), null);
|
|
});
|
|
|
|
await test("engine honors a from-date boundary", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("1000"), message("900"), message("800")] }
|
|
]);
|
|
|
|
const engine = new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 10,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
previews: false
|
|
});
|
|
|
|
const started = await engine.start({
|
|
channelId: "131",
|
|
guildId: null,
|
|
mode: "links",
|
|
backend,
|
|
afterId: "900"
|
|
});
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.messages, 1);
|
|
assert.equal(meta?.afterId, "900");
|
|
assert.equal(backend.calls.length, 1);
|
|
|
|
const store = new ScrapeStore(join(root, started.id));
|
|
assert.equal(store.queryMedia({}).length, 0);
|
|
store.close();
|
|
});
|
|
|
|
await test("retention marks the oldest completed scrapes up to the limit", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 100);
|
|
assert.deepEqual(plan.mark, ["old", "new"]);
|
|
assert.deepEqual(plan.delete, []);
|
|
});
|
|
|
|
await test("retention deletes marked scrapes past the overrun allowance", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 5);
|
|
assert.deepEqual(plan.delete, ["old", "new"]);
|
|
assert.deepEqual(plan.mark, []);
|
|
});
|
|
|
|
await test("retention leaves usage under the limit alone", () => {
|
|
const plan = planRetention(
|
|
[{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }],
|
|
1000,
|
|
5
|
|
);
|
|
|
|
assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 });
|
|
assert.equal(isNearLimit(900, 1000, 90), true);
|
|
assert.equal(isNearLimit(899, 1000, 90), false);
|
|
});
|
|
|
|
rmSync(root, { recursive: true, force: true });
|
|
|
|
if (failures > 0) {
|
|
console.error(`${failures} test(s) failed`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("all media tests passed");
|
|
})();
|