324 lines
12 KiB
TypeScript
324 lines
12 KiB
TypeScript
/*
|
|
* Media archive unit tests. No Discord, no network: a fake backend and a temp DB root.
|
|
* Run with: pnpm test:media (from poc/).
|
|
*/
|
|
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
|
import { extractMedia } from "../src/extract.js";
|
|
import { PuppetBackend } from "../src/puppet-backend.js";
|
|
import { isNearLimit, planRetention } from "../src/retention.js";
|
|
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
|
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
|
|
|
let failures = 0;
|
|
|
|
async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
|
|
try {
|
|
await fn();
|
|
console.log(`ok - ${name}`);
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(`not ok - ${name}`);
|
|
console.error(error);
|
|
}
|
|
}
|
|
|
|
function message(id: string, extra: Partial<CoreMessage> = {}): CoreMessage {
|
|
return {
|
|
id,
|
|
author: { id: "1", username: "tester", avatar: null },
|
|
content: "",
|
|
timestamp: "2026-09-16T00:00:00.000Z",
|
|
editedAt: null,
|
|
replyTo: null,
|
|
attachments: [],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
forwarded: [],
|
|
voiceMessage: false,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
function attachment(extra: Partial<CoreAttachment> = {}): CoreAttachment {
|
|
const filename = extra.filename ?? "file.bin";
|
|
return {
|
|
id: null,
|
|
filename,
|
|
url: `https://cdn.discordapp.com/attachments/1/2/${filename}`,
|
|
contentType: null,
|
|
size: null,
|
|
width: null,
|
|
height: null,
|
|
durationSecs: null,
|
|
...extra
|
|
};
|
|
}
|
|
|
|
class FakeBackend implements ScrapeBackend {
|
|
readonly name = "puppet" as const;
|
|
readonly calls: Array<{ before: string | null; limit: number }> = [];
|
|
|
|
constructor(private readonly pages: Array<FetchPage | Error>) {}
|
|
|
|
async fetchPage(_channelId: string, before: string | null, limit: number): Promise<FetchPage> {
|
|
this.calls.push({ before, limit });
|
|
const page = this.pages.shift();
|
|
if (!page) return { messages: [] };
|
|
if (page instanceof Error) throw page;
|
|
return page;
|
|
}
|
|
}
|
|
|
|
function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, unknown> = {}) {
|
|
return new ScrapeEngine({
|
|
rootDir: root,
|
|
pageSize: 2,
|
|
pageMinMs: 0,
|
|
pageMaxMs: 0,
|
|
sleep: async () => {},
|
|
random: () => 0,
|
|
log: () => {},
|
|
...extra
|
|
});
|
|
}
|
|
|
|
void (async () => {
|
|
const root = mkdtempSync(join(tmpdir(), "veskslop-media-"));
|
|
|
|
await test("extract classifies attachments, embeds, and stickers", () => {
|
|
const items = extractMedia(
|
|
message("100", {
|
|
attachments: [
|
|
attachment({ filename: "cat.png", contentType: "image/png" }),
|
|
attachment({ filename: "loop.gif" }),
|
|
attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234 }),
|
|
attachment({ filename: "note.ogg", durationSecs: 3.2 })
|
|
],
|
|
embeds: [
|
|
{
|
|
type: "video",
|
|
url: "https://www.youtube.com/watch?v=x",
|
|
title: "x",
|
|
provider: "YouTube",
|
|
videoUrl: "https://www.youtube.com/embed/x",
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
|
},
|
|
{
|
|
type: "gifv",
|
|
url: "https://tenor.com/view/x",
|
|
title: null,
|
|
provider: "Tenor",
|
|
videoUrl: null,
|
|
imageUrl: null,
|
|
thumbnailUrl: "https://media.tenor.com/x.gif"
|
|
}
|
|
],
|
|
stickerItems: [
|
|
{ id: "55", name: "wave", formatType: 4 },
|
|
{ id: "56", name: "lottie", formatType: 3 }
|
|
]
|
|
})
|
|
);
|
|
|
|
const kinds = items.map(item => item.kind).sort();
|
|
assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]);
|
|
const youtube = items.find(item => item.url.includes("youtube.com/watch"));
|
|
assert.equal(youtube?.kind, "video");
|
|
assert.equal(youtube?.provider, "YouTube");
|
|
assert.equal(youtube?.downloaded, false);
|
|
assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif");
|
|
assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
|
|
assert.equal(items.some(item => item.url.includes("lottie")), false);
|
|
assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2);
|
|
});
|
|
|
|
await test("extract finds media in forwarded snapshots", () => {
|
|
const items = extractMedia(
|
|
message("101", {
|
|
forwarded: [
|
|
{
|
|
content: "",
|
|
timestamp: null,
|
|
attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })],
|
|
embeds: [],
|
|
stickerItems: [],
|
|
voiceMessage: false
|
|
}
|
|
]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "video");
|
|
});
|
|
|
|
await test("extract treats a voice message as audio", () => {
|
|
const items = extractMedia(
|
|
message("102", {
|
|
voiceMessage: true,
|
|
attachments: [attachment({ filename: "voice-message.ogg" })]
|
|
})
|
|
);
|
|
|
|
assert.equal(items.length, 1);
|
|
assert.equal(items[0].kind, "audio");
|
|
});
|
|
|
|
await test("engine walks pages, persists meta, and stores media", async () => {
|
|
const backend = new FakeBackend([
|
|
{
|
|
messages: [
|
|
message("100", {
|
|
attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })]
|
|
}),
|
|
message("101")
|
|
]
|
|
},
|
|
{ messages: [message("99")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.pages, 2);
|
|
assert.equal(meta?.messages, 3);
|
|
assert.equal(meta?.cursor, "99");
|
|
assert.equal(meta?.media.image, 1);
|
|
assert.equal(backend.calls[0].limit, 2);
|
|
assert.equal(backend.calls[1].before, "100");
|
|
|
|
const dir = join(root, started.id);
|
|
assert.equal(existsSync(join(dir, "scrape.json")), true);
|
|
assert.equal(existsSync(join(dir, "index.db")), true);
|
|
assert.ok(dirSizeBytes(dir) > 0);
|
|
|
|
const stored = new ScrapeStore(dir);
|
|
assert.equal(stored.counts().image, 1);
|
|
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
|
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
|
assert.equal(stored.storedBytesByKind().image, 0);
|
|
stored.close();
|
|
|
|
assert.equal(listScrapes(root).length, 1);
|
|
});
|
|
|
|
await test("engine backs off and retries with retryAfterMs", async () => {
|
|
const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 });
|
|
const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]);
|
|
const sleeps: number[] = [];
|
|
|
|
const engine = engineFor(root, backend, {
|
|
maxRetries: 3,
|
|
sleep: async (ms: number) => {
|
|
sleeps.push(ms);
|
|
}
|
|
});
|
|
const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "done");
|
|
assert.equal(meta?.errors, 2);
|
|
assert.deepEqual(sleeps, [250, 3000]);
|
|
});
|
|
|
|
await test("engine pauses when the page budget is reached", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("200"), message("201")] },
|
|
{ messages: [message("199"), message("198")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend, { pagesPerDay: 1 });
|
|
const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend });
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
assert.equal(backend.calls.length, 1);
|
|
});
|
|
|
|
await test("engine stop pauses after the current page", async () => {
|
|
const backend = new FakeBackend([
|
|
{ messages: [message("300"), message("301")] },
|
|
{ messages: [message("299")] }
|
|
]);
|
|
|
|
const engine = engineFor(root, backend);
|
|
const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend });
|
|
engine.stop(started.id);
|
|
await engine.processQueue();
|
|
|
|
const meta = engine.get(started.id);
|
|
assert.equal(meta?.state, "paused");
|
|
assert.equal(meta?.pages, 1);
|
|
});
|
|
|
|
await test("puppet backend passes before and limit to the core", async () => {
|
|
const calls: Array<Record<string, unknown>> = [];
|
|
const backend = new PuppetBackend(async (channelId, limit, options) => {
|
|
calls.push({ channelId, limit, ...options });
|
|
return { messages: [message("50")] };
|
|
});
|
|
|
|
const page = await backend.fetchPage("123", "60", 5);
|
|
assert.equal(page.messages.length, 1);
|
|
assert.equal(page.messages[0].id, "50");
|
|
assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
|
|
});
|
|
|
|
await test("retention marks the oldest completed scrapes up to the limit", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 100);
|
|
assert.deepEqual(plan.mark, ["old", "new"]);
|
|
assert.deepEqual(plan.delete, []);
|
|
});
|
|
|
|
await test("retention deletes marked scrapes past the overrun allowance", () => {
|
|
const scrapes = [
|
|
{ id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
|
{ id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
|
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
|
];
|
|
|
|
const plan = planRetention(scrapes, 1000, 5);
|
|
assert.deepEqual(plan.delete, ["old", "new"]);
|
|
assert.deepEqual(plan.mark, []);
|
|
});
|
|
|
|
await test("retention leaves usage under the limit alone", () => {
|
|
const plan = planRetention(
|
|
[{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }],
|
|
1000,
|
|
5
|
|
);
|
|
|
|
assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 });
|
|
assert.equal(isNearLimit(900, 1000, 90), true);
|
|
assert.equal(isNearLimit(899, 1000, 90), false);
|
|
});
|
|
|
|
rmSync(root, { recursive: true, force: true });
|
|
|
|
if (failures > 0) {
|
|
console.error(`${failures} test(s) failed`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("all media tests passed");
|
|
})();
|