Add the media scrape engine, store, and puppet walker
This commit is contained in:
Fisto
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Media archive unit tests. No Discord, no network: a fake backend and a temp DB root.
|
||||
* Run with: pnpm test:media (from poc/).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
||||
import { extractMedia } from "../src/extract.js";
|
||||
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
||||
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
||||
|
||||
let failures = 0;
|
||||
|
||||
async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
|
||||
try {
|
||||
await fn();
|
||||
console.log(`ok - ${name}`);
|
||||
} catch (error) {
|
||||
failures++;
|
||||
console.error(`not ok - ${name}`);
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
function message(id: string, extra: Partial<CoreMessage> = {}): CoreMessage {
|
||||
return {
|
||||
id,
|
||||
author: { id: "1", username: "tester", avatar: null },
|
||||
content: "",
|
||||
timestamp: "2026-09-16T00:00:00.000Z",
|
||||
editedAt: null,
|
||||
replyTo: null,
|
||||
attachments: [],
|
||||
embeds: [],
|
||||
stickerItems: [],
|
||||
forwarded: [],
|
||||
voiceMessage: false,
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
function attachment(extra: Partial<CoreAttachment> = {}): CoreAttachment {
|
||||
const filename = extra.filename ?? "file.bin";
|
||||
return {
|
||||
id: null,
|
||||
filename,
|
||||
url: `https://cdn.discordapp.com/attachments/1/2/${filename}`,
|
||||
contentType: null,
|
||||
size: null,
|
||||
width: null,
|
||||
height: null,
|
||||
durationSecs: null,
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
class FakeBackend implements ScrapeBackend {
|
||||
readonly name = "puppet" as const;
|
||||
readonly calls: Array<{ before: string | null; limit: number }> = [];
|
||||
|
||||
constructor(private readonly pages: Array<FetchPage | Error>) {}
|
||||
|
||||
async fetchPage(_channelId: string, before: string | null, limit: number): Promise<FetchPage> {
|
||||
this.calls.push({ before, limit });
|
||||
const page = this.pages.shift();
|
||||
if (!page) return { messages: [] };
|
||||
if (page instanceof Error) throw page;
|
||||
return page;
|
||||
}
|
||||
}
|
||||
|
||||
function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, unknown> = {}) {
|
||||
return new ScrapeEngine({
|
||||
rootDir: root,
|
||||
pageSize: 2,
|
||||
pageMinMs: 0,
|
||||
pageMaxMs: 0,
|
||||
sleep: async () => {},
|
||||
random: () => 0,
|
||||
log: () => {},
|
||||
...extra
|
||||
});
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "veskslop-media-"));
|
||||
|
||||
await test("extract classifies attachments, embeds, and stickers", () => {
|
||||
const items = extractMedia(
|
||||
message("100", {
|
||||
attachments: [
|
||||
attachment({ filename: "cat.png", contentType: "image/png" }),
|
||||
attachment({ filename: "loop.gif" }),
|
||||
attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234 }),
|
||||
attachment({ filename: "note.ogg", durationSecs: 3.2 })
|
||||
],
|
||||
embeds: [
|
||||
{
|
||||
type: "video",
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
title: "x",
|
||||
provider: "YouTube",
|
||||
videoUrl: "https://www.youtube.com/embed/x",
|
||||
imageUrl: null,
|
||||
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
},
|
||||
{
|
||||
type: "gifv",
|
||||
url: "https://tenor.com/view/x",
|
||||
title: null,
|
||||
provider: "Tenor",
|
||||
videoUrl: null,
|
||||
imageUrl: null,
|
||||
thumbnailUrl: "https://media.tenor.com/x.gif"
|
||||
}
|
||||
],
|
||||
stickerItems: [
|
||||
{ id: "55", name: "wave", formatType: 4 },
|
||||
{ id: "56", name: "lottie", formatType: 3 }
|
||||
]
|
||||
})
|
||||
);
|
||||
|
||||
const kinds = items.map(item => item.kind).sort();
|
||||
assert.deepEqual(kinds, ["audio", "gif", "gif", "gif", "image", "video", "video"]);
|
||||
const youtube = items.find(item => item.url.includes("youtube.com/watch"));
|
||||
assert.equal(youtube?.kind, "video");
|
||||
assert.equal(youtube?.provider, "YouTube");
|
||||
assert.equal(youtube?.downloaded, false);
|
||||
assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif");
|
||||
assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
|
||||
assert.equal(items.some(item => item.url.includes("lottie")), false);
|
||||
assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2);
|
||||
});
|
||||
|
||||
await test("extract finds media in forwarded snapshots", () => {
|
||||
const items = extractMedia(
|
||||
message("101", {
|
||||
forwarded: [
|
||||
{
|
||||
content: "",
|
||||
timestamp: null,
|
||||
attachments: [attachment({ filename: "fwd.webm", contentType: "video/webm" })],
|
||||
embeds: [],
|
||||
stickerItems: [],
|
||||
voiceMessage: false
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(items.length, 1);
|
||||
assert.equal(items[0].kind, "video");
|
||||
});
|
||||
|
||||
await test("extract treats a voice message as audio", () => {
|
||||
const items = extractMedia(
|
||||
message("102", {
|
||||
voiceMessage: true,
|
||||
attachments: [attachment({ filename: "voice-message.ogg" })]
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(items.length, 1);
|
||||
assert.equal(items[0].kind, "audio");
|
||||
});
|
||||
|
||||
await test("engine walks pages, persists meta, and stores media", async () => {
|
||||
const backend = new FakeBackend([
|
||||
{
|
||||
messages: [
|
||||
message("100", { attachments: [attachment({ filename: "one.png", contentType: "image/png" })] }),
|
||||
message("101")
|
||||
]
|
||||
},
|
||||
{ messages: [message("99")] }
|
||||
]);
|
||||
|
||||
const engine = engineFor(root, backend);
|
||||
const started = await engine.start({ channelId: "123", guildId: null, mode: "links", backend });
|
||||
await engine.processQueue();
|
||||
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.pages, 2);
|
||||
assert.equal(meta?.messages, 3);
|
||||
assert.equal(meta?.cursor, "99");
|
||||
assert.equal(meta?.media.image, 1);
|
||||
assert.equal(backend.calls[0].limit, 2);
|
||||
assert.equal(backend.calls[1].before, "100");
|
||||
|
||||
const dir = join(root, started.id);
|
||||
assert.equal(existsSync(join(dir, "scrape.json")), true);
|
||||
assert.equal(existsSync(join(dir, "index.db")), true);
|
||||
assert.ok(dirSizeBytes(dir) > 0);
|
||||
|
||||
const stored = new ScrapeStore(dir);
|
||||
assert.equal(stored.counts().image, 1);
|
||||
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
||||
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
||||
stored.close();
|
||||
|
||||
assert.equal(listScrapes(root).length, 1);
|
||||
});
|
||||
|
||||
await test("engine backs off and retries with retryAfterMs", async () => {
|
||||
const rate = Object.assign(new Error("rate limited"), { retryAfterMs: 250 });
|
||||
const backend = new FakeBackend([rate, new Error("boom"), { messages: [] }]);
|
||||
const sleeps: number[] = [];
|
||||
|
||||
const engine = engineFor(root, backend, {
|
||||
maxRetries: 3,
|
||||
sleep: async (ms: number) => {
|
||||
sleeps.push(ms);
|
||||
}
|
||||
});
|
||||
const started = await engine.start({ channelId: "124", guildId: null, mode: "links", backend });
|
||||
await engine.processQueue();
|
||||
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.errors, 2);
|
||||
assert.deepEqual(sleeps, [250, 3000]);
|
||||
});
|
||||
|
||||
await test("engine pauses when the page budget is reached", async () => {
|
||||
const backend = new FakeBackend([
|
||||
{ messages: [message("200"), message("201")] },
|
||||
{ messages: [message("199"), message("198")] }
|
||||
]);
|
||||
|
||||
const engine = engineFor(root, backend, { pagesPerDay: 1 });
|
||||
const started = await engine.start({ channelId: "125", guildId: null, mode: "links", backend });
|
||||
await engine.processQueue();
|
||||
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "paused");
|
||||
assert.equal(meta?.pages, 1);
|
||||
assert.equal(backend.calls.length, 1);
|
||||
});
|
||||
|
||||
await test("engine stop pauses after the current page", async () => {
|
||||
const backend = new FakeBackend([
|
||||
{ messages: [message("300"), message("301")] },
|
||||
{ messages: [message("299")] }
|
||||
]);
|
||||
|
||||
const engine = engineFor(root, backend);
|
||||
const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend });
|
||||
engine.stop(started.id);
|
||||
await engine.processQueue();
|
||||
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "paused");
|
||||
assert.equal(meta?.pages, 1);
|
||||
});
|
||||
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`${failures} test(s) failed`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("all media tests passed");
|
||||
})();
|
||||
Reference in New Issue
Block a user