Add the media downloader with hash storage and local file serving

2026-09-16 10:35:55 +02:00
parent 3ea72f587d
commit f7664256d1
15 changed files with 431 additions and 28 deletions
@@ -4,9 +4,11 @@
 */

 import assert from "node:assert/strict";
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { downloadOne } from "../src/download.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
@@ -276,6 +278,122 @@ void (async () => {
        assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
    });

+    await test("download stores bytes by hash and skips expired URLs", async () => {
+        const dir = join(root, "download-test");
+        mkdirSync(dir, { recursive: true });
+
+        const ok = await downloadOne(
+            { id: 1, url: "https://cdn.discordapp.com/attachments/1/2/note.txt", kind: "image" },
+            dir,
+            {
+                fetchImpl: async () =>
+                    new Response("hello", { status: 200, headers: { "content-type": "text/plain" } })
+            }
+        );
+        assert.equal(ok.state, "downloaded");
+        assert.equal(ok.hash, createHash("sha256").update("hello").digest("hex"));
+        assert.equal(existsSync(join(dir, `${ok.hash}.txt`)), true);
+
+        const expired = await downloadOne(
+            { id: 2, url: "https://cdn.discordapp.com/attachments/1/2/gone.png", kind: "image" },
+            dir,
+            { fetchImpl: async () => new Response("no", { status: 404 }) }
+        );
+        assert.equal(expired.state, "skipped");
+
+        const large = await downloadOne(
+            { id: 3, url: "https://cdn.discordapp.com/attachments/1/2/big.mp4", kind: "video" },
+            dir,
+            {
+                fetchImpl: async () =>
+                    new Response("x", { status: 200, headers: { "content-length": "999999999" } })
+            }
+        );
+        assert.equal(large.state, "skipped");
+    });
+
+    await test("download retries a 5xx and then succeeds", async () => {
+        const dir = join(root, "download-retry");
+        mkdirSync(dir, { recursive: true });
+
+        let calls = 0;
+        const outcome = await downloadOne(
+            { id: 4, url: "https://cdn.discordapp.com/attachments/1/2/flaky.png", kind: "image" },
+            dir,
+            {
+                maxRetries: 2,
+                sleep: async () => {},
+                fetchImpl: async () => {
+                    calls++;
+                    return calls === 1
+                        ? new Response("boom", { status: 500 })
+                        : new Response("bytes", { status: 200, headers: { "content-type": "image/png" } });
+                }
+            }
+        );
+
+        assert.equal(outcome.state, "downloaded");
+        assert.equal(calls, 2);
+    });
+
+    await test("engine downloads attachments in download mode", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("500", {
+                        attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://www.youtube.com/watch?v=x",
+                                title: null,
+                                provider: "YouTube",
+                                videoUrl: "https://www.youtube.com/embed/x",
+                                imageUrl: null,
+                                thumbnailUrl: null
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            download: {
+                fetchImpl: async () =>
+                    new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "127", guildId: null, mode: "download", backend });
+        await engine.processQueue();
+
+        const meta = engine.get(started.id);
+        assert.equal(meta?.state, "done");
+        assert.equal(meta?.downloaded, 1);
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "image" })[0];
+        assert.equal(row.downloaded, 1);
+        assert.ok(row.file_path);
+        assert.ok((row.file_bytes ?? 0) > 0);
+        const embed = store.queryMedia({ kind: "video" })[0];
+        assert.equal(embed.downloaded, 0);
+        assert.equal(embed.file_path, null);
+        store.close();
+        assert.equal(existsSync(join(root, started.id, row.file_path)), true);
+    });
+
    await test("retention marks the oldest completed scrapes up to the limit", () => {
        const scrapes = [
            { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },