Fetch previews in batches with a rate-limit cooldown

2026-09-16 23:51:16 +02:00
parent 8f30ec14ad
commit 6c500f0cc5
7 changed files with 78 additions and 1 deletions
@@ -961,6 +961,53 @@ void (async () => {
        );
    });

+    await test("previews pause between batches", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("900", {
+                        attachments: [
+                            attachment({ filename: "a.png", contentType: "image/png" }),
+                            attachment({ filename: "b.png", contentType: "image/png" }),
+                            attachment({ filename: "c.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const sleeps: number[] = [];
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async (ms: number) => {
+                sleeps.push(ms);
+            },
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 2,
+                previewBatchPauseMs: 5000,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        assert.equal(engine.get(started.id)?.previewed, 3);
+        assert.equal(sleeps.filter(ms => ms === 5000).length, 1);
+    });
+
    await test("remoteFrame renders an audio waveform", async () => {
        const dir = join(root, "waveform-test");
        mkdirSync(dir, { recursive: true });