Expose preview batches, rate-limit status, and live option updates

2026-09-17 08:51:07 +02:00
parent e52623f1b2
commit 2e317ba1e1
4 changed files with 309 additions and 31 deletions
@@ -1118,6 +1118,136 @@ void (async () => {
        store.close();
    });

+    await test("previewBatch respects the batch limit and reports the remainder", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("655", {
+                        attachments: [
+                            attachment({ filename: "one.png", contentType: "image/png" }),
+                            attachment({ filename: "two.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            download: {
+                fetchImpl: async () =>
+                    new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "129", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const first = await engine.previewBatch(started.id, 1);
+        assert.equal(first.cached, 1);
+        assert.equal(first.remaining, 1);
+
+        const second = await engine.previewBatch(started.id, 5);
+        assert.equal(second.cached, 1);
+        assert.equal(second.remaining, 0);
+    });
+
+    await test("previewBatch registers a rate limit for the caller", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("656", {
+                        attachments: [attachment({ filename: "slow.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            rateLimitPauseMs: 30_000,
+            download: {
+                fetchImpl: async () =>
+                    new Response("slow down", { status: 429, headers: { "retry-after": "5" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "130", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const result = await engine.previewBatch(started.id, 5);
+        assert.equal(result.ratelimited, 1);
+        assert.equal(result.remaining, 1);
+        assert.equal(engine.rateLimitStatus().active, true);
+        assert.ok(engine.rateLimitStatus().remainingMs > 0);
+    });
+
+    await test("updateOptions retunes the attempt cap live", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("657", {
+                        attachments: [attachment({ filename: "dead.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            previewMaxAttempts: 5,
+            download: {
+                fetchImpl: async () => {
+                    throw new Error("fetch failed");
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "131", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        engine.updateOptions({ previewMaxAttempts: 1 });
+
+        const result = await engine.previewBatch(started.id, 5);
+        assert.equal(result.skipped, 1);
+        assert.equal(result.remaining, 0);
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "image" })[0];
+        assert.match(String(row.preview_error), /^unreachable after 1 attempt/);
+        store.close();
+    });
+
    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
        const backend = new FakeBackend([
            {