Add daemon restart, preview diagnostics, browse caching, and rate-limit pauses

2026-09-17 07:33:35 +02:00
parent cba7ae8143
commit dea1724eb3
13 changed files with 367 additions and 56 deletions
@@ -877,6 +877,79 @@ void (async () => {
        assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
    });

+    await test("markPreview stores the animated gif reference", () => {
+        const dir = join(root, "anim-store");
+        mkdirSync(dir, { recursive: true });
+
+        const store = new ScrapeStore(dir);
+        store.insertPage([
+            message("1200", { attachments: [attachment({ filename: "loop.gif" })] })
+        ]);
+
+        const row = store.queryMedia({ kind: "gif" })[0];
+        store.markPreview(row.id, { hash: "static-hash", bytes: 10, ext: "webp", animHash: "anim-hash" });
+
+        const updated = store.getMedia(row.id);
+        assert.equal(updated?.preview_hash, "static-hash");
+        assert.equal(updated?.anim_hash, "anim-hash");
+        store.close();
+    });
+
+    await test("rate limits pause media traffic", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("1300", {
+                        attachments: [
+                            attachment({ filename: "a.png", contentType: "image/png" }),
+                            attachment({ filename: "b.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const sleeps: number[] = [];
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async (ms: number) => {
+                sleeps.push(ms);
+            },
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            rateLimitPauseMs: 30_000,
+            download: {
+                fetchImpl: async () =>
+                    new Response("slow down", { status: 429, headers: { "retry-after": "10" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 100,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "160", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const store = new ScrapeStore(join(root, started.id));
+        for (const row of store.queryMedia({ kind: "image" })) {
+            assert.equal(row.preview_hash, null);
+            assert.equal(row.preview_error, null, "rate limited rows stay pending");
+        }
+        store.close();
+
+        assert.ok(
+            sleeps.some(ms => ms > 25_000 && ms <= 30_000),
+            "the engine paused after the rate limit"
+        );
+    });
+
    await test("previews run in cycles during the walk", async () => {
        const events: string[] = [];
        const pages: Array<{ messages: CoreMessage[] }> = [