Add the per-chat backfill queue with progress and ETA

2026-09-17 08:53:51 +02:00
parent 2e317ba1e1
commit 16ce809270
3 changed files with 571 additions and 0 deletions
@@ -19,6 +19,7 @@ import {
 } from "../src/download.js";
 import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
+import { BackfillQueue } from "../src/backfill.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { betterCandidate } from "../src/dedupe.js";
@@ -1248,6 +1249,149 @@ void (async () => {
        store.close();
    });

+    await test("backfill queue groups scrapes per chat and estimates an ETA", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("660", {
+                        attachments: [
+                            attachment({ filename: "a.png", contentType: "image/png" }),
+                            attachment({ filename: "b.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            },
+            {
+                messages: [
+                    message("661", {
+                        attachments: [attachment({ filename: "c.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = engineFor(root, backend);
+        const first = await engine.start({ channelId: "200", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        const second = await engine.start({ channelId: "200", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const queue = new BackfillQueue(engine, {
+            rootDir: root,
+            getSettings: () => ({ batchSize: 2, batchPauseMs: 0, auto: false }),
+            getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
+            sleep: async () => {}
+        });
+
+        const status = queue.list();
+        const job = status.jobs.find(candidate => candidate.chatId === "200");
+        assert.ok(job);
+        assert.equal(job.members.length, 2);
+        assert.equal(job.pending, 3);
+        assert.ok((job.etaMs ?? 0) > 0);
+        assert.ok(status.totals.byBackend.puppet.pending >= 3);
+        assert.equal(status.totals.byBackend.slopbot.pending, 0);
+        assert.ok(status.totals.total >= 3);
+        assert.equal(job.members.some(member => member.scrapeId === first.id), true);
+        assert.equal(job.members.some(member => member.scrapeId === second.id), true);
+    });
+
+    await test("backfill queue processes batches and finishes", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("662", {
+                        attachments: [
+                            attachment({ filename: "x.png", contentType: "image/png" }),
+                            attachment({ filename: "y.png", contentType: "image/png" }),
+                            attachment({ filename: "z.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = engineFor(root, backend, {
+            download: {
+                fetchImpl: async () =>
+                    new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "201", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const queue = new BackfillQueue(engine, {
+            rootDir: root,
+            getSettings: () => ({ batchSize: 1, batchPauseMs: 0, auto: false }),
+            getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
+            sleep: async () => {}
+        });
+
+        queue.start();
+        await queue.idle();
+
+        const status = queue.list();
+        const job = status.jobs.find(candidate => candidate.chatId === "201");
+        assert.ok(job);
+        assert.equal(job.state, "done");
+        assert.equal(job.pending, 0);
+        assert.equal(job.cached, 3);
+        assert.ok(job.processed >= 3);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.equal(store.previewCounts().cached, 3);
+        store.close();
+    });
+
+    await test("backfill queue pause leaves rows pending", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("663", {
+                        attachments: [
+                            attachment({ filename: "p.png", contentType: "image/png" }),
+                            attachment({ filename: "q.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = engineFor(root, backend, {
+            download: {
+                fetchImpl: async () =>
+                    new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        await engine.start({ channelId: "202", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const queue = new BackfillQueue(engine, {
+            rootDir: root,
+            getSettings: () => ({ batchSize: 1, batchPauseMs: 0, auto: false }),
+            getGaps: () => ({ minGapMs: 0, maxGapMs: 0 }),
+            sleep: async () => {}
+        });
+
+        queue.start();
+        queue.pause();
+        await queue.idle();
+
+        const status = queue.list();
+        const job = status.jobs.find(candidate => candidate.chatId === "202");
+        assert.ok(job);
+        assert.equal(job.state, "paused");
+        assert.equal(job.pending, 2);
+    });
+
    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
        const backend = new FakeBackend([
            {