Serialize walks and backfill, split the ETA into start and run

media-scrape/test/run.ts

@@ -95,6 +95,26 @@ class FakeBackend implements ScrapeBackend {
     }
 }
 
+/** A backend whose first page waits for `release()`, so a test can stop the walk mid-page. */
+function gatedBackend(page: FetchPage): { backend: ScrapeBackend; release: () => void } {
+    let release!: () => void;
+    const gate = new Promise<void>(resolve => (release = resolve));
+    let first = true;
+
+    const backend: ScrapeBackend = {
+        name: "puppet",
+        async fetchPage() {
+            if (first) {
+                first = false;
+                await gate;
+            }
+            return page;
+        }
+    };
+
+    return { backend, release };
+}
+
 function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, unknown> = {}) {
     return new ScrapeEngine({
         rootDir: root,
@@ -276,14 +296,15 @@ void (async () => {
     });
 
     await test("engine stop pauses after the current page", async () => {
-        const backend = new FakeBackend([
-            { messages: [message("300"), message("301")] },
-            { messages: [message("299")] }
-        ]);
+        const { backend, release } = gatedBackend({ messages: [message("300"), message("301")] });
 
         const engine = engineFor(root, backend);
         const started = await engine.start({ channelId: "126", guildId: null, mode: "links", backend });
+
+        // Stop while the first page is in flight; the page lands, then the walk pauses.
+        await new Promise(resolve => setImmediate(resolve));
         engine.stop(started.id);
+        release();
         await engine.processQueue();
 
         const meta = engine.get(started.id);
@@ -291,6 +312,20 @@ void (async () => {
         assert.equal(meta?.pages, 1);
     });
 
+    await test("an immediate stop fetches nothing", async () => {
+        const backend = new FakeBackend([{ messages: [message("310")] }]);
+
+        const engine = engineFor(root, backend);
+        const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
+        engine.stop(started.id);
+        await engine.processQueue();
+
+        const meta = engine.get(started.id);
+        assert.equal(meta?.state, "paused");
+        assert.equal(meta?.pages, 0);
+        assert.equal(backend.calls.length, 0);
+    });
+
     await test("puppet backend passes before and limit to the core", async () => {
         const calls: Array<Record<string, unknown>> = [];
         const backend = new PuppetBackend(async (channelId, limit, options) => {
@@ -1100,14 +1135,15 @@ void (async () => {
     });
 
     await test("engine resumes a paused scrape from its cursor", async () => {
-        const first = new FakeBackend([
-            { messages: [message("500"), message("499")] },
-            { messages: [message("498")] }
-        ]);
+        const { backend: first, release } = gatedBackend({ messages: [message("500"), message("499")] });
 
         const engine = engineFor(root, first);
         const started = await engine.start({ channelId: "140", guildId: null, mode: "links", backend: first });
+
+        // Stop while the first page is in flight so the cursor is stored, then pauses.
+        await new Promise(resolve => setImmediate(resolve));
         engine.stop(started.id);
+        release();
         await engine.processQueue();
 
         let meta = engine.get(started.id);
@@ -1345,17 +1381,70 @@ void (async () => {
             sleep: async () => {}
         });
 
+        const small = new FakeBackend([
+            {
+                messages: [
+                    message("662", {
+                        attachments: [attachment({ filename: "small.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+        await engine.start({ channelId: "203", guildId: null, mode: "links", backend: small });
+        await engine.processQueue();
+
         const status = queue.list();
         const job = status.jobs.find(candidate => candidate.chatId === "200");
+        const smallJob = status.jobs.find(candidate => candidate.chatId === "203");
         assert.ok(job);
+        assert.ok(smallJob);
         assert.equal(job.members.length, 2);
         assert.equal(job.pending, 3);
         assert.ok((job.etaMs ?? 0) > 0);
+
+        // Start everything: smallest first, so 203 runs and 200 waits behind it.
+        queue.start();
+        const scheduled = queue.list();
+        const firstJob = scheduled.jobs.find(candidate => candidate.chatId === "203");
+        const secondJob = scheduled.jobs.find(candidate => candidate.chatId === "200");
+        assert.ok(firstJob);
+        assert.ok(secondJob);
+        assert.equal(firstJob.etaStartMs, 0);
+        assert.ok((firstJob.etaRunMs ?? 0) > 0);
+        assert.equal(secondJob.etaTotalMs, (secondJob.etaStartMs ?? 0) + (secondJob.etaRunMs ?? 0));
+        assert.ok((secondJob.etaStartMs ?? 0) >= (firstJob.etaRunMs ?? 0));
         assert.ok(status.totals.byBackend.puppet.pending >= 3);
         assert.equal(status.totals.byBackend.slopbot.pending, 0);
         assert.ok(status.totals.total >= 3);
         assert.equal(job.members.some(member => member.scrapeId === first.id), true);
         assert.equal(job.members.some(member => member.scrapeId === second.id), true);
+        queue.pause();
+    });
+
+    await test("media traffic serializes walks and backfill batches", async () => {
+        const { backend, release } = gatedBackend({ messages: [message("320")] });
+        const engine = engineFor(root, backend);
+        await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
+        await new Promise(resolve => setImmediate(resolve));
+
+        assert.equal(engine.isWalking(), true);
+
+        const order: string[] = [];
+        const queued = engine.acquireTraffic().then(releaseSecond => {
+            order.push("batch");
+            releaseSecond();
+        });
+
+        // While the walk holds the lock, the batch cannot start.
+        await new Promise(resolve => setTimeout(resolve, 5));
+        assert.deepEqual(order, []);
+
+        release();
+        await engine.processQueue();
+        await queued;
+
+        assert.deepEqual(order, ["batch"]);
+        assert.equal(engine.isWalking(), false);
     });
 
     await test("backfill queue processes batches and finishes", async () => {