Cycle previews through the walk

media-scrape/src/engine.ts

@@ -60,8 +60,10 @@ export interface ScrapeEngineOptions {
     log?: (line: string) => void;
     /** Download settings for `mode: "download"`. */
     download?: DownloadOptions;
-    /** Cache Discord-hosted image previews after every walk. Default true. */
+    /** Cache previews after every walk. Default true. */
     previews?: boolean;
+    /** Walk pages between preview cycles. 0 defers all previews to the end. Default 5. */
+    walkCyclePages?: number;
     /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
     probe?: (
         path: string
@@ -238,6 +240,8 @@ export class ScrapeEngine {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);
         const pageSize = this.options.pageSize ?? 100;
+        const perCycle = this.options.download?.previewBatchSize ?? 25;
+        let pagesSincePreview = 0;
 
         try {
             if (this.stopRequested.has(meta.id)) {
@@ -286,6 +290,17 @@ export class ScrapeEngine {
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
+                // Cycle between walking and previewing so tiles fill in during a long walk
+                // instead of only at the end.
+                const cycle = this.options.walkCyclePages ?? 5;
+                if (cycle > 0 && this.options.previews !== false) {
+                    pagesSincePreview++;
+                    if (pagesSincePreview >= cycle) {
+                        pagesSincePreview = 0;
+                        await this.runPreviews(store, dir, meta, perCycle);
+                    }
+                }
+
                 if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
@@ -390,13 +405,13 @@ export class ScrapeEngine {
     }
 
     /** Cache small previews. Embed thumbnails are cached in both modes. */
-    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta, limit = 0): Promise<void> {
         const options = {
             ...(this.options.download ?? {}),
             embedPreviews: true,
             destinationDir: meta.downloadDir ?? null
         };
-        const pending = store.pendingPreviews(5000, true);
+        const pending = store.pendingPreviews(limit > 0 ? limit : 5000, true);
         const batchSize = Math.max(1, options.previewBatchSize ?? 25);
         const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
         let batch = 0;

media-scrape/test/run.ts

@@ -877,6 +877,67 @@ void (async () => {
         assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
     });
 
+    await test("previews run in cycles during the walk", async () => {
+        const events: string[] = [];
+        const pages: Array<{ messages: CoreMessage[] }> = [
+            {
+                messages: [
+                    message("1000", {
+                        attachments: [attachment({ filename: "a.png", contentType: "image/png" })]
+                    })
+                ]
+            },
+            {
+                messages: [
+                    message("999", {
+                        attachments: [attachment({ filename: "b.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ];
+
+        const backend: ScrapeBackend = {
+            name: "puppet",
+            async fetchPage() {
+                events.push("walk");
+                return pages.shift() ?? { messages: [] };
+            }
+        };
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 1,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            walkCyclePages: 1,
+            download: {
+                fetchImpl: async () => {
+                    events.push("preview");
+                    return new Response("webp", { status: 200, headers: { "content-type": "image/webp" } });
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 100,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "150", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        assert.equal(engine.get(started.id)?.previewed, 2);
+        const firstPreview = events.indexOf("preview");
+        const secondWalk = events.indexOf("walk", 1);
+        assert.ok(firstPreview !== -1, "a preview happened during the walk");
+        assert.ok(secondWalk > firstPreview, "the walk continued after previews");
+    });
+
     await test("engine resumes a paused scrape from its cursor", async () => {
         const first = new FakeBackend([
             { messages: [message("500"), message("499")] },