Cycle previews through the walk

media-scrape/src/engine.ts

@@ -60,8 +60,10 @@ export interface ScrapeEngineOptions {
     log?: (line: string) => void;
     /** Download settings for `mode: "download"`. */
     download?: DownloadOptions;
-    /** Cache Discord-hosted image previews after every walk. Default true. */
+    /** Cache previews after every walk. Default true. */
     previews?: boolean;
+    /** Walk pages between preview cycles. 0 defers all previews to the end. Default 5. */
+    walkCyclePages?: number;
     /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
     probe?: (
         path: string
@@ -238,6 +240,8 @@ export class ScrapeEngine {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);
         const pageSize = this.options.pageSize ?? 100;
+        const perCycle = this.options.download?.previewBatchSize ?? 25;
+        let pagesSincePreview = 0;
 
         try {
             if (this.stopRequested.has(meta.id)) {
@@ -286,6 +290,17 @@ export class ScrapeEngine {
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
+                // Cycle between walking and previewing so tiles fill in during a long walk
+                // instead of only at the end.
+                const cycle = this.options.walkCyclePages ?? 5;
+                if (cycle > 0 && this.options.previews !== false) {
+                    pagesSincePreview++;
+                    if (pagesSincePreview >= cycle) {
+                        pagesSincePreview = 0;
+                        await this.runPreviews(store, dir, meta, perCycle);
+                    }
+                }
+
                 if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
@@ -390,13 +405,13 @@ export class ScrapeEngine {
     }
 
     /** Cache small previews. Embed thumbnails are cached in both modes. */
-    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta, limit = 0): Promise<void> {
         const options = {
             ...(this.options.download ?? {}),
             embedPreviews: true,
             destinationDir: meta.downloadDir ?? null
         };
-        const pending = store.pendingPreviews(5000, true);
+        const pending = store.pendingPreviews(limit > 0 ? limit : 5000, true);
         const batchSize = Math.max(1, options.previewBatchSize ?? 25);
         const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
         let batch = 0;