Backfill previews for paused scrapes and prefer richer dedupe rows

media-scrape/src/engine.ts

@@ -10,6 +10,7 @@ import {
     downloadOne,
     downloadPreview,
     gifPoster,
+    previewGapMs,
     probeDimensions,
     videoPoster,
     type DownloadOptions
@@ -180,6 +181,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */
+    async backfillPreviews(scrapeId: string): Promise<number> {
+        const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId));
+        if (!meta || meta.state === "running" || meta.state === "queued") return 0;
+
+        const dir = join(this.options.rootDir, scrapeId);
+        const store = new ScrapeStore(dir);
+        const before = meta.previewed ?? 0;
+
+        try {
+            await this.runPreviews(store, dir, meta);
+        } finally {
+            store.close();
+        }
+
+        return (meta.previewed ?? 0) - before;
+    }
+
     /** Persist a state change made outside a job, such as retention marking. */
     setState(id: string, state: ScrapeState): ScrapeMeta | null {
         const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
@@ -258,9 +277,15 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+            const wasPaused = meta.state === "paused";
+
+            if (
+                (meta.state === "done" || wasPaused) &&
+                (meta.mode === "download" || this.options.previews !== false)
+            ) {
                 // Keep the job running until post-processing finishes, so a client never
-                // treats a half-archived scrape as complete.
+                // treats a half-archived scrape as complete. A paused walk still caches
+                // previews and downloads for the pages it did collect.
                 meta.state = "running";
                 this.persist(dir, meta);
 
@@ -273,7 +298,7 @@ export class ScrapeEngine {
                 }
 
                 if (meta.state === "running") {
-                    meta.state = "done";
+                    meta.state = wasPaused ? "paused" : "done";
                 }
                 this.persist(dir, meta);
             }
@@ -370,7 +395,7 @@ export class ScrapeEngine {
                 if (!resolved) {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
-                    await this.sleep(downloadGapMs(options));
+                    await this.sleep(previewGapMs(options));
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -409,7 +434,7 @@ export class ScrapeEngine {
             }
 
             this.persist(dir, meta);
-            await this.sleep(downloadGapMs(options));
+            await this.sleep(previewGapMs(options));
         }
     }