Backfill previews for paused scrapes and prefer richer dedupe rows

media-scrape/src/dedupe.ts

@@ -0,0 +1,19 @@
+/*
+ * Cross-scrape dedupe preference: among rows for the same URL, the richest wins.
+ * Newer scrapes are seen first; a candidate replaces the current one only when it
+ * carries something the current one lacks.
+ */
+
+export interface DedupeCandidate {
+    downloaded: boolean;
+    hasPreview: boolean;
+}
+
+export function candidateScore(candidate: DedupeCandidate): number {
+    return (candidate.downloaded ? 2 : 0) + (candidate.hasPreview ? 1 : 0);
+}
+
+/** True when `candidate` should replace `current` for the same URL. */
+export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
+    return candidateScore(candidate) > candidateScore(current);
+}

media-scrape/src/download.ts

@@ -62,6 +62,9 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Preview gap. Default 200-600 ms, much faster than file downloads. */
+    previewMinGapMs?: number;
+    previewMaxGapMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
@@ -351,3 +354,11 @@ export function downloadGapMs(options: DownloadOptions = {}): number {
     const random = options.random ?? Math.random;
     return min + Math.floor(random() * (max - min + 1));
 }
+
+/** The gap before the next preview, in milliseconds. Previews are small and quick. */
+export function previewGapMs(options: DownloadOptions = {}): number {
+    const min = options.previewMinGapMs ?? 200;
+    const max = Math.max(min, options.previewMaxGapMs ?? 600);
+    const random = options.random ?? Math.random;
+    return min + Math.floor(random() * (max - min + 1));
+}

media-scrape/src/engine.ts

@@ -10,6 +10,7 @@ import {
     downloadOne,
     downloadPreview,
     gifPoster,
+    previewGapMs,
     probeDimensions,
     videoPoster,
     type DownloadOptions
@@ -180,6 +181,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */
+    async backfillPreviews(scrapeId: string): Promise<number> {
+        const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId));
+        if (!meta || meta.state === "running" || meta.state === "queued") return 0;
+
+        const dir = join(this.options.rootDir, scrapeId);
+        const store = new ScrapeStore(dir);
+        const before = meta.previewed ?? 0;
+
+        try {
+            await this.runPreviews(store, dir, meta);
+        } finally {
+            store.close();
+        }
+
+        return (meta.previewed ?? 0) - before;
+    }
+
     /** Persist a state change made outside a job, such as retention marking. */
     setState(id: string, state: ScrapeState): ScrapeMeta | null {
         const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
@@ -258,9 +277,15 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+            const wasPaused = meta.state === "paused";
+
+            if (
+                (meta.state === "done" || wasPaused) &&
+                (meta.mode === "download" || this.options.previews !== false)
+            ) {
                 // Keep the job running until post-processing finishes, so a client never
-                // treats a half-archived scrape as complete.
+                // treats a half-archived scrape as complete. A paused walk still caches
+                // previews and downloads for the pages it did collect.
                 meta.state = "running";
                 this.persist(dir, meta);
 
@@ -273,7 +298,7 @@ export class ScrapeEngine {
                 }
 
                 if (meta.state === "running") {
-                    meta.state = "done";
+                    meta.state = wasPaused ? "paused" : "done";
                 }
                 this.persist(dir, meta);
             }
@@ -370,7 +395,7 @@ export class ScrapeEngine {
                 if (!resolved) {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
-                    await this.sleep(downloadGapMs(options));
+                    await this.sleep(previewGapMs(options));
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -409,7 +434,7 @@ export class ScrapeEngine {
             }
 
             this.persist(dir, meta);
-            await this.sleep(downloadGapMs(options));
+            await this.sleep(previewGapMs(options));
         }
     }
 

media-scrape/src/index.ts

@@ -4,6 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
 export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";