Backfill previews for paused scrapes and prefer richer dedupe rows

media-scrape/src/dedupe.ts

@@ -0,0 +1,19 @@
+/*
+ * Cross-scrape dedupe preference: among rows for the same URL, the richest wins.
+ * Newer scrapes are seen first; a candidate replaces the current one only when it
+ * carries something the current one lacks.
+ */
+
+export interface DedupeCandidate {
+    downloaded: boolean;
+    hasPreview: boolean;
+}
+
+export function candidateScore(candidate: DedupeCandidate): number {
+    return (candidate.downloaded ? 2 : 0) + (candidate.hasPreview ? 1 : 0);
+}
+
+/** True when `candidate` should replace `current` for the same URL. */
+export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
+    return candidateScore(candidate) > candidateScore(current);
+}

media-scrape/src/download.ts

@@ -62,6 +62,9 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Preview gap. Default 200-600 ms, much faster than file downloads. */
+    previewMinGapMs?: number;
+    previewMaxGapMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
@@ -351,3 +354,11 @@ export function downloadGapMs(options: DownloadOptions = {}): number {
     const random = options.random ?? Math.random;
     return min + Math.floor(random() * (max - min + 1));
 }
+
+/** The gap before the next preview, in milliseconds. Previews are small and quick. */
+export function previewGapMs(options: DownloadOptions = {}): number {
+    const min = options.previewMinGapMs ?? 200;
+    const max = Math.max(min, options.previewMaxGapMs ?? 600);
+    const random = options.random ?? Math.random;
+    return min + Math.floor(random() * (max - min + 1));
+}

media-scrape/src/engine.ts

@@ -10,6 +10,7 @@ import {
     downloadOne,
     downloadPreview,
     gifPoster,
+    previewGapMs,
     probeDimensions,
     videoPoster,
     type DownloadOptions
@@ -180,6 +181,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */
+    async backfillPreviews(scrapeId: string): Promise<number> {
+        const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId));
+        if (!meta || meta.state === "running" || meta.state === "queued") return 0;
+
+        const dir = join(this.options.rootDir, scrapeId);
+        const store = new ScrapeStore(dir);
+        const before = meta.previewed ?? 0;
+
+        try {
+            await this.runPreviews(store, dir, meta);
+        } finally {
+            store.close();
+        }
+
+        return (meta.previewed ?? 0) - before;
+    }
+
     /** Persist a state change made outside a job, such as retention marking. */
     setState(id: string, state: ScrapeState): ScrapeMeta | null {
         const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
@@ -258,9 +277,15 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+            const wasPaused = meta.state === "paused";
+
+            if (
+                (meta.state === "done" || wasPaused) &&
+                (meta.mode === "download" || this.options.previews !== false)
+            ) {
                 // Keep the job running until post-processing finishes, so a client never
-                // treats a half-archived scrape as complete.
+                // treats a half-archived scrape as complete. A paused walk still caches
+                // previews and downloads for the pages it did collect.
                 meta.state = "running";
                 this.persist(dir, meta);
 
@@ -273,7 +298,7 @@ export class ScrapeEngine {
                 }
 
                 if (meta.state === "running") {
-                    meta.state = "done";
+                    meta.state = wasPaused ? "paused" : "done";
                 }
                 this.persist(dir, meta);
             }
@@ -370,7 +395,7 @@ export class ScrapeEngine {
                 if (!resolved) {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
-                    await this.sleep(downloadGapMs(options));
+                    await this.sleep(previewGapMs(options));
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -409,7 +434,7 @@ export class ScrapeEngine {
             }
 
             this.persist(dir, meta);
-            await this.sleep(downloadGapMs(options));
+            await this.sleep(previewGapMs(options));
         }
     }
 

media-scrape/src/index.ts

@@ -4,6 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
 export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";

media-scrape/test/run.ts

@@ -14,6 +14,7 @@ import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { betterCandidate } from "../src/dedupe.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -861,6 +862,90 @@ void (async () => {
         assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
     });
 
+    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("800", {
+                        attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://www.youtube.com/watch?v=x",
+                                title: null,
+                                provider: "YouTube",
+                                videoUrl: "https://www.youtube.com/embed/x",
+                                imageUrl: null,
+                                thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        assert.equal(engine.get(started.id)?.previewed, undefined);
+
+        const added = await engine.backfillPreviews(started.id);
+        assert.equal(added, 2);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
+        assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
+        store.close();
+    });
+
+    await test("dedupe prefers the richer row for a URL", () => {
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            true
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: false, hasPreview: false }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: true, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: true, hasPreview: false }
+            ),
+            true
+        );
+    });
+
     await test("snowflake helpers round-trip dates", () => {
         const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
         assert.ok(id);