Backfill previews for paused scrapes and prefer richer dedupe rows

docs/daemon.md

@@ -176,6 +176,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
 | `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. |
 | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
 | `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
+| `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. |
 | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
 | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
 | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |

docs/media-scraper.md

@@ -133,7 +133,11 @@ Previews run after every walk:
   (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the
   counter is `previewed`. Previews count toward the storage limit.
 - The job stays `running` through the download and preview passes, so a client never sees `done`
-  on a half-archived scrape.
+  on a half-archived scrape. A paused walk still gets these passes for the pages it collected.
+- At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their
+  previews were cached, so a stopped scrape does not leave blank tiles.
+- Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than
+  file downloads.
 
 ## Job states
 

media-scrape/src/dedupe.ts

@@ -0,0 +1,19 @@
+/*
+ * Cross-scrape dedupe preference: among rows for the same URL, the richest wins.
+ * Newer scrapes are seen first; a candidate replaces the current one only when it
+ * carries something the current one lacks.
+ */
+
+export interface DedupeCandidate {
+    downloaded: boolean;
+    hasPreview: boolean;
+}
+
+export function candidateScore(candidate: DedupeCandidate): number {
+    return (candidate.downloaded ? 2 : 0) + (candidate.hasPreview ? 1 : 0);
+}
+
+/** True when `candidate` should replace `current` for the same URL. */
+export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
+    return candidateScore(candidate) > candidateScore(current);
+}

media-scrape/src/download.ts

@@ -62,6 +62,9 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Preview gap. Default 200-600 ms, much faster than file downloads. */
+    previewMinGapMs?: number;
+    previewMaxGapMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
@@ -351,3 +354,11 @@ export function downloadGapMs(options: DownloadOptions = {}): number {
     const random = options.random ?? Math.random;
     return min + Math.floor(random() * (max - min + 1));
 }
+
+/** The gap before the next preview, in milliseconds. Previews are small and quick. */
+export function previewGapMs(options: DownloadOptions = {}): number {
+    const min = options.previewMinGapMs ?? 200;
+    const max = Math.max(min, options.previewMaxGapMs ?? 600);
+    const random = options.random ?? Math.random;
+    return min + Math.floor(random() * (max - min + 1));
+}

media-scrape/src/engine.ts

@@ -10,6 +10,7 @@ import {
     downloadOne,
     downloadPreview,
     gifPoster,
+    previewGapMs,
     probeDimensions,
     videoPoster,
     type DownloadOptions
@@ -180,6 +181,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */
+    async backfillPreviews(scrapeId: string): Promise<number> {
+        const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId));
+        if (!meta || meta.state === "running" || meta.state === "queued") return 0;
+
+        const dir = join(this.options.rootDir, scrapeId);
+        const store = new ScrapeStore(dir);
+        const before = meta.previewed ?? 0;
+
+        try {
+            await this.runPreviews(store, dir, meta);
+        } finally {
+            store.close();
+        }
+
+        return (meta.previewed ?? 0) - before;
+    }
+
     /** Persist a state change made outside a job, such as retention marking. */
     setState(id: string, state: ScrapeState): ScrapeMeta | null {
         const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
@@ -258,9 +277,15 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+            const wasPaused = meta.state === "paused";
+
+            if (
+                (meta.state === "done" || wasPaused) &&
+                (meta.mode === "download" || this.options.previews !== false)
+            ) {
                 // Keep the job running until post-processing finishes, so a client never
-                // treats a half-archived scrape as complete.
+                // treats a half-archived scrape as complete. A paused walk still caches
+                // previews and downloads for the pages it did collect.
                 meta.state = "running";
                 this.persist(dir, meta);
 
@@ -273,7 +298,7 @@ export class ScrapeEngine {
                 }
 
                 if (meta.state === "running") {
-                    meta.state = "done";
+                    meta.state = wasPaused ? "paused" : "done";
                 }
                 this.persist(dir, meta);
             }
@@ -370,7 +395,7 @@ export class ScrapeEngine {
                 if (!resolved) {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
-                    await this.sleep(downloadGapMs(options));
+                    await this.sleep(previewGapMs(options));
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -409,7 +434,7 @@ export class ScrapeEngine {
             }
 
             this.persist(dir, meta);
-            await this.sleep(downloadGapMs(options));
+            await this.sleep(previewGapMs(options));
         }
     }
 

media-scrape/src/index.ts

@@ -4,6 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
 export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";

media-scrape/test/run.ts

@@ -14,6 +14,7 @@ import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { betterCandidate } from "../src/dedupe.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -861,6 +862,90 @@ void (async () => {
         assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
     });
 
+    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("800", {
+                        attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://www.youtube.com/watch?v=x",
+                                title: null,
+                                provider: "YouTube",
+                                videoUrl: "https://www.youtube.com/embed/x",
+                                imageUrl: null,
+                                thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        assert.equal(engine.get(started.id)?.previewed, undefined);
+
+        const added = await engine.backfillPreviews(started.id);
+        assert.equal(added, 2);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
+        assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
+        store.close();
+    });
+
+    await test("dedupe prefers the richer row for a URL", () => {
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            true
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: false, hasPreview: false }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: true, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: true, hasPreview: false }
+            ),
+            true
+        );
+    });
+
     await test("snowflake helpers round-trip dates", () => {
         const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
         assert.ok(id);

poc/src/media.ts

@@ -8,6 +8,7 @@
 import { rmSync } from "node:fs";
 import { isAbsolute, join } from "node:path";
 import {
+    betterCandidate,
     dirSizeBytes,
     isNearLimit,
     planRetention,
@@ -174,10 +175,28 @@ export class MediaService {
             download: {
                 maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
                 previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
+                previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
+                previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
                 minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
                 maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
             }
         });
+
+        // Preview passes only run when a walk ends; fill in scrapes that paused early.
+        void this.backfillPreviews();
+    }
+
+    private async backfillPreviews(): Promise<void> {
+        for (const meta of this.engine.list()) {
+            if (meta.state === "running" || meta.state === "queued") continue;
+
+            try {
+                const added = await this.engine.backfillPreviews(meta.id);
+                if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
+            } catch (error) {
+                this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
+            }
+        }
     }
 
     /** Log slopbot in on first use. False when it is disabled or the login failed. */
@@ -546,37 +565,8 @@ export class MediaService {
         after?: string;
         before?: string;
     } = {}): Record<MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number }> {
-        // Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
+        // Same richest-per-URL dedupe as queryMedia.
-        const byUrl = new Map<string, { kind: MediaKind; downloaded: boolean; cached: boolean }>();
+        const candidates = this.collectCandidates(options);
-
-        for (const meta of this.engine.list()) {
-            if (meta.state === "deleted") continue;
-            if (options.channelId && meta.channelId !== options.channelId) continue;
-
-            const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
-            try {
-                let storeOffset = 0;
-
-                for (;;) {
-                    const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
-                    if (!rows.length) break;
-                    storeOffset += rows.length;
-
-                    for (const row of rows) {
-                        if (byUrl.has(row.url)) continue;
-                        byUrl.set(row.url, {
-                            kind: row.kind,
-                            downloaded: row.downloaded === 1,
-                            cached: row.preview_hash !== null
-                        });
-                    }
-
-                    if (rows.length < 500) break;
-                }
-            } finally {
-                store.close();
-            }
-        }
 
         const totals: Record<
             MediaKind,
@@ -588,14 +578,14 @@ export class MediaService {
             audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }
         };
 
-        for (const entry of byUrl.values()) {
+        for (const { row } of candidates) {
-            const bucket = totals[entry.kind];
+            const bucket = totals[row.kind];
             bucket.total++;
-            if (entry.downloaded) {
+            if (row.downloaded === 1) {
                 bucket.downloaded++;
             } else {
                 bucket.notDownloaded++;
-                if (entry.cached) bucket.cached++;
+                if (row.preview_hash !== null) bucket.cached++;
             }
         }
 
@@ -639,27 +629,23 @@ export class MediaService {
         }
     }
 
-    /** Read media rows across every scrape, newest scrape first. */
+    /**
-    queryMedia(
+     * One entry per URL across every scrape. Scrapes are visited newest first, so a newer row
-        options: {
+     * wins by default; an older row replaces it only when it is richer (downloaded, or with a
-            kind?: MediaKind;
+     * cached preview). Without this, a stopped or partial scrape hides usable previews.
-            source?: string;
+     */
-            scrapeId?: string;
+    private collectCandidates(options: {
-            channelId?: string;
+        kind?: MediaKind;
-            authorId?: string;
+        source?: string;
-            aspect?: string;
+        scrapeId?: string;
-            resolution?: string;
+        channelId?: string;
-            after?: string;
+        authorId?: string;
-            before?: string;
+        aspect?: string;
-            limit?: number;
+        resolution?: string;
-            offset?: number;
+        after?: string;
-        } = {}
+        before?: string;
-    ): MediaItemResult[] {
+    }): Array<{ meta: ScrapeMeta; row: MediaRow }> {
-        const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
+        const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
-        const offset = Math.max(0, options.offset ?? 0);
-        const results: MediaItemResult[] = [];
-        const seen = new Set<string>();
-        let skipped = 0;
 
         for (const meta of this.engine.list()) {
             if (meta.state === "deleted") continue;
@@ -686,53 +672,81 @@ export class MediaService {
                     storeOffset += rows.length;
 
                     for (const row of rows) {
-                        // One card per URL, newest scrape wins.
+                        const current = candidates.get(row.url);
-                        if (seen.has(row.url)) continue;
+                        if (!current) {
-                        seen.add(row.url);
+                            candidates.set(row.url, { meta, row });
-
-                        if (skipped < offset) {
-                            skipped++;
                             continue;
                         }
-                        if (results.length >= limit) break;
 
-                        results.push({
+                        if (
-                            scrapeId: meta.id,
+                            betterCandidate(
-                            itemId: row.id,
+                                {
-                            channelId: meta.channelId,
+                                    downloaded: current.row.downloaded === 1,
-                            kind: row.kind,
+                                    hasPreview: current.row.preview_hash !== null
-                            source: row.source,
+                                },
-                            authorId: row.author_id,
+                                { downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null }
-                            authorName: row.author_name,
+                            )
-                            postedAt: row.posted_at,
+                        ) {
-                            width: row.width,
+                            candidates.set(row.url, { meta, row });
-                            height: row.height,
+                        }
-                            aspectRatio: row.aspect_ratio,
-                            contentType: row.content_type,
-                            durationSecs: row.duration_secs,
-                            url: row.url,
-                            playUrl: row.play_url,
-                            remotePreviewUrl: row.preview_url,
-                            localUrl:
-                                row.file_path && !meta.downloadDir
-                                    ? `/api/media/${meta.id}/${row.id}/file`
-                                    : null,
-                            thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
-                            provider: row.provider,
-                            downloaded: row.downloaded === 1,
-                            createdAt: row.created_at
-                        });
                     }
 
-                    if (results.length >= limit || rows.length < 500) break;
+                    if (rows.length < 500) break;
                 }
             } finally {
                 store.close();
             }
-
-            if (results.length >= limit) break;
         }
 
-        return results;
+        return [...candidates.values()];
+    }
+
+    /** Read media rows across every scrape, richest per URL, newest scrape first. */
+    queryMedia(
+        options: {
+            kind?: MediaKind;
+            source?: string;
+            scrapeId?: string;
+            channelId?: string;
+            authorId?: string;
+            aspect?: string;
+            resolution?: string;
+            after?: string;
+            before?: string;
+            limit?: number;
+            offset?: number;
+        } = {}
+    ): MediaItemResult[] {
+        const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
+        const offset = Math.max(0, options.offset ?? 0);
+
+        return this.collectCandidates(options)
+            .slice(offset, offset + limit)
+            .map(({ meta, row }) => ({
+                scrapeId: meta.id,
+                itemId: row.id,
+                channelId: meta.channelId,
+                kind: row.kind,
+                source: row.source,
+                authorId: row.author_id,
+                authorName: row.author_name,
+                postedAt: row.posted_at,
+                width: row.width,
+                height: row.height,
+                aspectRatio: row.aspect_ratio,
+                contentType: row.content_type,
+                durationSecs: row.duration_secs,
+                url: row.url,
+                playUrl: row.play_url,
+                remotePreviewUrl: row.preview_url,
+                localUrl:
+                    row.file_path && !meta.downloadDir
+                        ? `/api/media/${meta.id}/${row.id}/file`
+                        : null,
+                thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
+                provider: row.provider,
+                downloaded: row.downloaded === 1,
+                createdAt: row.created_at
+            }));
     }
 }