Cycle previews through the walk

CURRENT.md

@@ -40,6 +40,11 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C35 Walk/preview cycling. A live scrape now alternates: walk 5 pages, preview up to one
+    batch (25, with the cooldown), walk on. Tiles fill in during a long walk instead of only at the
+    end, and preview traffic is spread through the walk. Configurable with
+    `VESKSLOP_MEDIA_WALK_CYCLE_PAGES` (0 defers previews to the end). The final pass still drains
+    whatever remains.
   - [x] C34 Resume from cursor. A paused scrape can continue where it stopped instead of
     re-walking the top: `POST /api/media/scrape/resume`, MCP `resume_media_scrape`, and a Resume
     button in History. Previews never needed the walk — backfill fetches them straight from the

docs/daemon.md

@@ -180,6 +180,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
 | `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. |
 | `VESKSLOP_MEDIA_PREVIEW_BATCH` | `25` | Previews per batch before the cooldown. |
 | `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS` | `60000` | Cooldown between preview batches. `0` disables batching. |
+| `VESKSLOP_MEDIA_WALK_CYCLE_PAGES` | `5` | Walk pages between preview cycles on a live scrape. `0` defers previews to the end. |
 | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
 | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
 | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |

media-scrape/src/engine.ts

@@ -60,8 +60,10 @@ export interface ScrapeEngineOptions {
     log?: (line: string) => void;
     /** Download settings for `mode: "download"`. */
     download?: DownloadOptions;
-    /** Cache Discord-hosted image previews after every walk. Default true. */
+    /** Cache previews after every walk. Default true. */
     previews?: boolean;
+    /** Walk pages between preview cycles. 0 defers all previews to the end. Default 5. */
+    walkCyclePages?: number;
     /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
     probe?: (
         path: string
@@ -238,6 +240,8 @@ export class ScrapeEngine {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);
         const pageSize = this.options.pageSize ?? 100;
+        const perCycle = this.options.download?.previewBatchSize ?? 25;
+        let pagesSincePreview = 0;
 
         try {
             if (this.stopRequested.has(meta.id)) {
@@ -286,6 +290,17 @@ export class ScrapeEngine {
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
+                // Cycle between walking and previewing so tiles fill in during a long walk
+                // instead of only at the end.
+                const cycle = this.options.walkCyclePages ?? 5;
+                if (cycle > 0 && this.options.previews !== false) {
+                    pagesSincePreview++;
+                    if (pagesSincePreview >= cycle) {
+                        pagesSincePreview = 0;
+                        await this.runPreviews(store, dir, meta, perCycle);
+                    }
+                }
+
                 if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
@@ -390,13 +405,13 @@ export class ScrapeEngine {
     }
 
     /** Cache small previews. Embed thumbnails are cached in both modes. */
-    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta, limit = 0): Promise<void> {
         const options = {
             ...(this.options.download ?? {}),
             embedPreviews: true,
             destinationDir: meta.downloadDir ?? null
         };
-        const pending = store.pendingPreviews(5000, true);
+        const pending = store.pendingPreviews(limit > 0 ? limit : 5000, true);
         const batchSize = Math.max(1, options.previewBatchSize ?? 25);
         const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
         let batch = 0;

media-scrape/test/run.ts

@@ -877,6 +877,67 @@ void (async () => {
         assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
     });
 
+    await test("previews run in cycles during the walk", async () => {
+        const events: string[] = [];
+        const pages: Array<{ messages: CoreMessage[] }> = [
+            {
+                messages: [
+                    message("1000", {
+                        attachments: [attachment({ filename: "a.png", contentType: "image/png" })]
+                    })
+                ]
+            },
+            {
+                messages: [
+                    message("999", {
+                        attachments: [attachment({ filename: "b.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ];
+
+        const backend: ScrapeBackend = {
+            name: "puppet",
+            async fetchPage() {
+                events.push("walk");
+                return pages.shift() ?? { messages: [] };
+            }
+        };
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 1,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            walkCyclePages: 1,
+            download: {
+                fetchImpl: async () => {
+                    events.push("preview");
+                    return new Response("webp", { status: 200, headers: { "content-type": "image/webp" } });
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 100,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "150", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        assert.equal(engine.get(started.id)?.previewed, 2);
+        const firstPreview = events.indexOf("preview");
+        const secondWalk = events.indexOf("walk", 1);
+        assert.ok(firstPreview !== -1, "a preview happened during the walk");
+        assert.ok(secondWalk > firstPreview, "the walk continued after previews");
+    });
+
     await test("engine resumes a paused scrape from its cursor", async () => {
         const first = new FakeBackend([
             { messages: [message("500"), message("499")] },

poc/src/media.ts

@@ -174,6 +174,7 @@ export class MediaService {
             rootDir: this.rootDir,
             log: line => this.deps.log?.(line),
             onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
+            walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
             download: {
                 maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
                 previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),