Fetch previews in batches with a rate-limit cooldown

CURRENT.md

@@ -40,6 +40,10 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C31 Batched previews. The preview pass now runs in batches (default 25) with a cooldown
+    (default 60 s), configurable via `VESKSLOP_MEDIA_PREVIEW_BATCH` and
+    `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`; batching counts rows, not successes. Live log:
+    `preview batch of 25 done, pausing 60000 ms`.
   - [x] C30 Preview completeness. Audio attachments and voice messages get an ffmpeg waveform
     strip; oversized animated GIFs fall back to an ffmpeg-read first frame instead of `too large`;
     transient network failures stay pending for the next backfill instead of becoming permanent.

docs/daemon.md

@@ -177,6 +177,8 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
 | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
 | `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
 | `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. |
+| `VESKSLOP_MEDIA_PREVIEW_BATCH` | `25` | Previews per batch before the cooldown. |
+| `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS` | `60000` | Cooldown between preview batches. `0` disables batching. |
 | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
 | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
 | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |

docs/media-scraper.md

@@ -145,7 +145,9 @@ Previews run after every walk:
 - At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their
   previews were cached, so a stopped scrape does not leave blank tiles.
 - Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than
-  file downloads.
+  file downloads. Previews are fetched in **batches of 25 with a 60 s cooldown** between batches
+  (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of
+  thumbnail fetches trigger Discord and provider rate limits. A batch counts rows, not successes.
 
 ## Job states
 

media-scrape/src/download.ts

@@ -65,6 +65,10 @@ export interface DownloadOptions {
     /** Preview gap. Default 200-600 ms, much faster than file downloads. */
     previewMinGapMs?: number;
     previewMaxGapMs?: number;
+    /** Previews per batch before a cooldown pause. Default 25. */
+    previewBatchSize?: number;
+    /** Cooldown between preview batches. Default 60 s. Set 0 to disable batching. */
+    previewBatchPauseMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */

media-scrape/src/engine.ts

@@ -381,6 +381,19 @@ export class ScrapeEngine {
             destinationDir: meta.downloadDir ?? null
         };
         const pending = store.pendingPreviews(5000, true);
+        const batchSize = Math.max(1, options.previewBatchSize ?? 25);
+        const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
+        let batch = 0;
+
+        // Previews are fetched in batches: Discord and the provider APIs throttle bursts.
+        const afterRow = async () => {
+            batch++;
+            if (batchPauseMs > 0 && batch >= batchSize) {
+                batch = 0;
+                this.log(`${meta.id}: preview batch of ${batchSize} done, pausing ${batchPauseMs} ms`);
+                await this.sleep(batchPauseMs);
+            }
+        };
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {
@@ -403,6 +416,7 @@ export class ScrapeEngine {
                 }
                 this.persist(dir, meta);
                 await this.sleep(previewGapMs(options));
+                await afterRow();
                 continue;
             }
 
@@ -415,6 +429,7 @@ export class ScrapeEngine {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
                     await this.sleep(previewGapMs(options));
+                    await afterRow();
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -470,6 +485,7 @@ export class ScrapeEngine {
 
             this.persist(dir, meta);
             await this.sleep(previewGapMs(options));
+            await afterRow();
         }
     }
 

media-scrape/test/run.ts

@@ -961,6 +961,53 @@ void (async () => {
         );
     });
 
+    await test("previews pause between batches", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("900", {
+                        attachments: [
+                            attachment({ filename: "a.png", contentType: "image/png" }),
+                            attachment({ filename: "b.png", contentType: "image/png" }),
+                            attachment({ filename: "c.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const sleeps: number[] = [];
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async (ms: number) => {
+                sleeps.push(ms);
+            },
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 2,
+                previewBatchPauseMs: 5000,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        assert.equal(engine.get(started.id)?.previewed, 3);
+        assert.equal(sleeps.filter(ms => ms === 5000).length, 1);
+    });
+
     await test("remoteFrame renders an audio waveform", async () => {
         const dir = join(root, "waveform-test");
         mkdirSync(dir, { recursive: true });

poc/src/media.ts

@@ -179,6 +179,8 @@ export class MediaService {
                 previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
                 previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
                 previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
+                previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25),
+                previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000),
                 minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
                 maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
             }