Fetch previews in batches with a rate-limit cooldown

media-scrape/src/download.ts

@@ -65,6 +65,10 @@ export interface DownloadOptions {
     /** Preview gap. Default 200-600 ms, much faster than file downloads. */
     previewMinGapMs?: number;
     previewMaxGapMs?: number;
+    /** Previews per batch before a cooldown pause. Default 25. */
+    previewBatchSize?: number;
+    /** Cooldown between preview batches. Default 60 s. Set 0 to disable batching. */
+    previewBatchPauseMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */

media-scrape/src/engine.ts

@@ -381,6 +381,19 @@ export class ScrapeEngine {
             destinationDir: meta.downloadDir ?? null
         };
         const pending = store.pendingPreviews(5000, true);
+        const batchSize = Math.max(1, options.previewBatchSize ?? 25);
+        const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
+        let batch = 0;
+
+        // Previews are fetched in batches: Discord and the provider APIs throttle bursts.
+        const afterRow = async () => {
+            batch++;
+            if (batchPauseMs > 0 && batch >= batchSize) {
+                batch = 0;
+                this.log(`${meta.id}: preview batch of ${batchSize} done, pausing ${batchPauseMs} ms`);
+                await this.sleep(batchPauseMs);
+            }
+        };
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {
@@ -403,6 +416,7 @@ export class ScrapeEngine {
                 }
                 this.persist(dir, meta);
                 await this.sleep(previewGapMs(options));
+                await afterRow();
                 continue;
             }
 
@@ -415,6 +429,7 @@ export class ScrapeEngine {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
                     await this.sleep(previewGapMs(options));
+                    await afterRow();
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -470,6 +485,7 @@ export class ScrapeEngine {
 
             this.persist(dir, meta);
             await this.sleep(previewGapMs(options));
+            await afterRow();
         }
     }
 

media-scrape/test/run.ts

@@ -961,6 +961,53 @@ void (async () => {
         );
     });
 
+    await test("previews pause between batches", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("900", {
+                        attachments: [
+                            attachment({ filename: "a.png", contentType: "image/png" }),
+                            attachment({ filename: "b.png", contentType: "image/png" }),
+                            attachment({ filename: "c.png", contentType: "image/png" })
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const sleeps: number[] = [];
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async (ms: number) => {
+                sleeps.push(ms);
+            },
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                previewMinGapMs: 0,
+                previewMaxGapMs: 0,
+                previewBatchSize: 2,
+                previewBatchPauseMs: 5000,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        assert.equal(engine.get(started.id)?.previewed, 3);
+        assert.equal(sleeps.filter(ms => ms === 5000).length, 1);
+    });
+
     await test("remoteFrame renders an audio waveform", async () => {
         const dir = join(root, "waveform-test");
         mkdirSync(dir, { recursive: true });