Fetch previews in batches with a rate-limit cooldown

media-scrape/src/download.ts

@@ -65,6 +65,10 @@ export interface DownloadOptions {
     /** Preview gap. Default 200-600 ms, much faster than file downloads. */
     previewMinGapMs?: number;
     previewMaxGapMs?: number;
+    /** Previews per batch before a cooldown pause. Default 25. */
+    previewBatchSize?: number;
+    /** Cooldown between preview batches. Default 60 s. Set 0 to disable batching. */
+    previewBatchPauseMs?: number;
     /** Cache third-party embed thumbnails. */
     embedPreviews?: boolean;
     /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */

media-scrape/src/engine.ts

@@ -381,6 +381,19 @@ export class ScrapeEngine {
             destinationDir: meta.downloadDir ?? null
         };
         const pending = store.pendingPreviews(5000, true);
+        const batchSize = Math.max(1, options.previewBatchSize ?? 25);
+        const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000);
+        let batch = 0;
+
+        // Previews are fetched in batches: Discord and the provider APIs throttle bursts.
+        const afterRow = async () => {
+            batch++;
+            if (batchPauseMs > 0 && batch >= batchSize) {
+                batch = 0;
+                this.log(`${meta.id}: preview batch of ${batchSize} done, pausing ${batchPauseMs} ms`);
+                await this.sleep(batchPauseMs);
+            }
+        };
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {
@@ -403,6 +416,7 @@ export class ScrapeEngine {
                 }
                 this.persist(dir, meta);
                 await this.sleep(previewGapMs(options));
+                await afterRow();
                 continue;
             }
 
@@ -415,6 +429,7 @@ export class ScrapeEngine {
                     store.markPreviewError(row.id, "no preview source");
                     this.persist(dir, meta);
                     await this.sleep(previewGapMs(options));
+                    await afterRow();
                     continue;
                 }
                 previewRow = { ...row, preview_url: resolved };
@@ -470,6 +485,7 @@ export class ScrapeEngine {
 
             this.persist(dir, meta);
             await this.sleep(previewGapMs(options));
+            await afterRow();
         }
     }