From 6c500f0cc5e5822046809fcf500f4f878b3d39d6 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 23:51:16 +0200 Subject: [PATCH] Fetch previews in batches with a rate-limit cooldown --- CURRENT.md | 4 +++ docs/daemon.md | 2 ++ docs/media-scraper.md | 4 ++- media-scrape/src/download.ts | 4 +++ media-scrape/src/engine.ts | 16 ++++++++++++ media-scrape/test/run.ts | 47 ++++++++++++++++++++++++++++++++++++ poc/src/media.ts | 2 ++ 7 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CURRENT.md b/CURRENT.md index a0caab8..d523d01 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,10 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C31 Batched previews. The preview pass now runs in batches (default 25) with a cooldown + (default 60 s), configurable via `VESKSLOP_MEDIA_PREVIEW_BATCH` and + `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`; batching counts rows, not successes. Live log: + `preview batch of 25 done, pausing 60000 ms`. - [x] C30 Preview completeness. Audio attachments and voice messages get an ffmpeg waveform strip; oversized animated GIFs fall back to an ffmpeg-read first frame instead of `too large`; transient network failures stay pending for the next backfill instead of becoming permanent. diff --git a/docs/daemon.md b/docs/daemon.md index f69c607..0c52081 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -177,6 +177,8 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. | | `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. | | `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. | +| `VESKSLOP_MEDIA_PREVIEW_BATCH` | `25` | Previews per batch before the cooldown. | +| `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS` | `60000` | Cooldown between preview batches. `0` disables batching. | | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 2b88633..70d1e5b 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -145,7 +145,9 @@ Previews run after every walk: - At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their previews were cached, so a stopped scrape does not leave blank tiles. - Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than - file downloads. + file downloads. Previews are fetched in **batches of 25 with a 60 s cooldown** between batches + (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of + thumbnail fetches trigger Discord and provider rate limits. A batch counts rows, not successes. ## Job states diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index 448f48e..e151134 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -65,6 +65,10 @@ export interface DownloadOptions { /** Preview gap. Default 200-600 ms, much faster than file downloads. */ previewMinGapMs?: number; previewMaxGapMs?: number; + /** Previews per batch before a cooldown pause. Default 25. */ + previewBatchSize?: number; + /** Cooldown between preview batches. Default 60 s. Set 0 to disable batching. */ + previewBatchPauseMs?: number; /** Cache third-party embed thumbnails. */ embedPreviews?: boolean; /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */ diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 723a48b..2acc04c 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -381,6 +381,19 @@ export class ScrapeEngine { destinationDir: meta.downloadDir ?? null }; const pending = store.pendingPreviews(5000, true); + const batchSize = Math.max(1, options.previewBatchSize ?? 25); + const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000); + let batch = 0; + + // Previews are fetched in batches: Discord and the provider APIs throttle bursts. + const afterRow = async () => { + batch++; + if (batchPauseMs > 0 && batch >= batchSize) { + batch = 0; + this.log(`${meta.id}: preview batch of ${batchSize} done, pausing ${batchPauseMs} ms`); + await this.sleep(batchPauseMs); + } + }; for (const row of pending) { if (this.stopRequested.has(meta.id)) { @@ -403,6 +416,7 @@ export class ScrapeEngine { } this.persist(dir, meta); await this.sleep(previewGapMs(options)); + await afterRow(); continue; } @@ -415,6 +429,7 @@ export class ScrapeEngine { store.markPreviewError(row.id, "no preview source"); this.persist(dir, meta); await this.sleep(previewGapMs(options)); + await afterRow(); continue; } previewRow = { ...row, preview_url: resolved }; @@ -470,6 +485,7 @@ export class ScrapeEngine { this.persist(dir, meta); await this.sleep(previewGapMs(options)); + await afterRow(); } } diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 2a3d99a..c78bee1 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -961,6 +961,53 @@ void (async () => { ); }); + await test("previews pause between batches", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("900", { + attachments: [ + attachment({ filename: "a.png", contentType: "image/png" }), + attachment({ filename: "b.png", contentType: "image/png" }), + attachment({ filename: "c.png", contentType: "image/png" }) + ] + }) + ] + } + ]); + + const sleeps: number[] = []; + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 10, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async (ms: number) => { + sleeps.push(ms); + }, + random: () => 0, + log: () => {}, + previews: true, + download: { + fetchImpl: async () => + new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }), + minGapMs: 0, + maxGapMs: 0, + previewMinGapMs: 0, + previewMaxGapMs: 0, + previewBatchSize: 2, + previewBatchPauseMs: 5000, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "133", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + assert.equal(engine.get(started.id)?.previewed, 3); + assert.equal(sleeps.filter(ms => ms === 5000).length, 1); + }); + await test("remoteFrame renders an audio waveform", async () => { const dir = join(root, "waveform-test"); mkdirSync(dir, { recursive: true }); diff --git a/poc/src/media.ts b/poc/src/media.ts index bbaa3fc..2f426e4 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -179,6 +179,8 @@ export class MediaService { previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024), previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200), previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600), + previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25), + previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000), minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000), maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000) }