From e52623f1b2ac2c50c8c5a57822a84fbeee3872cf Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 08:44:55 +0200 Subject: [PATCH] Cap transient preview retries and close dead hosts --- CURRENT.md | 4 +++ docs/media-scraper.md | 6 ++++- media-scrape/src/engine.ts | 30 ++++++++++++++++----- media-scrape/src/store.ts | 15 +++++++++++ media-scrape/test/run.ts | 55 ++++++++++++++++++++++++++++++++++++++ poc/src/media.ts | 1 + 6 files changed, 104 insertions(+), 7 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 5d1e261..79eb4c6 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,10 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C42 Preview attempts cap. The restart log showed the backfill retrying dead hosts (Gfycat + DNS is gone, Streamable answers 400) forever as "transient". Rows now carry + `preview_attempts`; after `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` (3) the row is marked + `unreachable after N attempt(s)` and leaves the pending queue. Unit-tested. - [x] C41 Restart keeps its logs. A button restart spawned the replacement with `stdio: ignore`, so anything after the restart vanished. The replacement now appends to `daemon.log` beside the token file (`VESKSLOP_DAEMON_LOG` overrides); documented in [docs/daemon.md](docs/daemon.md). diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 4753fff..e7a332b 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -147,7 +147,8 @@ Previews run after every walk: (`ffmpeg -filter_complex showwavespic`) as their preview, rendered by ffmpeg from the URL. - Transient network failures are left pending for the next backfill pass; only skips (4xx, too large, no source) are marked permanent. -- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape. +- One attempt per row, no retries; a permanent failure (`http 404`, `http 403`, `too large`) marks + `preview_error` and never fails the scrape. - Stored by hash in `thumbs/.`, where the extension follows the response type (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the counter is `previewed`. Previews count toward the storage limit. @@ -155,6 +156,9 @@ Previews run after every walk: on a half-archived scrape. A paused walk still gets these passes for the pages it collected. - At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their previews were cached, so a stopped scrape does not leave blank tiles. +- Transient fetch failures keep the row pending and count up `preview_attempts`. After + `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` tries (default 3) the row is marked + `unreachable after N attempt(s)`, so dead hosts (Gfycat, Streamable) stop burning batches forever. - Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than file downloads. Previews are fetched in **batches of 25 with a 60 s cooldown** between batches (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 1606a70..db3952b 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -66,6 +66,8 @@ export interface ScrapeEngineOptions { walkCyclePages?: number; /** Pause after a rate limit is detected. Default 60 s. */ rateLimitPauseMs?: number; + /** Transient preview failures allowed before a row is marked permanently unreachable. Default 3. */ + previewMaxAttempts?: number; /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */ probe?: ( path: string @@ -562,12 +564,28 @@ export class ScrapeEngine { this.registerRateLimit(outcome.retryAfterMs); this.log(`${meta.id}: preview #${row.id} rate limited (${outcome.error ?? "429"})`); } else if (outcome.state === "failed") { - // Transient network failure: leave the row pending for the next pass. - failed++; - this.log( - `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed (transient): ` + - `${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})` - ); + // Transient network failure: retry on the next pass, but give up on hosts that + // stay unreachable (dead services like Gfycat and Streamable never recover). + const attempts = store.bumpPreviewAttempt(row.id); + const limit = this.options.previewMaxAttempts ?? 3; + + if (attempts >= limit) { + store.markPreviewError( + row.id, + `unreachable after ${attempts} attempt(s): ${outcome.error ?? "unknown"}` + ); + skipped++; + this.log( + `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} gave up after ` + + `${attempts} transient failures (${row.url.slice(0, 60)})` + ); + } else { + failed++; + this.log( + `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed ` + + `(transient ${attempts}/${limit}): ${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})` + ); + } } else { store.markPreviewError(row.id, outcome.error ?? outcome.state); skipped++; diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 1f04cf0..71e5c02 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -145,6 +145,7 @@ export interface MediaRow { file_path: string | null; download_error: string | null; preview_error: string | null; + preview_attempts: number; created_at: string; } @@ -184,6 +185,7 @@ CREATE TABLE IF NOT EXISTS media ( file_path TEXT, download_error TEXT, preview_error TEXT, + preview_attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL ); @@ -226,6 +228,9 @@ export class ScrapeStore { if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT"); if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT"); if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL"); + if (!names.has("preview_attempts")) { + this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0"); + } } close(): void { @@ -379,6 +384,16 @@ export class ScrapeStore { .run(preview.hash, preview.bytes, preview.ext ?? "webp", id); } + /** Count one transient preview failure and return the new total. */ + bumpPreviewAttempt(id: number): number { + this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id); + const row = this.db.prepare("SELECT preview_attempts FROM media WHERE id = ?").get(id) as + | { preview_attempts: number | null } + | undefined; + + return row?.preview_attempts ?? 0; + } + markPreviewError(id: number, error: string): void { this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id); } diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index bc2fd13..c5070ba 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -1063,6 +1063,61 @@ void (async () => { assert.equal(second.calls[0].before, "499"); }); + await test("transient preview failures give up after the attempt limit", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("650", { + embeds: [ + { + type: "video", + url: "https://gfycat.com/deadclip", + title: null, + provider: "Gfycat", + videoUrl: "https://gfycat.com/deadclip", + imageUrl: null, + thumbnailUrl: "https://thumbs.gfycat.com/dead.webp" + } + ] + }) + ] + } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 2, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: true, + previewMaxAttempts: 3, + download: { + fetchImpl: async () => { + throw new Error("fetch failed"); + }, + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + const dir = join(root, started.id); + for (let pass = 0; pass < 3; pass++) await engine.backfillPreviews(started.id); + + const store = new ScrapeStore(dir); + const row = store.queryMedia({ kind: "video" })[0]; + assert.equal(row.preview_attempts, 3); + assert.match(String(row.preview_error), /^unreachable after 3 attempt/); + assert.equal(store.pendingPreviews(10, true).length, 0); + store.close(); + }); + await test("backfillPreviews fills rows left by a preview-less walk", async () => { const backend = new FakeBackend([ { diff --git a/poc/src/media.ts b/poc/src/media.ts index 422d9d0..7ad1861 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -178,6 +178,7 @@ export class MediaService { onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`), walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5), rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000), + previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),