Cap transient preview retries and close dead hosts

2026-09-17 08:44:55 +02:00
parent 445c6ca2fb
commit e52623f1b2
6 changed files with 104 additions and 7 deletions
@@ -40,6 +40,10 @@ mobile draft at 390px.
  - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
    cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
    `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C42 Preview attempts cap. The restart log showed the backfill retrying dead hosts (Gfycat
+    DNS is gone, Streamable answers 400) forever as "transient". Rows now carry
+    `preview_attempts`; after `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` (3) the row is marked
+    `unreachable after N attempt(s)` and leaves the pending queue. Unit-tested.
  - [x] C41 Restart keeps its logs. A button restart spawned the replacement with `stdio: ignore`,
    so anything after the restart vanished. The replacement now appends to `daemon.log` beside the
    token file (`VESKSLOP_DAEMON_LOG` overrides); documented in [docs/daemon.md](docs/daemon.md).
@@ -147,7 +147,8 @@ Previews run after every walk:
  (`ffmpeg -filter_complex showwavespic`) as their preview, rendered by ffmpeg from the URL.
 - Transient network failures are left pending for the next backfill pass; only skips (4xx, too
  large, no source) are marked permanent.
- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
+- One attempt per row, no retries; a permanent failure (`http 404`, `http 403`, `too large`) marks
+  `preview_error` and never fails the scrape.
 - Stored by hash in `thumbs/<sha256>.<ext>`, where the extension follows the response type
  (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the
  counter is `previewed`. Previews count toward the storage limit.
@@ -155,6 +156,9 @@ Previews run after every walk:
  on a half-archived scrape. A paused walk still gets these passes for the pages it collected.
 - At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their
  previews were cached, so a stopped scrape does not leave blank tiles.
+- Transient fetch failures keep the row pending and count up `preview_attempts`. After
+  `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` tries (default 3) the row is marked
+  `unreachable after N attempt(s)`, so dead hosts (Gfycat, Streamable) stop burning batches forever.
 - Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than
  file downloads. Previews are fetched in **batches of 25 with a 60 s cooldown** between batches
  (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of
@@ -66,6 +66,8 @@ export interface ScrapeEngineOptions {
    walkCyclePages?: number;
    /** Pause after a rate limit is detected. Default 60 s. */
    rateLimitPauseMs?: number;
+    /** Transient preview failures allowed before a row is marked permanently unreachable. Default 3. */
+    previewMaxAttempts?: number;
    /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
    probe?: (
        path: string
@@ -562,12 +564,28 @@ export class ScrapeEngine {
                this.registerRateLimit(outcome.retryAfterMs);
                this.log(`${meta.id}: preview #${row.id} rate limited (${outcome.error ?? "429"})`);
            } else if (outcome.state === "failed") {
-                // Transient network failure: leave the row pending for the next pass.
-                failed++;
-                this.log(
-                    `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed (transient): ` +
-                        `${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
-                );
+                // Transient network failure: retry on the next pass, but give up on hosts that
+                // stay unreachable (dead services like Gfycat and Streamable never recover).
+                const attempts = store.bumpPreviewAttempt(row.id);
+                const limit = this.options.previewMaxAttempts ?? 3;
+
+                if (attempts >= limit) {
+                    store.markPreviewError(
+                        row.id,
+                        `unreachable after ${attempts} attempt(s): ${outcome.error ?? "unknown"}`
+                    );
+                    skipped++;
+                    this.log(
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} gave up after ` +
+                            `${attempts} transient failures (${row.url.slice(0, 60)})`
+                    );
+                } else {
+                    failed++;
+                    this.log(
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed ` +
+                            `(transient ${attempts}/${limit}): ${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
+                    );
+                }
            } else {
                store.markPreviewError(row.id, outcome.error ?? outcome.state);
                skipped++;
@@ -145,6 +145,7 @@ export interface MediaRow {
    file_path: string | null;
    download_error: string | null;
    preview_error: string | null;
+    preview_attempts: number;
    created_at: string;
 }

@@ -184,6 +185,7 @@ CREATE TABLE IF NOT EXISTS media (
    file_path TEXT,
    download_error TEXT,
    preview_error TEXT,
+    preview_attempts INTEGER NOT NULL DEFAULT 0,
    created_at TEXT NOT NULL
 );

@@ -226,6 +228,9 @@ export class ScrapeStore {
        if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT");
        if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
        if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
+        if (!names.has("preview_attempts")) {
+            this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0");
+        }
    }

    close(): void {
@@ -379,6 +384,16 @@ export class ScrapeStore {
            .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
    }

+    /** Count one transient preview failure and return the new total. */
+    bumpPreviewAttempt(id: number): number {
+        this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id);
+        const row = this.db.prepare("SELECT preview_attempts FROM media WHERE id = ?").get(id) as
+            | { preview_attempts: number | null }
+            | undefined;
+
+        return row?.preview_attempts ?? 0;
+    }
+
    markPreviewError(id: number, error: string): void {
        this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
    }
@@ -1063,6 +1063,61 @@ void (async () => {
        assert.equal(second.calls[0].before, "499");
    });

+    await test("transient preview failures give up after the attempt limit", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("650", {
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://gfycat.com/deadclip",
+                                title: null,
+                                provider: "Gfycat",
+                                videoUrl: "https://gfycat.com/deadclip",
+                                imageUrl: null,
+                                thumbnailUrl: "https://thumbs.gfycat.com/dead.webp"
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            previewMaxAttempts: 3,
+            download: {
+                fetchImpl: async () => {
+                    throw new Error("fetch failed");
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const dir = join(root, started.id);
+        for (let pass = 0; pass < 3; pass++) await engine.backfillPreviews(started.id);
+
+        const store = new ScrapeStore(dir);
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.preview_attempts, 3);
+        assert.match(String(row.preview_error), /^unreachable after 3 attempt/);
+        assert.equal(store.pendingPreviews(10, true).length, 0);
+        store.close();
+    });
+
    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
        const backend = new FakeBackend([
            {
@@ -178,6 +178,7 @@ export class MediaService {
            onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
            walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
            rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
+            previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3),
            download: {
                maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
                previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),