Cap transient preview retries and close dead hosts

CURRENT.md

@@ -40,6 +40,10 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C42 Preview attempts cap. The restart log showed the backfill retrying dead hosts (Gfycat
+    DNS is gone, Streamable answers 400) forever as "transient". Rows now carry
+    `preview_attempts`; after `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` (3) the row is marked
+    `unreachable after N attempt(s)` and leaves the pending queue. Unit-tested.
   - [x] C41 Restart keeps its logs. A button restart spawned the replacement with `stdio: ignore`,
     so anything after the restart vanished. The replacement now appends to `daemon.log` beside the
     token file (`VESKSLOP_DAEMON_LOG` overrides); documented in [docs/daemon.md](docs/daemon.md).

docs/media-scraper.md

@@ -147,7 +147,8 @@ Previews run after every walk:
   (`ffmpeg -filter_complex showwavespic`) as their preview, rendered by ffmpeg from the URL.
 - Transient network failures are left pending for the next backfill pass; only skips (4xx, too
   large, no source) are marked permanent.
-- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
+- One attempt per row, no retries; a permanent failure (`http 404`, `http 403`, `too large`) marks
+  `preview_error` and never fails the scrape.
 - Stored by hash in `thumbs/<sha256>.<ext>`, where the extension follows the response type
   (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the
   counter is `previewed`. Previews count toward the storage limit.
@@ -155,6 +156,9 @@ Previews run after every walk:
   on a half-archived scrape. A paused walk still gets these passes for the pages it collected.
 - At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their
   previews were cached, so a stopped scrape does not leave blank tiles.
+- Transient fetch failures keep the row pending and count up `preview_attempts`. After
+  `VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS` tries (default 3) the row is marked
+  `unreachable after N attempt(s)`, so dead hosts (Gfycat, Streamable) stop burning batches forever.
 - Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than
   file downloads. Previews are fetched in **batches of 25 with a 60 s cooldown** between batches
   (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of

media-scrape/src/engine.ts

@@ -66,6 +66,8 @@ export interface ScrapeEngineOptions {
     walkCyclePages?: number;
     /** Pause after a rate limit is detected. Default 60 s. */
     rateLimitPauseMs?: number;
+    /** Transient preview failures allowed before a row is marked permanently unreachable. Default 3. */
+    previewMaxAttempts?: number;
     /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
     probe?: (
         path: string
@@ -562,12 +564,28 @@ export class ScrapeEngine {
                 this.registerRateLimit(outcome.retryAfterMs);
                 this.log(`${meta.id}: preview #${row.id} rate limited (${outcome.error ?? "429"})`);
             } else if (outcome.state === "failed") {
-                // Transient network failure: leave the row pending for the next pass.
+                // Transient network failure: retry on the next pass, but give up on hosts that
+                // stay unreachable (dead services like Gfycat and Streamable never recover).
+                const attempts = store.bumpPreviewAttempt(row.id);
+                const limit = this.options.previewMaxAttempts ?? 3;
+
+                if (attempts >= limit) {
+                    store.markPreviewError(
+                        row.id,
+                        `unreachable after ${attempts} attempt(s): ${outcome.error ?? "unknown"}`
+                    );
+                    skipped++;
+                    this.log(
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} gave up after ` +
+                            `${attempts} transient failures (${row.url.slice(0, 60)})`
+                    );
+                } else {
                     failed++;
                     this.log(
-                    `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed (transient): ` +
-                        `${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed ` +
+                            `(transient ${attempts}/${limit}): ${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
                     );
+                }
             } else {
                 store.markPreviewError(row.id, outcome.error ?? outcome.state);
                 skipped++;

media-scrape/src/store.ts

@@ -145,6 +145,7 @@ export interface MediaRow {
     file_path: string | null;
     download_error: string | null;
     preview_error: string | null;
+    preview_attempts: number;
     created_at: string;
 }
 
@@ -184,6 +185,7 @@ CREATE TABLE IF NOT EXISTS media (
     file_path TEXT,
     download_error TEXT,
     preview_error TEXT,
+    preview_attempts INTEGER NOT NULL DEFAULT 0,
     created_at TEXT NOT NULL
 );
 
@@ -226,6 +228,9 @@ export class ScrapeStore {
         if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT");
         if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
         if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
+        if (!names.has("preview_attempts")) {
+            this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0");
+        }
     }
 
     close(): void {
@@ -379,6 +384,16 @@ export class ScrapeStore {
             .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
     }
 
+    /** Count one transient preview failure and return the new total. */
+    bumpPreviewAttempt(id: number): number {
+        this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id);
+        const row = this.db.prepare("SELECT preview_attempts FROM media WHERE id = ?").get(id) as
+            | { preview_attempts: number | null }
+            | undefined;
+
+        return row?.preview_attempts ?? 0;
+    }
+
     markPreviewError(id: number, error: string): void {
         this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }

media-scrape/test/run.ts

@@ -1063,6 +1063,61 @@ void (async () => {
         assert.equal(second.calls[0].before, "499");
     });
 
+    await test("transient preview failures give up after the attempt limit", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("650", {
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://gfycat.com/deadclip",
+                                title: null,
+                                provider: "Gfycat",
+                                videoUrl: "https://gfycat.com/deadclip",
+                                imageUrl: null,
+                                thumbnailUrl: "https://thumbs.gfycat.com/dead.webp"
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            previewMaxAttempts: 3,
+            download: {
+                fetchImpl: async () => {
+                    throw new Error("fetch failed");
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const dir = join(root, started.id);
+        for (let pass = 0; pass < 3; pass++) await engine.backfillPreviews(started.id);
+
+        const store = new ScrapeStore(dir);
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.preview_attempts, 3);
+        assert.match(String(row.preview_error), /^unreachable after 3 attempt/);
+        assert.equal(store.pendingPreviews(10, true).length, 0);
+        store.close();
+    });
+
     await test("backfillPreviews fills rows left by a preview-less walk", async () => {
         const backend = new FakeBackend([
             {

poc/src/media.ts

@@ -178,6 +178,7 @@ export class MediaService {
             onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
             walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
             rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
+            previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3),
             download: {
                 maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
                 previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),