Cap transient preview retries and close dead hosts

media-scrape/src/engine.ts

@@ -66,6 +66,8 @@ export interface ScrapeEngineOptions {
     walkCyclePages?: number;
     /** Pause after a rate limit is detected. Default 60 s. */
     rateLimitPauseMs?: number;
+    /** Transient preview failures allowed before a row is marked permanently unreachable. Default 3. */
+    previewMaxAttempts?: number;
     /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
     probe?: (
         path: string
@@ -562,12 +564,28 @@ export class ScrapeEngine {
                 this.registerRateLimit(outcome.retryAfterMs);
                 this.log(`${meta.id}: preview #${row.id} rate limited (${outcome.error ?? "429"})`);
             } else if (outcome.state === "failed") {
-                // Transient network failure: leave the row pending for the next pass.
-                failed++;
-                this.log(
-                    `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed (transient): ` +
-                        `${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
-                );
+                // Transient network failure: retry on the next pass, but give up on hosts that
+                // stay unreachable (dead services like Gfycat and Streamable never recover).
+                const attempts = store.bumpPreviewAttempt(row.id);
+                const limit = this.options.previewMaxAttempts ?? 3;
+
+                if (attempts >= limit) {
+                    store.markPreviewError(
+                        row.id,
+                        `unreachable after ${attempts} attempt(s): ${outcome.error ?? "unknown"}`
+                    );
+                    skipped++;
+                    this.log(
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} gave up after ` +
+                            `${attempts} transient failures (${row.url.slice(0, 60)})`
+                    );
+                } else {
+                    failed++;
+                    this.log(
+                        `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed ` +
+                            `(transient ${attempts}/${limit}): ${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})`
+                    );
+                }
             } else {
                 store.markPreviewError(row.id, outcome.error ?? outcome.state);
                 skipped++;

media-scrape/src/store.ts

@@ -145,6 +145,7 @@ export interface MediaRow {
     file_path: string | null;
     download_error: string | null;
     preview_error: string | null;
+    preview_attempts: number;
     created_at: string;
 }
 
@@ -184,6 +185,7 @@ CREATE TABLE IF NOT EXISTS media (
     file_path TEXT,
     download_error TEXT,
     preview_error TEXT,
+    preview_attempts INTEGER NOT NULL DEFAULT 0,
     created_at TEXT NOT NULL
 );
 
@@ -226,6 +228,9 @@ export class ScrapeStore {
         if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT");
         if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
         if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
+        if (!names.has("preview_attempts")) {
+            this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0");
+        }
     }
 
     close(): void {
@@ -379,6 +384,16 @@ export class ScrapeStore {
             .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
     }
 
+    /** Count one transient preview failure and return the new total. */
+    bumpPreviewAttempt(id: number): number {
+        this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id);
+        const row = this.db.prepare("SELECT preview_attempts FROM media WHERE id = ?").get(id) as
+            | { preview_attempts: number | null }
+            | undefined;
+
+        return row?.preview_attempts ?? 0;
+    }
+
     markPreviewError(id: number, error: string): void {
         this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }