Add message links, suspect retries, and Discord preview recovery

media-scrape/src/backfill.ts

@@ -31,6 +31,10 @@ export interface BackfillMember {
     pending: number;
     cached: number;
     failed: number;
+    /** Preview failures kept for the delayed suspect retry. */
+    suspect: number;
+    /** Suspect rows whose retry failed too. */
+    unverified: number;
     total: number;
     downloaded: number;
     bytes: number;
@@ -154,6 +158,8 @@ export class BackfillQueue {
                 pending: 0,
                 cached: 0,
                 failed: 0,
+                suspect: 0,
+                unverified: 0,
                 total: 0,
                 downloaded: meta.downloaded ?? 0,
                 bytes: 0,
@@ -167,6 +173,8 @@ export class BackfillQueue {
                     member.pending = counts.pending;
                     member.cached = counts.cached;
                     member.failed = counts.failed;
+                    member.suspect = counts.suspect;
+                    member.unverified = counts.unverified;
                     member.total = counts.total;
                     for (const kind of Object.keys(member.pendingByKind) as MediaKind[]) {
                         member.pendingByKind[kind] = counts.byKind[kind].pending;
@@ -408,6 +416,9 @@ export class BackfillQueue {
 
     private async runJob(job: Job): Promise<void> {
         let stalled = 0;
+        // The cooldown doubles when a batch is mostly suspect failures; it stays raised for the
+        // rest of this job so throttling is not immediately forgotten.
+        let pauseMs = this.options.getSettings().batchPauseMs;
 
         while (!job.stop) {
             const group = this.plan(false).get(job.chatId);
@@ -457,8 +468,30 @@ export class BackfillQueue {
                 stalled = 0;
             }
 
-            if (settings.batchPauseMs > 0 && progressed > 0) {
-                await this.sleep(settings.batchPauseMs);
+            // Suspect failures get one delayed retry. When they reach two thirds of a batch the
+            // cooldown doubles (floor 60 s) and the retry runs before the next batch.
+            const suspectThreshold = Math.ceil((settings.batchSize * 2) / 3);
+            if (result.suspect > 0 && result.suspect >= suspectThreshold) {
+                pauseMs = Math.min(300_000, Math.max(60_000, pauseMs * 2));
+                this.options.log?.(
+                    `backfill: ${job.chatId} ${result.suspect} suspect failures, cooldown now ${pauseMs} ms`
+                );
+            }
+
+            if (pauseMs > 0 && (progressed > 0 || result.suspect > 0)) {
+                await this.sleep(pauseMs);
+                if (job.stop) break;
+            }
+
+            if (result.suspect > 0 || result.unverified > 0) {
+                const retried = await this.engine.previewSuspects(
+                    member.scrapeId,
+                    Math.max(2, Math.ceil(settings.batchSize / 2))
+                );
+                this.options.log?.(
+                    `backfill: ${job.chatId} suspect retry: ${retried.cached} recovered, ` +
+                        `${retried.unverified} left unverified`
+                );
             }
         }