Sweep due suspects and count them in recovery

media-scrape/src/backfill.ts

@@ -422,7 +422,9 @@ export class BackfillQueue {
 
         while (!job.stop) {
             const group = this.plan(false).get(job.chatId);
-            const member = group?.members.find(candidate => candidate.pending > 0);
+            const member = group?.members.find(
+                candidate => candidate.pending > 0 || candidate.suspect > 0
+            );
             if (!member) break;
 
             const limit = this.engine.rateLimitStatus();
@@ -442,6 +444,26 @@ export class BackfillQueue {
             }
             if (job.stop) break;
 
+            // No preview work left, but delayed suspect retries are due: run those alone.
+            if (member.pending === 0) {
+                const retried = await this.engine.previewSuspects(
+                    member.scrapeId,
+                    Math.max(2, Math.ceil(this.options.getSettings().batchSize / 2))
+                );
+                job.processed += retried.cached + retried.unverified;
+                if (retried.cached + retried.unverified + retried.suspect > 0) {
+                    this.options.log?.(
+                        `backfill: ${job.chatId} suspect sweep: ${retried.cached} recovered, ` +
+                            `${retried.unverified} unverified`
+                    );
+                    await this.sleep(1_000);
+                    continue;
+                }
+
+                // Nothing due; the rest wait for another run.
+                break;
+            }
+
             const settings = this.options.getSettings();
             const result: PreviewBatchResult = await this.engine.previewBatch(
                 member.scrapeId,

media-scrape/src/store.ts

@@ -576,17 +576,18 @@ export class ScrapeStore {
      * Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows
      * whose single retry already failed (they may have been hit by the same stale-URL bug).
      */
-    discordPreviewFailures(limit = 5000): MediaRow[] {
+    discordPreviewFailures(limit = 5000, nowMs = Date.now()): MediaRow[] {
         return this.db
             .prepare(
                 `SELECT * FROM media
                  WHERE preview_hash IS NULL
                    AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
                    AND (preview_error IS NOT NULL
-                        OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))
+                        OR (preview_suspect IS NOT NULL
+                            AND (preview_retry_at IS NULL OR preview_retry_at <= ?)))
                  ORDER BY id LIMIT ?`
             )
-            .all(limit) as unknown as MediaRow[];
+            .all(nowMs, limit) as unknown as MediaRow[];
     }
 
     clearPreviewError(id: number): void {
@@ -599,8 +600,7 @@ export class ScrapeStore {
                 `SELECT COUNT(*) AS count FROM media
                  WHERE preview_hash IS NULL
                    AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
-                   AND (preview_error IS NOT NULL
+                   AND (preview_error IS NOT NULL OR preview_suspect IS NOT NULL)`
-                        OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))`
             )
             .get() as { count: number | null } | undefined;
         return row?.count ?? 0;