diff --git a/media-scrape/src/backfill.ts b/media-scrape/src/backfill.ts index 4af1c9d..34dc2c6 100644 --- a/media-scrape/src/backfill.ts +++ b/media-scrape/src/backfill.ts @@ -422,7 +422,9 @@ export class BackfillQueue { while (!job.stop) { const group = this.plan(false).get(job.chatId); - const member = group?.members.find(candidate => candidate.pending > 0); + const member = group?.members.find( + candidate => candidate.pending > 0 || candidate.suspect > 0 + ); if (!member) break; const limit = this.engine.rateLimitStatus(); @@ -442,6 +444,26 @@ export class BackfillQueue { } if (job.stop) break; + // No preview work left, but delayed suspect retries are due: run those alone. + if (member.pending === 0) { + const retried = await this.engine.previewSuspects( + member.scrapeId, + Math.max(2, Math.ceil(this.options.getSettings().batchSize / 2)) + ); + job.processed += retried.cached + retried.unverified; + if (retried.cached + retried.unverified + retried.suspect > 0) { + this.options.log?.( + `backfill: ${job.chatId} suspect sweep: ${retried.cached} recovered, ` + + `${retried.unverified} unverified` + ); + await this.sleep(1_000); + continue; + } + + // Nothing due; the rest wait for another run. + break; + } + const settings = this.options.getSettings(); const result: PreviewBatchResult = await this.engine.previewBatch( member.scrapeId, diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index c373170..59243f5 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -576,17 +576,18 @@ export class ScrapeStore { * Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows * whose single retry already failed (they may have been hit by the same stale-URL bug). */ - discordPreviewFailures(limit = 5000): MediaRow[] { + discordPreviewFailures(limit = 5000, nowMs = Date.now()): MediaRow[] { return this.db .prepare( `SELECT * FROM media WHERE preview_hash IS NULL AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%' AND (preview_error IS NOT NULL - OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL)) + OR (preview_suspect IS NOT NULL + AND (preview_retry_at IS NULL OR preview_retry_at <= ?))) ORDER BY id LIMIT ?` ) - .all(limit) as unknown as MediaRow[]; + .all(nowMs, limit) as unknown as MediaRow[]; } clearPreviewError(id: number): void { @@ -599,8 +600,7 @@ export class ScrapeStore { `SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%' - AND (preview_error IS NOT NULL - OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))` + AND (preview_error IS NOT NULL OR preview_suspect IS NOT NULL)` ) .get() as { count: number | null } | undefined; return row?.count ?? 0;