zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sweep due suspects and count them in recovery

Browse Source
This commit is contained in:
Fisto
2026-09-18 12:04:41 +02:00
parent e3edd9c200
commit faacedd8f5
2 changed files with 28 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/backfill.ts
+23 -1
View File
@@ -422,7 +422,9 @@ export class BackfillQueue {
while (!job.stop) {
const group = this.plan(false).get(job.chatId);
const member = group?.members.find(candidate => candidate.pending > 0);
const member = group?.members.find(
candidate => candidate.pending > 0 || candidate.suspect > 0
);
if (!member) break;
const limit = this.engine.rateLimitStatus();
@@ -442,6 +444,26 @@ export class BackfillQueue {
}
if (job.stop) break;
// No preview work left, but delayed suspect retries are due: run those alone.
if (member.pending === 0) {
const retried = await this.engine.previewSuspects(
member.scrapeId,
Math.max(2, Math.ceil(this.options.getSettings().batchSize / 2))
);
job.processed += retried.cached + retried.unverified;
if (retried.cached + retried.unverified + retried.suspect > 0) {
this.options.log?.(
`backfill: ${job.chatId} suspect sweep: ${retried.cached} recovered, ` +
`${retried.unverified} unverified`
);
await this.sleep(1_000);
continue;
}
// Nothing due; the rest wait for another run.
break;
}
const settings = this.options.getSettings();
const result: PreviewBatchResult = await this.engine.previewBatch(
member.scrapeId,
media-scrape/src/store.ts
+5 -5
View File
@@ -576,17 +576,18 @@ export class ScrapeStore {
* Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows
* whose single retry already failed (they may have been hit by the same stale-URL bug).
*/
discordPreviewFailures(limit = 5000): MediaRow[] {
discordPreviewFailures(limit = 5000, nowMs = Date.now()): MediaRow[] {
return this.db
.prepare(
`SELECT * FROM media
WHERE preview_hash IS NULL
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
AND (preview_error IS NOT NULL
OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))
OR (preview_suspect IS NOT NULL
AND (preview_retry_at IS NULL OR preview_retry_at <= ?)))
ORDER BY id LIMIT ?`
)
.all(limit) as unknown as MediaRow[];
.all(nowMs, limit) as unknown as MediaRow[];
}
clearPreviewError(id: number): void {
@@ -599,8 +600,7 @@ export class ScrapeStore {
`SELECT COUNT(*) AS count FROM media
WHERE preview_hash IS NULL
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
AND (preview_error IS NOT NULL
OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))`
AND (preview_error IS NOT NULL OR preview_suspect IS NOT NULL)`
)
.get() as { count: number | null } | undefined;
return row?.count ?? 0;