Fix recovery stale-URL handling and task accounting
This commit is contained in:
Fisto
@@ -911,6 +911,7 @@
|
||||
const parts = [`${status.total} to retry`];
|
||||
if (status.processed) parts.push(`${status.processed} processed`);
|
||||
if (status.recovered) parts.push(`${status.recovered} recovered`);
|
||||
if (status.deferred) parts.push(`${status.deferred} deferred`);
|
||||
if (status.gone) parts.push(`${status.gone} unavailable`);
|
||||
if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
|
||||
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
|
||||
|
||||
+34
-7
@@ -194,6 +194,7 @@ export class MediaService {
|
||||
state: "idle" as "idle" | "running" | "paused" | "done",
|
||||
processed: 0,
|
||||
recovered: 0,
|
||||
deferred: 0,
|
||||
gone: 0,
|
||||
startedAt: null as number | null,
|
||||
finishedAt: null as number | null,
|
||||
@@ -327,6 +328,7 @@ export class MediaService {
|
||||
total: number;
|
||||
processed: number;
|
||||
recovered: number;
|
||||
deferred: number;
|
||||
gone: number;
|
||||
startedAt: string | null;
|
||||
finishedAt: string | null;
|
||||
@@ -353,6 +355,7 @@ export class MediaService {
|
||||
total,
|
||||
processed: this.recover.processed,
|
||||
recovered: this.recover.recovered,
|
||||
deferred: this.recover.deferred,
|
||||
gone: this.recover.gone,
|
||||
startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null,
|
||||
finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null,
|
||||
@@ -367,8 +370,15 @@ export class MediaService {
|
||||
this.recover.startedAt = this.recover.startedAt ?? Date.now();
|
||||
this.recover.processed = 0;
|
||||
this.recover.recovered = 0;
|
||||
this.recover.deferred = 0;
|
||||
this.recover.gone = 0;
|
||||
this.deps.log?.("recover: retrying Discord-CDN previews marked gone");
|
||||
// Guild scrapes refresh fastest through the bot REST; without it the puppet is tried.
|
||||
if (this.engine.list().some(meta => meta.backend === "slopbot")) {
|
||||
void this.ensureSlopbot().then(ok => {
|
||||
if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet");
|
||||
});
|
||||
}
|
||||
void this.runRecover();
|
||||
}
|
||||
|
||||
@@ -389,6 +399,10 @@ export class MediaService {
|
||||
.filter(meta => meta.state !== "deleted" && meta.state !== "deleting")
|
||||
.map(meta => meta.id);
|
||||
|
||||
// Each row is tried once per run; rows whose URL cannot be refreshed keep their state
|
||||
// instead of being re-processed forever.
|
||||
const attempted = new Set<string>();
|
||||
|
||||
while (this.recover.state === "running") {
|
||||
let acted = false;
|
||||
|
||||
@@ -406,7 +420,10 @@ export class MediaService {
|
||||
}
|
||||
|
||||
try {
|
||||
const rows = store.discordPreviewFailures(10);
|
||||
const rows = store
|
||||
.discordPreviewFailures(200)
|
||||
.filter(row => !attempted.has(`${scrapeId}:${row.id}`))
|
||||
.slice(0, 10);
|
||||
if (!rows.length) continue;
|
||||
|
||||
acted = true;
|
||||
@@ -415,6 +432,9 @@ export class MediaService {
|
||||
|
||||
for (const row of rows) {
|
||||
if (this.recover.state !== "running") break;
|
||||
attempted.add(`${scrapeId}:${row.id}`);
|
||||
this.recover.processed++;
|
||||
|
||||
const fresh = await this.refreshUrls(meta, {
|
||||
...row,
|
||||
scrapeId,
|
||||
@@ -422,17 +442,19 @@ export class MediaService {
|
||||
} as MediaRow & { scrapeId: string; channelId: string });
|
||||
if (fresh) {
|
||||
store.clearPreviewError(row.id);
|
||||
store.clearSuspect(row.id);
|
||||
cleared.push(row);
|
||||
} else {
|
||||
this.recover.gone++;
|
||||
}
|
||||
this.recover.processed++;
|
||||
}
|
||||
|
||||
if (cleared.length && this.recover.state === "running") {
|
||||
await this.engine.previewBatch(scrapeId, cleared.length);
|
||||
for (const row of cleared) {
|
||||
if (store.getMedia(row.id)?.preview_hash) this.recover.recovered++;
|
||||
const after = store.getMedia(row.id);
|
||||
if (after?.preview_hash) this.recover.recovered++;
|
||||
else if (after?.preview_suspect) this.recover.deferred++;
|
||||
else this.recover.gone++;
|
||||
}
|
||||
}
|
||||
@@ -452,11 +474,14 @@ export class MediaService {
|
||||
this.taskHistory.unshift({
|
||||
name: "Recover Discord previews",
|
||||
at: new Date().toISOString(),
|
||||
detail: `${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
|
||||
detail:
|
||||
`${this.recover.recovered} recovered, ` +
|
||||
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
|
||||
});
|
||||
if (this.taskHistory.length > 10) this.taskHistory.length = 10;
|
||||
this.deps.log?.(
|
||||
`recover: done, ${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
|
||||
`recover: done, ${this.recover.recovered} recovered, ` +
|
||||
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -767,8 +792,9 @@ export class MediaService {
|
||||
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
|
||||
);
|
||||
if (embed?.video?.url) {
|
||||
// The direct media link is always fresh; a bare embed.url can be stale.
|
||||
return saveAndReturn(
|
||||
{ playUrl: embed.video.url, url: embed.url ?? item.url },
|
||||
{ playUrl: embed.video.url, url: embed.url ?? embed.video.url },
|
||||
embed.video.url
|
||||
);
|
||||
}
|
||||
@@ -799,8 +825,9 @@ export class MediaService {
|
||||
(candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
|
||||
);
|
||||
if (embed?.videoUrl) {
|
||||
// Same as the bot path: keep the canonical url fresh too.
|
||||
return saveAndReturn(
|
||||
{ playUrl: embed.videoUrl, url: embed.url ?? item.url },
|
||||
{ playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl },
|
||||
embed.videoUrl
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user