From e3edd9c200d8fae9bdbff8aeb1fa55c25b230915 Mon Sep 17 00:00:00 2001 From: Fisto Date: Fri, 18 Sep 2026 11:59:41 +0200 Subject: [PATCH] Fix recovery stale-URL handling and task accounting --- CURRENT.md | 7 +++++++ docs/media-scraper.md | 4 ++++ media-scrape/src/engine.ts | 10 ++++++++-- media-scrape/src/store.ts | 17 ++++++++++++---- poc/public/media.js | 1 + poc/src/media.ts | 41 +++++++++++++++++++++++++++++++------- 6 files changed, 67 insertions(+), 13 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index ec8527e..3f6af38 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,13 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C55 Recovery fix. The first recovery run parked live files because refreshed embed URLs + land in `play_url` while preview resolution kept using the stale `url`; the engine now resolves + embed previews from `play_url` first and re-resolves from the refreshed URL on retry, and both + refresh branches canonicalize `url` to the direct media link. The task also counts rows that + park as suspects as `deferred` (not gone), tries each row once per run instead of looping, and + initializes the slopbot so guild scrapes refresh through the bot. The 192 suspects from the + broken run were converted back to unverified so they are retried. - [x] C54 Message links, suspect retries, and preview recovery. Items now carry `messageUrl` (from `media_sources` + scrape guild), the viewer shows/copies it instead of the expiring CDN link and opens a fresh stream for View full, YouTube items offer an **Open on YouTube** diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 15415d5..1c05a39 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -217,6 +217,10 @@ The archive never treats a transient HTTP answer as a dead link on the first try - A preview or download that fails with an HTTP status on Discord's CDN is retried once with a **refreshed URL** from the source message (the signed link expires in about a day). Only if the refreshed fetch fails is the row parked. +- Refreshing an embed canonicalizes both links: `play_url` gets the provider's direct media URL and + `url` falls back to it when Discord's `embed.url` is missing. Preview resolution for embeds uses + `play_url` first, so a refreshed file is never fetched through an expired `url`. This is what the + recovery task relies on. - Parked rows become **suspects**: they get one delayed retry (about 50 s). The retry runs after the batch, not immediately. If a batch is two-thirds suspect failures, the cooldown doubles (floor 60 s) and the retries run before the next batch. diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 659c99a..81a78e8 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -709,7 +709,8 @@ export class ScrapeEngine { // resolve one from the provider or the page before caching it. let previewRow = row; if (row.source === "embed" && !row.preview_url) { - const resolved = await resolveEmbedPreview(row, options); + // play_url is the refreshed direct media link; url may carry an expired signature. + const resolved = await resolveEmbedPreview({ ...row, url: row.play_url ?? row.url }, options); if (!resolved) { store.markPreviewError(row.id, "no preview source"); result.skipped++; @@ -812,7 +813,12 @@ export class ScrapeEngine { playUrl: row.play_url }); if (refreshed && refreshed !== previewRow.url) { - const retry = await downloadPreview({ ...previewRow, url: refreshed }, join(dir, "thumbs"), options); + let retryRow = { ...previewRow, url: refreshed }; + if (row.source === "embed") { + const again = await resolveEmbedPreview({ ...row, url: refreshed }, options); + if (again) retryRow = { ...retryRow, preview_url: again }; + } + const retry = await downloadPreview(retryRow, join(dir, "thumbs"), options); if (retry.state === "downloaded" && retry.hash) { store.markPreview(row.id, { hash: retry.hash, diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 738b380..c373170 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -572,13 +572,18 @@ export class ScrapeStore { this.db.prepare("UPDATE media SET preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?").run(id); } - /** Discord-CDN rows closed out with an HTTP error; the recovery task retries these. */ + /** + * Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows + * whose single retry already failed (they may have been hit by the same stale-URL bug). + */ discordPreviewFailures(limit = 5000): MediaRow[] { return this.db .prepare( `SELECT * FROM media - WHERE preview_hash IS NULL AND preview_error IS NOT NULL - AND lower(url) LIKE '%discordapp%' + WHERE preview_hash IS NULL + AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%' + AND (preview_error IS NOT NULL + OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL)) ORDER BY id LIMIT ?` ) .all(limit) as unknown as MediaRow[]; @@ -591,7 +596,11 @@ export class ScrapeStore { discordPreviewFailureCount(): number { const row = this.db .prepare( - "SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_error IS NOT NULL AND lower(url) LIKE '%discordapp%'" + `SELECT COUNT(*) AS count FROM media + WHERE preview_hash IS NULL + AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%' + AND (preview_error IS NOT NULL + OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))` ) .get() as { count: number | null } | undefined; return row?.count ?? 0; diff --git a/poc/public/media.js b/poc/public/media.js index bcd4487..e6dd6be 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -911,6 +911,7 @@ const parts = [`${status.total} to retry`]; if (status.processed) parts.push(`${status.processed} processed`); if (status.recovered) parts.push(`${status.recovered} recovered`); + if (status.deferred) parts.push(`${status.deferred} deferred`); if (status.gone) parts.push(`${status.gone} unavailable`); if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`); head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 "))); diff --git a/poc/src/media.ts b/poc/src/media.ts index a33ace6..5b8924c 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -194,6 +194,7 @@ export class MediaService { state: "idle" as "idle" | "running" | "paused" | "done", processed: 0, recovered: 0, + deferred: 0, gone: 0, startedAt: null as number | null, finishedAt: null as number | null, @@ -327,6 +328,7 @@ export class MediaService { total: number; processed: number; recovered: number; + deferred: number; gone: number; startedAt: string | null; finishedAt: string | null; @@ -353,6 +355,7 @@ export class MediaService { total, processed: this.recover.processed, recovered: this.recover.recovered, + deferred: this.recover.deferred, gone: this.recover.gone, startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null, finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null, @@ -367,8 +370,15 @@ export class MediaService { this.recover.startedAt = this.recover.startedAt ?? Date.now(); this.recover.processed = 0; this.recover.recovered = 0; + this.recover.deferred = 0; this.recover.gone = 0; this.deps.log?.("recover: retrying Discord-CDN previews marked gone"); + // Guild scrapes refresh fastest through the bot REST; without it the puppet is tried. + if (this.engine.list().some(meta => meta.backend === "slopbot")) { + void this.ensureSlopbot().then(ok => { + if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet"); + }); + } void this.runRecover(); } @@ -389,6 +399,10 @@ export class MediaService { .filter(meta => meta.state !== "deleted" && meta.state !== "deleting") .map(meta => meta.id); + // Each row is tried once per run; rows whose URL cannot be refreshed keep their state + // instead of being re-processed forever. + const attempted = new Set(); + while (this.recover.state === "running") { let acted = false; @@ -406,7 +420,10 @@ export class MediaService { } try { - const rows = store.discordPreviewFailures(10); + const rows = store + .discordPreviewFailures(200) + .filter(row => !attempted.has(`${scrapeId}:${row.id}`)) + .slice(0, 10); if (!rows.length) continue; acted = true; @@ -415,6 +432,9 @@ export class MediaService { for (const row of rows) { if (this.recover.state !== "running") break; + attempted.add(`${scrapeId}:${row.id}`); + this.recover.processed++; + const fresh = await this.refreshUrls(meta, { ...row, scrapeId, @@ -422,17 +442,19 @@ export class MediaService { } as MediaRow & { scrapeId: string; channelId: string }); if (fresh) { store.clearPreviewError(row.id); + store.clearSuspect(row.id); cleared.push(row); } else { this.recover.gone++; } - this.recover.processed++; } if (cleared.length && this.recover.state === "running") { await this.engine.previewBatch(scrapeId, cleared.length); for (const row of cleared) { - if (store.getMedia(row.id)?.preview_hash) this.recover.recovered++; + const after = store.getMedia(row.id); + if (after?.preview_hash) this.recover.recovered++; + else if (after?.preview_suspect) this.recover.deferred++; else this.recover.gone++; } } @@ -452,11 +474,14 @@ export class MediaService { this.taskHistory.unshift({ name: "Recover Discord previews", at: new Date().toISOString(), - detail: `${this.recover.recovered} recovered, ${this.recover.gone} unavailable` + detail: + `${this.recover.recovered} recovered, ` + + `${this.recover.deferred} deferred, ${this.recover.gone} unavailable` }); if (this.taskHistory.length > 10) this.taskHistory.length = 10; this.deps.log?.( - `recover: done, ${this.recover.recovered} recovered, ${this.recover.gone} unavailable` + `recover: done, ${this.recover.recovered} recovered, ` + + `${this.recover.deferred} deferred, ${this.recover.gone} unavailable` ); } @@ -767,8 +792,9 @@ export class MediaService { (candidate.video?.url && path(candidate.url ?? "") === path(item.url)) ); if (embed?.video?.url) { + // The direct media link is always fresh; a bare embed.url can be stale. return saveAndReturn( - { playUrl: embed.video.url, url: embed.url ?? item.url }, + { playUrl: embed.video.url, url: embed.url ?? embed.video.url }, embed.video.url ); } @@ -799,8 +825,9 @@ export class MediaService { (candidate?.videoUrl && path(candidate.url ?? "") === path(item.url)) ); if (embed?.videoUrl) { + // Same as the bot path: keep the canonical url fresh too. return saveAndReturn( - { playUrl: embed.videoUrl, url: embed.url ?? item.url }, + { playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl }, embed.videoUrl ); }