Fix recovery stale-URL handling and task accounting

CURRENT.md

@@ -40,6 +40,13 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C55 Recovery fix. The first recovery run parked live files because refreshed embed URLs
+    land in `play_url` while preview resolution kept using the stale `url`; the engine now resolves
+    embed previews from `play_url` first and re-resolves from the refreshed URL on retry, and both
+    refresh branches canonicalize `url` to the direct media link. The task also counts rows that
+    park as suspects as `deferred` (not gone), tries each row once per run instead of looping, and
+    initializes the slopbot so guild scrapes refresh through the bot. The 192 suspects from the
+    broken run were converted back to unverified so they are retried.
   - [x] C54 Message links, suspect retries, and preview recovery. Items now carry `messageUrl`
     (from `media_sources` + scrape guild), the viewer shows/copies it instead of the expiring CDN
     link and opens a fresh stream for View full, YouTube items offer an **Open on YouTube**

docs/media-scraper.md

@@ -217,6 +217,10 @@ The archive never treats a transient HTTP answer as a dead link on the first try
 - A preview or download that fails with an HTTP status on Discord's CDN is retried once with a
   **refreshed URL** from the source message (the signed link expires in about a day). Only if the
   refreshed fetch fails is the row parked.
+- Refreshing an embed canonicalizes both links: `play_url` gets the provider's direct media URL and
+  `url` falls back to it when Discord's `embed.url` is missing. Preview resolution for embeds uses
+  `play_url` first, so a refreshed file is never fetched through an expired `url`. This is what the
+  recovery task relies on.
 - Parked rows become **suspects**: they get one delayed retry (about 50 s). The retry runs after
   the batch, not immediately. If a batch is two-thirds suspect failures, the cooldown doubles
   (floor 60 s) and the retries run before the next batch.

media-scrape/src/engine.ts

@@ -709,7 +709,8 @@ export class ScrapeEngine {
             // resolve one from the provider or the page before caching it.
             let previewRow = row;
             if (row.source === "embed" && !row.preview_url) {
-                const resolved = await resolveEmbedPreview(row, options);
+                // play_url is the refreshed direct media link; url may carry an expired signature.
+                const resolved = await resolveEmbedPreview({ ...row, url: row.play_url ?? row.url }, options);
                 if (!resolved) {
                     store.markPreviewError(row.id, "no preview source");
                     result.skipped++;
@@ -812,7 +813,12 @@ export class ScrapeEngine {
                         playUrl: row.play_url
                     });
                     if (refreshed && refreshed !== previewRow.url) {
-                        const retry = await downloadPreview({ ...previewRow, url: refreshed }, join(dir, "thumbs"), options);
+                        let retryRow = { ...previewRow, url: refreshed };
+                        if (row.source === "embed") {
+                            const again = await resolveEmbedPreview({ ...row, url: refreshed }, options);
+                            if (again) retryRow = { ...retryRow, preview_url: again };
+                        }
+                        const retry = await downloadPreview(retryRow, join(dir, "thumbs"), options);
                         if (retry.state === "downloaded" && retry.hash) {
                             store.markPreview(row.id, {
                                 hash: retry.hash,

media-scrape/src/store.ts

@@ -572,13 +572,18 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?").run(id);
     }
 
-    /** Discord-CDN rows closed out with an HTTP error; the recovery task retries these. */
+    /**
+     * Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows
+     * whose single retry already failed (they may have been hit by the same stale-URL bug).
+     */
     discordPreviewFailures(limit = 5000): MediaRow[] {
         return this.db
             .prepare(
                 `SELECT * FROM media
-                 WHERE preview_hash IS NULL AND preview_error IS NOT NULL
+                 WHERE preview_hash IS NULL
-                   AND lower(url) LIKE '%discordapp%'
+                   AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
+                   AND (preview_error IS NOT NULL
+                        OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))
                  ORDER BY id LIMIT ?`
             )
             .all(limit) as unknown as MediaRow[];
@@ -591,7 +596,11 @@ export class ScrapeStore {
     discordPreviewFailureCount(): number {
         const row = this.db
             .prepare(
-                "SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_error IS NOT NULL AND lower(url) LIKE '%discordapp%'"
+                `SELECT COUNT(*) AS count FROM media
+                 WHERE preview_hash IS NULL
+                   AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
+                   AND (preview_error IS NOT NULL
+                        OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))`
             )
             .get() as { count: number | null } | undefined;
         return row?.count ?? 0;

poc/public/media.js

@@ -911,6 +911,7 @@
         const parts = [`${status.total} to retry`];
         if (status.processed) parts.push(`${status.processed} processed`);
         if (status.recovered) parts.push(`${status.recovered} recovered`);
+        if (status.deferred) parts.push(`${status.deferred} deferred`);
         if (status.gone) parts.push(`${status.gone} unavailable`);
         if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
         head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));

poc/src/media.ts

@@ -194,6 +194,7 @@ export class MediaService {
         state: "idle" as "idle" | "running" | "paused" | "done",
         processed: 0,
         recovered: 0,
+        deferred: 0,
         gone: 0,
         startedAt: null as number | null,
         finishedAt: null as number | null,
@@ -327,6 +328,7 @@ export class MediaService {
         total: number;
         processed: number;
         recovered: number;
+        deferred: number;
         gone: number;
         startedAt: string | null;
         finishedAt: string | null;
@@ -353,6 +355,7 @@ export class MediaService {
             total,
             processed: this.recover.processed,
             recovered: this.recover.recovered,
+            deferred: this.recover.deferred,
             gone: this.recover.gone,
             startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null,
             finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null,
@@ -367,8 +370,15 @@ export class MediaService {
         this.recover.startedAt = this.recover.startedAt ?? Date.now();
         this.recover.processed = 0;
         this.recover.recovered = 0;
+        this.recover.deferred = 0;
         this.recover.gone = 0;
         this.deps.log?.("recover: retrying Discord-CDN previews marked gone");
+        // Guild scrapes refresh fastest through the bot REST; without it the puppet is tried.
+        if (this.engine.list().some(meta => meta.backend === "slopbot")) {
+            void this.ensureSlopbot().then(ok => {
+                if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet");
+            });
+        }
         void this.runRecover();
     }
 
@@ -389,6 +399,10 @@ export class MediaService {
             .filter(meta => meta.state !== "deleted" && meta.state !== "deleting")
             .map(meta => meta.id);
 
+        // Each row is tried once per run; rows whose URL cannot be refreshed keep their state
+        // instead of being re-processed forever.
+        const attempted = new Set<string>();
+
         while (this.recover.state === "running") {
             let acted = false;
 
@@ -406,7 +420,10 @@ export class MediaService {
                 }
 
                 try {
-                    const rows = store.discordPreviewFailures(10);
+                    const rows = store
+                        .discordPreviewFailures(200)
+                        .filter(row => !attempted.has(`${scrapeId}:${row.id}`))
+                        .slice(0, 10);
                     if (!rows.length) continue;
 
                     acted = true;
@@ -415,6 +432,9 @@ export class MediaService {
 
                     for (const row of rows) {
                         if (this.recover.state !== "running") break;
+                        attempted.add(`${scrapeId}:${row.id}`);
+                        this.recover.processed++;
+
                         const fresh = await this.refreshUrls(meta, {
                             ...row,
                             scrapeId,
@@ -422,17 +442,19 @@ export class MediaService {
                         } as MediaRow & { scrapeId: string; channelId: string });
                         if (fresh) {
                             store.clearPreviewError(row.id);
+                            store.clearSuspect(row.id);
                             cleared.push(row);
                         } else {
                             this.recover.gone++;
                         }
-                        this.recover.processed++;
                     }
 
                     if (cleared.length && this.recover.state === "running") {
                         await this.engine.previewBatch(scrapeId, cleared.length);
                         for (const row of cleared) {
-                            if (store.getMedia(row.id)?.preview_hash) this.recover.recovered++;
+                            const after = store.getMedia(row.id);
+                            if (after?.preview_hash) this.recover.recovered++;
+                            else if (after?.preview_suspect) this.recover.deferred++;
                             else this.recover.gone++;
                         }
                     }
@@ -452,11 +474,14 @@ export class MediaService {
         this.taskHistory.unshift({
             name: "Recover Discord previews",
             at: new Date().toISOString(),
-            detail: `${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
+            detail:
+                `${this.recover.recovered} recovered, ` +
+                `${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
         });
         if (this.taskHistory.length > 10) this.taskHistory.length = 10;
         this.deps.log?.(
-            `recover: done, ${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
+            `recover: done, ${this.recover.recovered} recovered, ` +
+                `${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
         );
     }
 
@@ -767,8 +792,9 @@ export class MediaService {
                             (candidate.video?.url && path(candidate.url ?? "") === path(item.url))
                     );
                     if (embed?.video?.url) {
+                        // The direct media link is always fresh; a bare embed.url can be stale.
                         return saveAndReturn(
-                            { playUrl: embed.video.url, url: embed.url ?? item.url },
+                            { playUrl: embed.video.url, url: embed.url ?? embed.video.url },
                             embed.video.url
                         );
                     }
@@ -799,8 +825,9 @@ export class MediaService {
                             (candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
                     );
                     if (embed?.videoUrl) {
+                        // Same as the bot path: keep the canonical url fresh too.
                         return saveAndReturn(
-                            { playUrl: embed.videoUrl, url: embed.url ?? item.url },
+                            { playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl },
                             embed.videoUrl
                         );
                     }