Add message links, suspect retries, and Discord preview recovery

poc/public/media.css

@@ -802,3 +802,40 @@
     flex-direction: column;
     gap: 4px;
 }
+
+.vk-bot-progress {
+    display: flex;
+}
+
+.vk-bot-progress-cached {
+    background: var(--color-accent-400, #4f8cff);
+}
+
+.vk-bot-progress-pending {
+    background: var(--color-neutral-700);
+}
+
+.vk-bot-progress-failed {
+    background: #c04040;
+}
+
+.vk-bot-history {
+    margin-top: 8px;
+}
+
+.vk-bot-history-title {
+    font: 600 11px/1.6 ui-monospace, Menlo, monospace;
+    color: var(--color-neutral-300);
+    margin-bottom: 2px;
+}
+
+.vk-bot-history-entry {
+    font: 400 10px/1.5 ui-monospace, Menlo, monospace;
+    color: var(--color-neutral-500);
+}
+
+.vk-bot-hint {
+    margin-top: 8px;
+    font: 400 10px/1.5 ui-monospace, Menlo, monospace;
+    color: var(--color-neutral-500);
+}

poc/public/media.js

@@ -41,6 +41,7 @@
         pageSize: 60,
         backfill: null,
         titles: null,
+        recover: null,
         preferences: null,
         /** Scrapes that finished while this page was open, until the tab is reloaded. */
         completed: new Set()
@@ -77,7 +78,16 @@
                     </div>
                     <div id="backfill-summary" class="vk-bot-summary"></div>
                     <div id="title-lane" class="vk-bot-lane"></div>
+                    <div id="recover-lane" class="vk-bot-lane"></div>
                     <div id="backfill-jobs" class="vk-bot-jobs"></div>
+                    <div id="task-history" class="vk-bot-history"></div>
+                    <div class="vk-bot-hint">
+                        Failure policy: a failed preview gets one delayed retry (about 50 s). If a
+                        batch is two-thirds suspect failures, the cooldown doubles (60 s floor) and
+                        the retries run before the next batch. Rows that still fail are left
+                        unverified and skipped by later passes; the recovery task above retries
+                        Discord-CDN ones on demand.
+                    </div>
                 </div>
 
                 <div class="vk-bot-section">
@@ -104,6 +114,8 @@
     const backfillSummary = overlay.querySelector("#backfill-summary");
     const backfillJobs = overlay.querySelector("#backfill-jobs");
     const titleLane = overlay.querySelector("#title-lane");
+    const recoverLane = overlay.querySelector("#recover-lane");
+    const taskHistory = overlay.querySelector("#task-history");
     const tunablesHost = overlay.querySelector("#bot-tunables");
     const botErrors = overlay.querySelector("#bot-errors");
 
@@ -793,15 +805,24 @@
                         "vk-bot-job-member",
                         `${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` +
                             `${member.backend} \u00b7 ${member.pending} pending` +
+                            (member.suspect ? ` + ${member.suspect} suspect` : "") +
+                            (member.unverified ? ` + ${member.unverified} unverified` : "") +
                             (kinds ? ` (${kinds})` : "")
                     )
                 );
             }
 
+            const total = Math.max(1, job.total);
             const progress = el("div", "vk-bot-progress");
-            const fill = el("div", "vk-bot-progress-fill");
-            fill.style.width = `${Math.min(100, Math.round((job.cached / Math.max(1, job.total)) * 100))}%`;
-            progress.append(fill);
+            for (const [className, value] of [
+                ["vk-bot-progress-cached", job.cached],
+                ["vk-bot-progress-pending", job.pending + (job.suspect ?? 0)],
+                ["vk-bot-progress-failed", job.failed]
+            ]) {
+                const segment = el("div", className);
+                segment.style.width = `${Math.min(100, (value / total) * 100)}%`;
+                progress.append(segment);
+            }
             row.append(progress);
 
             const actions = el("div", "vk-bot-job-actions");
@@ -879,6 +900,76 @@
         renderTitleLane(state.titles);
     }
 
+    function renderRecover(status) {
+        if (!status) return;
+        recoverLane.replaceChildren();
+
+        const head = el("div", "vk-bot-job-head");
+        head.append(el("span", "vk-bot-job-title", "Recover Discord previews"));
+        head.append(el("span", `vk-bot-state vk-bot-state-${status.state}`, status.state));
+
+        const parts = [`${status.total} to retry`];
+        if (status.processed) parts.push(`${status.processed} processed`);
+        if (status.recovered) parts.push(`${status.recovered} recovered`);
+        if (status.gone) parts.push(`${status.gone} unavailable`);
+        if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
+        head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
+        recoverLane.append(head);
+
+        const actions = el("div", "vk-bot-job-actions");
+        if (status.state === "running") {
+            actions.append(button("Pause", () => void recoverAction("pause")));
+        } else if (status.total > 0) {
+            actions.append(button("Start", () => void recoverAction("start")));
+        }
+        recoverLane.append(actions);
+        recoverLane.append(
+            el(
+                "div",
+                "vk-bot-job-member",
+                "Re-fetches previews for Discord CDN rows closed as gone, refreshing each signed " +
+                    "URL through the source message (bot or puppet). Runs once; the result lands in " +
+                    "the history below."
+            )
+        );
+
+        renderTaskHistory(status.history ?? []);
+    }
+
+    function renderTaskHistory(history) {
+        taskHistory.replaceChildren();
+        if (!history.length) return;
+
+        taskHistory.append(el("div", "vk-bot-history-title", "Task history"));
+        for (const entry of history) {
+            taskHistory.append(
+                el(
+                    "div",
+                    "vk-bot-history-entry",
+                    `${new Date(entry.at).toLocaleString()} \u00b7 ${entry.name} \u00b7 ${entry.detail}`
+                )
+            );
+        }
+    }
+
+    async function recoverAction(action) {
+        try {
+            state.recover = await window.VK.api.post(`/api/media/recover/${action}`, {});
+            renderRecover(state.recover);
+        } catch (error) {
+            setStatus(`recover ${action} failed: ${error.message}`);
+        }
+    }
+
+    async function refreshRecover() {
+        try {
+            state.recover = await window.VK.api.get("/api/media/recover");
+        } catch {
+            state.recover = null;
+        }
+        renderRecover(state.recover);
+    }
+
     async function refreshBackfill() {
         try {
             state.backfill = await window.VK.api.get("/api/media/backfill");
@@ -1043,7 +1134,10 @@
         ["date", "Date"],
         ["source", "Source"],
         ["resolution", "Resolution"],
-        ["length", "Length"]
+        ["length", "Length"],
+        ["channel", "Channel"],
+        ["scrape", "Scrape"],
+        ["title", "Title"]
     ];
 
     /** "mp4 video", "webp image", "YouTube video", ... */
@@ -1062,7 +1156,14 @@
             date: item.postedAt ? new Date(item.postedAt).toLocaleString() : "—",
             source: item.provider ?? "discord",
             resolution: item.width && item.height ? `${item.width}x${item.height}` : "—",
-            length: item.durationSecs ? `${item.durationSecs.toFixed(1)}s` : "—"
+            length: item.durationSecs ? `${item.durationSecs.toFixed(1)}s` : "—",
+            channel: channelLabel(item.channelId),
+            scrape: item.scrapeId,
+            title: item.title
+                ? item.titleAuthor
+                    ? `${item.title} — ${item.titleAuthor}`
+                    : item.title
+                : ""
         };
     }
 
@@ -1819,6 +1920,7 @@
             if (!state.preferences) void refreshPreferences();
             void refreshBackfill();
             void refreshTitles();
+            void refreshRecover();
             void loadScrapes().then(renderScrapes);
             void refreshErrors();
         } else {
@@ -1839,6 +1941,7 @@
         if (state.view === "bot") {
             void refreshBackfill();
             void refreshTitles();
+            void refreshRecover();
         }
     });
 
@@ -1872,6 +1975,7 @@
         if (state.view === "bot") {
             void refreshBackfill();
             void refreshTitles();
+            void refreshRecover();
         }
     }, 3000);
 
@@ -2290,6 +2394,22 @@
 
     // ---- Media viewer ----
 
+    /** The canonical watch URL for YouTube items, for the fallback button. */
+    function youtubeWatchUrl(item) {
+        try {
+            const url = new URL(item.url);
+            const host = url.hostname.replace(/^www\./, "");
+            if (host === "youtube.com" || host === "m.youtube.com") {
+                const id = url.searchParams.get("v") ?? /^\/shorts\/([A-Za-z0-9_-]+)/.exec(url.pathname)?.[1];
+                return id ? `https://www.youtube.com/watch?v=${id}` : null;
+            }
+            if (host === "youtu.be") return `https://www.youtube.com/watch${url.pathname}`;
+        } catch {
+            // not a URL
+        }
+        return null;
+    }
+
     function embedUrlFor(item) {
         if (item.source !== "embed") return null;
 
@@ -2396,26 +2516,38 @@
         meta.replaceChildren();
         const values = metaValues(item);
         for (const [key, label] of META_FIELDS) {
+            if (!values[key]) continue;
             const line = el("div", "vk-media-meta-line");
             line.append(el("span", "vk-media-meta-key", `${label}: `));
             line.append(el("span", null, values[key]));
             meta.append(line);
         }
 
-        const link = el("button", "vk-media-link", item.url);
+        // The permanent message link, not the expiring CDN URL.
+        const linkUrl = item.messageUrl ?? item.url;
+        const link = el("button", "vk-media-link", linkUrl);
         link.type = "button";
-        link.title = "Open the original link";
-        link.addEventListener("click", () => openLinkWarning(item.url));
+        link.title = item.messageUrl ? "Open the message" : "Open the original link";
+        link.addEventListener("click", () => openLinkWarning(linkUrl));
         meta.append(link);
 
-        // "View full" opens the embedded media (player, local file, or a refreshed stream).
+        // "View full" plays in a new tab: the stream endpoint for Discord media (always a fresh
+        // source), the provider player for embeddable items.
         const streamPath = `/api/media/${item.scrapeId}/${item.itemId}/stream`;
         const fullUrl =
-            embedUrlFor(item) ??
-            item.localUrl ??
-            (item.kind === "video" ? streamPath : item.playUrl ?? item.thumbUrl ?? streamPath);
+            item.messageUrl && item.kind === "video"
+                ? streamPath
+                : embedUrlFor(item) ??
+                  item.localUrl ??
+                  (item.kind === "video" ? streamPath : item.playUrl ?? item.thumbUrl ?? streamPath);
         meta.append(button("View full", () => window.open(fullUrl, "_blank", "noreferrer")));
 
+        // Embeds break for age-gated or embed-disabled videos: keep a direct provider link.
+        const watchUrl = youtubeWatchUrl(item);
+        if (watchUrl) {
+            meta.append(button("Open on YouTube", () => window.open(watchUrl, "_blank", "noreferrer")));
+        }
+
         viewerModal.hidden = false;
     }
 
@@ -2437,6 +2569,7 @@
             <div id="link-target" class="vk-media-status"></div>
             <div class="vk-media-status">This opens a new website in a new tab.</div>
             <div class="vk-media-toolbar">
+                <button id="link-copy" class="btn btn-secondary vk-small-btn" type="button">Copy link</button>
                 <button id="link-cancel" class="btn btn-secondary vk-small-btn" type="button">Cancel</button>
                 <button id="link-continue" class="btn btn-primary vk-small-btn" type="button">Continue</button>
             </div>
@@ -2448,12 +2581,29 @@
 
     function openLinkWarning(url) {
         pendingLink = url;
+        linkModal.querySelector("#link-copy").textContent = "Copy link";
         linkModal.querySelector("#link-target").textContent = url;
         linkModal.hidden = false;
     }
 
+    linkModal.querySelector("#link-copy").addEventListener("click", async () => {
+        if (!pendingLink) return;
+        try {
+            await navigator.clipboard.writeText(pendingLink);
+        } catch {
+            const area = document.createElement("textarea");
+            area.value = pendingLink;
+            document.body.append(area);
+            area.select();
+            document.execCommand("copy");
+            area.remove();
+        }
+        linkModal.querySelector("#link-copy").textContent = "Copied";
+    });
+
     linkModal.querySelector("#link-cancel").addEventListener("click", () => {
         linkModal.hidden = true;
+        linkModal.querySelector("#link-copy").textContent = "Copy link";
     });
     linkModal.querySelector("#link-continue").addEventListener("click", () => {
         if (pendingLink) window.open(pendingLink, "_blank", "noreferrer");

poc/src/media.ts

@@ -10,6 +10,7 @@ import { isAbsolute, join } from "node:path";
 import {
     BackfillQueue,
     betterCandidate,
+    messageUrlFor,
     sortCandidates,
     dirSizeBytes,
     isNearLimit,
@@ -115,6 +116,10 @@ export interface MediaItemResult {
     scrapeId: string;
     itemId: number;
     channelId: string;
+    guildId: string | null;
+    /** The message that carried the media; the permanent link target. */
+    messageId: string | null;
+    messageUrl: string | null;
     kind: MediaKind;
     source: string | null;
     authorId: string | null;
@@ -139,6 +144,8 @@ export interface MediaItemResult {
     downloaded: boolean;
     /** Embed title (YouTube and friends), when Discord provided one. */
     title: string | null;
+    /** YouTube channel name kept from oEmbed, even if the video later disappears. */
+    titleAuthor: string | null;
     createdAt: string;
 }
 
@@ -182,6 +189,19 @@ export class MediaService {
     /** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */
     private readonly errors: MediaErrorEntry[] = [];
 
+    /** One-shot maintenance: re-fetch previews for Discord rows closed out with HTTP errors. */
+    private readonly recover = {
+        state: "idle" as "idle" | "running" | "paused" | "done",
+        processed: 0,
+        recovered: 0,
+        gone: 0,
+        startedAt: null as number | null,
+        finishedAt: null as number | null,
+        currentScrapeId: null as string | null
+    };
+    /** Finished maintenance tasks, newest first. */
+    private readonly taskHistory: Array<{ name: string; at: string; detail: string }> = [];
+
     constructor(private readonly deps: MediaServiceDeps) {
         const tunables = this.deps.getMediaPrefs().tunables;
 
@@ -195,6 +215,14 @@ export class MediaService {
             pageMaxMs: tunables.timers.pageMaxMs,
             maxRetries: tunables.scrape.maxRetries,
             previewMaxAttempts: tunables.backfill.maxAttempts,
+            suspectRetryMs: 50_000,
+            refreshUrl: async (scrapeId, media) => {
+                const meta = this.engine.get(scrapeId);
+                if (!meta) return null;
+                const item = this.getMediaItem(scrapeId, media.id);
+                if (!item) return null;
+                return this.refreshUrls(meta, item);
+            },
             download: {
                 maxFileBytes: tunables.scrape.maxFileBytes,
                 previewMaxBytes: tunables.backfill.maxBytes,
@@ -293,6 +321,145 @@ export class MediaService {
         return this.titles.status();
     }
 
+    /** The recovery task with its live remaining count and the task history. */
+    recoverStatus(): {
+        state: string;
+        total: number;
+        processed: number;
+        recovered: number;
+        gone: number;
+        startedAt: string | null;
+        finishedAt: string | null;
+        currentScrapeId: string | null;
+        history: Array<{ name: string; at: string; detail: string }>;
+    } {
+        let total = 0;
+        for (const meta of this.engine.list()) {
+            if (meta.state === "deleted" || meta.state === "deleting") continue;
+            try {
+                const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
+                try {
+                    total += store.discordPreviewFailureCount();
+                } finally {
+                    store.close();
+                }
+            } catch {
+                // an unreadable index has nothing to recover
+            }
+        }
+
+        return {
+            state: this.recover.state,
+            total,
+            processed: this.recover.processed,
+            recovered: this.recover.recovered,
+            gone: this.recover.gone,
+            startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null,
+            finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null,
+            currentScrapeId: this.recover.currentScrapeId,
+            history: [...this.taskHistory]
+        };
+    }
+
+    startRecover(): void {
+        if (this.recover.state === "running") return;
+        this.recover.state = "running";
+        this.recover.startedAt = this.recover.startedAt ?? Date.now();
+        this.recover.processed = 0;
+        this.recover.recovered = 0;
+        this.recover.gone = 0;
+        this.deps.log?.("recover: retrying Discord-CDN previews marked gone");
+        void this.runRecover();
+    }
+
+    pauseRecover(): void {
+        if (this.recover.state !== "running") return;
+        this.recover.state = "paused";
+        this.deps.log?.("recover: pausing");
+    }
+
+    /**
+     * Walk every scrape's Discord-CDN preview failures, refresh the signed URL from the source
+     * message (slopbot REST for guild scrapes, the puppet for DMs and group chats), then let the
+     * engine fetch the preview. Rows whose URL cannot be refreshed are counted as gone.
+     */
+    private async runRecover(): Promise<void> {
+        const scrapeIds = this.engine
+            .list()
+            .filter(meta => meta.state !== "deleted" && meta.state !== "deleting")
+            .map(meta => meta.id);
+
+        while (this.recover.state === "running") {
+            let acted = false;
+
+            for (const scrapeId of scrapeIds) {
+                if (this.recover.state !== "running") break;
+
+                const meta = this.engine.get(scrapeId);
+                if (!meta) continue;
+
+                let store: ScrapeStore | null = null;
+                try {
+                    store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
+                } catch {
+                    continue;
+                }
+
+                try {
+                    const rows = store.discordPreviewFailures(10);
+                    if (!rows.length) continue;
+
+                    acted = true;
+                    this.recover.currentScrapeId = scrapeId;
+                    const cleared: MediaRow[] = [];
+
+                    for (const row of rows) {
+                        if (this.recover.state !== "running") break;
+                        const fresh = await this.refreshUrls(meta, {
+                            ...row,
+                            scrapeId,
+                            channelId: meta.channelId
+                        } as MediaRow & { scrapeId: string; channelId: string });
+                        if (fresh) {
+                            store.clearPreviewError(row.id);
+                            cleared.push(row);
+                        } else {
+                            this.recover.gone++;
+                        }
+                        this.recover.processed++;
+                    }
+
+                    if (cleared.length && this.recover.state === "running") {
+                        await this.engine.previewBatch(scrapeId, cleared.length);
+                        for (const row of cleared) {
+                            if (store.getMedia(row.id)?.preview_hash) this.recover.recovered++;
+                            else this.recover.gone++;
+                        }
+                    }
+                } finally {
+                    store?.close();
+                }
+
+                if (acted) break;
+            }
+
+            if (!acted) break;
+        }
+
+        if (this.recover.state === "running") this.recover.state = "done";
+        this.recover.finishedAt = Date.now();
+        this.recover.currentScrapeId = null;
+        this.taskHistory.unshift({
+            name: "Recover Discord previews",
+            at: new Date().toISOString(),
+            detail: `${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
+        });
+        if (this.taskHistory.length > 10) this.taskHistory.length = 10;
+        this.deps.log?.(
+            `recover: done, ${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
+        );
+    }
+
     startBackfill(chatId?: string): BackfillStatus {
         this.backfill.start(chatId);
         return this.backfillStatus();
@@ -1006,6 +1173,9 @@ export class MediaService {
                 scrapeId: meta.id,
                 itemId: row.id,
                 channelId: meta.channelId,
+                guildId: meta.guildId,
+                messageId: row.message_id,
+                messageUrl: messageUrlFor(meta.guildId, meta.channelId, row.message_id),
                 kind: row.kind,
                 source: row.source,
                 authorId: row.author_id,
@@ -1029,6 +1199,7 @@ export class MediaService {
                 provider: row.provider,
                 downloaded: row.downloaded === 1,
                 title: row.title,
+                titleAuthor: row.title_author,
                 createdAt: row.created_at
             }));
     }

poc/src/webui.ts

@@ -842,6 +842,29 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         })
     );
 
+    app.get(
+        "/api/media/recover",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: deps.media.recoverStatus() });
+        })
+    );
+
+    app.post(
+        "/api/media/recover/start",
+        wrap(async (_req, res) => {
+            deps.media.startRecover();
+            res.json({ ok: true, result: deps.media.recoverStatus() });
+        })
+    );
+
+    app.post(
+        "/api/media/recover/pause",
+        wrap(async (_req, res) => {
+            deps.media.pauseRecover();
+            res.json({ ok: true, result: deps.media.recoverStatus() });
+        })
+    );
+
     app.post(
         "/api/media/titles/start",
         wrap(async (_req, res) => {

poc/test/media-ui-smoke.mjs

@@ -206,6 +206,17 @@ try {
         () => document.querySelector("#title-lane")?.textContent?.includes("Titles (YouTube)"),
         { timeout: 15000 }
     );
+    check(
+        "recovery card renders",
+        (await page.locator("#recover-lane").textContent())?.includes("Recover Discord previews") === true
+    );
+    const withMessage = await page.evaluate(async () => {
+        const response = await fetch("/api/media?kind=video&limit=5");
+        const body = await response.json();
+        return (body.result ?? []).filter(item => item.messageUrl).length;
+    });
+    step("items with a message link", withMessage);
+    check("message links present", withMessage > 0, withMessage);
     check("title lane card renders", (await page.locator("#title-lane .vk-bot-job-head").count()) === 1);
 
     const summaryLines = await page.locator(".vk-bot-summary-line").count();