Stream refreshed media for playback and view full

CURRENT.md

@@ -40,6 +40,13 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C21 Playback of non-downloaded and expired media. `play_url` stores the embed's direct
+    video (gifv/Tenor mp4); a new `/api/media/<scrape>/<item>/stream` route serves the local file
+    or re-fetches the source message through slopbot to refresh expired signed CDN URLs and streams
+    it with Range support. The viewer and **View full** use it, so Discord attachment videos play
+    without a prior download and old links no longer dead-end. Live: 2.5 MB webm streamed 200 with
+    a refreshed URL; play_url populated for a media.discordapp.net gifv mp4 and provider
+    redirectors.
   - [x] C20 Cross-scrape dedupe and viewer polish. `queryMedia` and `stats` deduplicate by URL
     (newest scrape wins), so overlapping scrapes no longer duplicate cards or counters (live: 17
     items / 17 unique). Viewer: big play icon top-right; play embeds a provider player or a direct

docs/media.md

@@ -154,10 +154,12 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun
   shows the metadata banner below, vertical media shows it to the right, and videos carry a large
   play icon top-right. The play action embeds a YouTube, Vimeo, Twitch, or Streamable player, or
   plays a direct mp4/webm/m4v; when neither is possible it opens the external-site warning, since
-  the page is the only way to watch. **View full** opens the embedded media (player, local file,
+  the page is the only way to watch. Playback and **View full** go through
-  or cached preview) in a new tab; the full link is a separate button that opens the original
+  `/api/media/<scrape>/<item>/stream`, which serves the local file or re-fetches the source message
-  page with the warning. Length comes from Discord's attachment duration and, for downloads, an
+  for a fresh signed URL, so expired CDN links still play. Embed `videoUrl`s (gifv/Tenor mp4s) are
-  ffprobe probe. The status row reads
+  stored as `play_url` and used directly. The full link is a separate button that opens the
+  original page with the warning. Length comes from Discord's attachment duration and, for
+  downloads, an ffprobe probe. The status row reads
   `(Images/Videos) | Downloaded A/B | Cached C/B (excl Downl) | Loaded L/B`, where B is the total
   for the active filters. A scrape strip shows live progress with Stop.
   **New scrape…** (rightmost) opens a popup with Server and Group chat trees (servers expand to

docs/webui.md

@@ -260,7 +260,8 @@ replaceable.
 | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments |
 | GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`; `localUrl`/`thumbUrl` when a file or preview exists |
 | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension |
-| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview |
+| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) |
+| GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range` |
 | GET | `/api/media/scrapes` | none | Every scrape with state and counters |
 | GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
 | GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts |

media-scrape/src/extract.ts

@@ -21,6 +21,8 @@ export interface ExtractedMedia {
     kind: MediaKind;
     url: string;
     source: MediaSource;
+    /** Direct playable file URL when the embed carries one (gifv/Tenor mp4). */
+    playUrl: string | null;
     provider: string | null;
     contentType: string | null;
     size: number | null;
@@ -73,6 +75,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
     const provider = embed.provider ?? hostname(embed.url);
     const common = {
         source: "embed" as const,
+        playUrl: embed.videoUrl ?? null,
         provider,
         downloaded: false,
         size: null,
@@ -123,6 +126,7 @@ function extractSticker(sticker: CoreSticker): ExtractedMedia | null {
     return {
         kind: ext === "gif" ? "gif" : "image",
         source: "sticker",
+        playUrl: null,
         url: `https://cdn.discordapp.com/stickers/${sticker.id}.${ext}`,
         provider: "discord",
         contentType: null,
@@ -145,6 +149,7 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] {
         items.push({
             kind,
             source: "attachment",
+            playUrl: null,
             url: attachment.url,
             provider: "discord",
             contentType: attachment.contentType,

media-scrape/src/store.ts

@@ -122,6 +122,7 @@ export interface MediaRow {
     author_name: string | null;
     posted_at: string | null;
     url: string;
+    play_url: string | null;
     provider: string | null;
     content_type: string | null;
     size: number | null;
@@ -159,6 +160,7 @@ CREATE TABLE IF NOT EXISTS media (
     author_name TEXT,
     posted_at TEXT,
     url TEXT NOT NULL UNIQUE,
+    play_url TEXT,
     provider TEXT,
     content_type TEXT,
     size INTEGER,
@@ -215,6 +217,7 @@ export class ScrapeStore {
         if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
         if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
         if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT");
+        if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT");
         if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
     }
 
@@ -229,7 +232,7 @@ export class ScrapeStore {
             "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
         );
         const insertMedia = this.db.prepare(
-            "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+            "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
         );
         const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
         const linkSource = this.db.prepare(
@@ -256,6 +259,7 @@ export class ScrapeStore {
                         message.author.username,
                         message.timestamp,
                         item.url,
+                        item.playUrl,
                         item.provider,
                         item.contentType,
                         item.size,
@@ -362,6 +366,32 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }
 
+    /** Refresh stored URLs after an expired CDN link is re-fetched. */
+    updateUrls(id: number, urls: { url?: string; playUrl?: string | null }): void {
+        const sets: string[] = [];
+        const values: Array<string | null> = [];
+
+        if (urls.url) {
+            sets.push("url = ?");
+            values.push(urls.url);
+        }
+        if (urls.playUrl !== undefined) {
+            sets.push("play_url = ?");
+            values.push(urls.playUrl);
+        }
+
+        if (!sets.length) return;
+        this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
+    }
+
+    /** The message a media row came from, for URL refreshes. */
+    sourceMessageId(mediaId: number): string | null {
+        const row = this.db
+            .prepare("SELECT message_id FROM media_sources WHERE media_id = ? LIMIT 1")
+            .get(mediaId) as unknown as { message_id: string } | undefined;
+        return row?.message_id ?? null;
+    }
+
     /** Fill in dimensions and duration after a download probe. */
     updateMediaInfo(
         id: number,

media-scrape/test/run.ts

@@ -139,6 +139,7 @@ void (async () => {
         assert.equal(youtube?.kind, "video");
         assert.equal(youtube?.provider, "YouTube");
         assert.equal(youtube?.downloaded, false);
+        assert.equal(youtube?.playUrl, "https://www.youtube.com/embed/x");
         assert.equal(items.find(item => item.provider === "Tenor")?.kind, "gif");
         assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
         assert.equal(items.some(item => item.url.includes("lottie")), false);

poc/public/media.js

@@ -1080,13 +1080,15 @@
         body.className = `vk-media-viewer ${vertical ? "vertical" : "wide"}`;
         media.replaceChildren();
 
-        const source = item.localUrl || item.thumbUrl || item.url;
+        // The stream endpoint serves the local file or a refreshed remote URL, so signed
-        const direct = item.localUrl || (item.source !== "embed" ? item.url : null);
+        // CDN links that expired no longer break playback.
-        const playableFile = !!direct && /\.(mp4|webm|m4v)(\?|$)/i.test(direct);
+        const streamUrl = `/api/media/${item.scrapeId}/${item.itemId}/stream`;
+        const source = item.localUrl || item.thumbUrl || streamUrl;
         const embed = embedUrlFor(item);
-        const canPlay = !!embed || playableFile;
+        const directVideo = item.localUrl || item.playUrl || (item.source !== "embed" ? streamUrl : null);
+        const canPlay = !!embed || (item.kind === "video" && !!directVideo);
 
-        // A play request on something we cannot embed falls back to the external page.
+        // A play request on something we cannot play falls back to the external page.
         if (play && !canPlay) openLinkWarning(item.url);
 
         if (play && embed) {
@@ -1095,11 +1097,25 @@
             frame.allow = "autoplay; fullscreen; picture-in-picture";
             frame.allowFullscreen = true;
             media.append(frame);
-        } else if (play && playableFile && direct) {
+        } else if (play && directVideo) {
             const video = document.createElement("video");
             video.controls = true;
             video.autoplay = true;
-            video.src = direct;
+            video.src = directVideo;
+            video.addEventListener(
+                "error",
+                () => {
+                    media.replaceChildren(
+                        el("div", "vk-media-status", "This format is not supported in the browser.")
+                    );
+                    media.append(
+                        button("Open in new tab", () =>
+                            window.open(directVideo, "_blank", "noreferrer")
+                        )
+                    );
+                },
+                { once: true }
+            );
             media.append(video);
         } else if (/^(https?:|\/)/.test(source)) {
             const img = document.createElement("img");
@@ -1127,8 +1143,12 @@
         link.addEventListener("click", () => openLinkWarning(item.url));
         meta.append(link);
 
-        // "View full" opens the embedded media (player, local file, or preview) in a new tab.
+        // "View full" opens the embedded media (player, local file, or a refreshed stream).
-        const fullUrl = embedUrlFor(item) ?? item.localUrl ?? item.thumbUrl ?? item.url;
+        const streamPath = `/api/media/${item.scrapeId}/${item.itemId}/stream`;
+        const fullUrl =
+            embedUrlFor(item) ??
+            item.localUrl ??
+            (item.kind === "video" ? streamPath : item.playUrl ?? item.thumbUrl ?? streamPath);
         meta.append(button("View full", () => window.open(fullUrl, "_blank", "noreferrer")));
 
         viewerModal.hidden = false;

poc/src/media.ts

@@ -117,6 +117,7 @@ export interface MediaItemResult {
     contentType: string | null;
     durationSecs: number | null;
     url: string;
+    playUrl: string | null;
     localUrl: string | null;
     thumbUrl: string | null;
     provider: string | null;
@@ -373,6 +374,89 @@ export class MediaService {
         };
     }
 
+    /**
+     * A servable media source: the local file, or a remote URL refreshed from the source
+     * message (signed CDN links expire). Null when nothing can be resolved.
+     */
+    async resolveMediaSource(
+        scrapeId: string,
+        itemId: number
+    ): Promise<
+        | { kind: "file"; path: string; contentType: string }
+        | { kind: "remote"; url: string }
+        | null
+    > {
+        const file = this.getMediaFile(scrapeId, itemId);
+        if (file) return { kind: "file", ...file };
+
+        const meta = this.engine.get(scrapeId);
+        const item = this.getMediaItem(scrapeId, itemId);
+        if (!meta || !item) return null;
+
+        if (meta.backend === "slopbot" && meta.guildId && this.bot) {
+            const refreshed = await this.refreshUrls(meta.id, meta.channelId, item);
+            if (refreshed) return { kind: "remote", url: refreshed };
+        }
+
+        return { kind: "remote", url: item.play_url ?? item.url };
+    }
+
+    /** Re-fetch the source message and update expired URLs. Returns the fresh playable URL. */
+    private async refreshUrls(
+        scrapeId: string,
+        channelId: string,
+        item: MediaRow
+    ): Promise<string | null> {
+        try {
+            const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
+            let messageId: string | null = null;
+            try {
+                messageId = store.sourceMessageId(item.id);
+            } finally {
+                store.close();
+            }
+            if (!messageId || !this.bot) return null;
+
+            const channel = await this.bot.channels.fetch(channelId);
+            if (!channel?.isTextBased()) return null;
+            const message = await channel.messages.fetch(messageId);
+
+            const path = (url: string) => url.split("?")[0];
+            const attachment = [...message.attachments.values()].find(
+                candidate => path(candidate.url) === path(item.url)
+            );
+            if (attachment) {
+                const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
+                try {
+                    update.updateUrls(item.id, { url: attachment.url });
+                } finally {
+                    update.close();
+                }
+                return attachment.url;
+            }
+
+            const embed = message.embeds.find(
+                candidate =>
+                    (item.play_url && candidate.video?.url && path(candidate.video.url) === path(item.play_url)) ||
+                    (candidate.video?.url && path(candidate.url ?? "") === path(item.url))
+            );
+            if (embed?.video?.url) {
+                const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
+                try {
+                    update.updateUrls(item.id, { playUrl: embed.video.url, url: embed.url ?? item.url });
+                } finally {
+                    update.close();
+                }
+                return embed.video.url;
+            }
+
+            return null;
+        } catch (error) {
+            this.deps.log?.(`refresh failed for ${scrapeId}:${item.id}: ${String(error)}`);
+            return null;
+        }
+    }
+
     /** Absolute path and content type for a downloaded file, or null. */
     getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
         // A custom destination is the user's filesystem, outside the archive: not served.
@@ -609,6 +693,7 @@ export class MediaService {
                             contentType: row.content_type,
                             durationSecs: row.duration_secs,
                             url: row.url,
+                            playUrl: row.play_url,
                             localUrl:
                                 row.file_path && !meta.downloadDir
                                     ? `/api/media/${meta.id}/${row.id}/file`

poc/src/webui.ts

@@ -8,7 +8,8 @@
  */
 
 import express, { type Request, type Response } from "express";
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
+import { Readable } from "node:stream";
 import http from "node:http";
 import https from "node:https";
 import { dirname } from "node:path";
@@ -758,7 +759,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         "/api/media/:scrapeId/:itemId/thumb",
         wrap(async (req, res) => {
             const file = deps.media.getMediaThumb(String(req.params.scrapeId), Number(req.params.itemId));
-            if (!file) throw new HttpError(404, "thumb not found");
+            if (!file || !existsSync(file.path)) throw new HttpError(404, "thumb not found");
             res.type(file.contentType).sendFile(file.path);
         })
     );
@@ -767,11 +768,48 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         "/api/media/:scrapeId/:itemId/file",
         wrap(async (req, res) => {
             const file = deps.media.getMediaFile(String(req.params.scrapeId), Number(req.params.itemId));
-            if (!file) throw new HttpError(404, "file not found");
+            if (!file || !existsSync(file.path)) throw new HttpError(404, "file not found");
             res.type(file.contentType).sendFile(file.path);
         })
     );
 
+    // Playback and full-view source: local file when downloaded, otherwise a refreshed remote URL.
+    app.get(
+        "/api/media/:scrapeId/:itemId/stream",
+        wrap(async (req, res) => {
+            const source = await deps.media.resolveMediaSource(
+                String(req.params.scrapeId),
+                Number(req.params.itemId)
+            );
+            if (!source) throw new HttpError(404, "media source not found");
+
+            if (source.kind === "file") {
+                if (!existsSync(source.path)) throw new HttpError(404, "file not found");
+                res.type(source.contentType).sendFile(source.path);
+                return;
+            }
+
+            const upstream = await fetch(source.url, {
+                headers: req.headers.range ? { range: String(req.headers.range) } : {}
+            });
+            if (!upstream.ok && upstream.status !== 206) {
+                throw new HttpError(502, `upstream ${upstream.status}`);
+            }
+
+            res.status(upstream.status);
+            for (const header of ["content-type", "content-length", "accept-ranges", "content-range"]) {
+                const value = upstream.headers.get(header);
+                if (value) res.setHeader(header, value);
+            }
+
+            if (!upstream.body) {
+                res.end();
+                return;
+            }
+            Readable.fromWeb(upstream.body as any).pipe(res);
+        })
+    );
+
     app.get(
         "/api/media/storage",
         wrap(async (_req, res) => {