Refuse non-media streams and gate video framing

This commit is contained in:
Fisto
2026-09-16 14:04:57 +02:00
parent c100bc269c
commit d522681553
5 changed files with 28 additions and 6 deletions
+6
View File
@@ -40,6 +40,12 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200). `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C27 Non-media stream guard. Rows awaiting backfill fell back to first-frame `<video>` with
a provider page URL (YouTube embed, redirectors), and the stream route piped the HTML through
(`text/html` errors). Now only real video files (mp4/webm/m4v) are framed, the stream route
returns `415` for non-media upstreams, the viewer's still-frame is gated the same way, and
"Open in new tab" prefers `play_url`. Live: YouTube item → 415 JSON; backfill reached
184/271 previews during the run.
- [x] C26 YouTube preview root cause. Dedupe picked the newest row per URL; a paused scrape - [x] C26 YouTube preview root cause. Dedupe picked the newest row per URL; a paused scrape
(`gernal_chut`, stopped mid-walk) had no previews yet and hid an older scrape that did. Fixes: (`gernal_chut`, stopped mid-walk) had no previews yet and hid an older scrape that did. Fixes:
dedupe prefers the richer row (downloaded > preview > newer), paused walks still run the dedupe prefers the richer row (downloaded > preview > newer), paused walks still run the
+3 -1
View File
@@ -158,7 +158,9 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun
` /api/media/<scrape>/<item>/stream`, which serves the local file or re-fetches the source message ` /api/media/<scrape>/<item>/stream`, which serves the local file or re-fetches the source message
for a fresh signed URL, so expired CDN links still play. The grid prefers the cached still; a for a fresh signed URL, so expired CDN links still play. The grid prefers the cached still; a
video without one gets its first frame through the same-origin stream endpoint, which avoids video without one gets its first frame through the same-origin stream endpoint, which avoids
cross-origin blocking and URL expiry. Embed `videoUrl`s (gifv/Tenor mp4s) are stored as cross-origin blocking and URL expiry. Only real video-file URLs are framed; provider pages
(YouTube embeds, redirector URLs) are never, and the stream route answers `415` when the
upstream is not media — so a pending preview shows a placeholder instead of a browser error. Embed `videoUrl`s (gifv/Tenor mp4s) are stored as
`play_url` and used through the stream. The full link is a separate button that opens the `play_url` and used through the stream. The full link is a separate button that opens the
original page with the warning. Length comes from Discord's attachment duration and, for original page with the warning. Length comes from Discord's attachment duration and, for
downloads, an ffprobe probe. GIF tiles carry a translucent **GIF** label top-left; the cached downloads, an ffprobe probe. GIF tiles carry a translucent **GIF** label top-left; the cached
+1 -1
View File
@@ -261,7 +261,7 @@ replaceable.
| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`; `localUrl`/`thumbUrl` when a file or preview exists | | GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`; `localUrl`/`thumbUrl` when a file or preview exists |
| GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension |
| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) | | GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) |
| GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range` | | GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range`. Non-media upstream responses (provider pages) return `415` |
| GET | `/api/media/scrapes` | none | Every scrape with state and counters | | GET | `/api/media/scrapes` | none | Every scrape with state and counters |
| GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
| GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts | | GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts |
+11 -4
View File
@@ -424,6 +424,13 @@
media.replaceChildren(el("span", "vk-media-card-kind", item.kind.toUpperCase())); media.replaceChildren(el("span", "vk-media-card-kind", item.kind.toUpperCase()));
} }
/** Only real video files can be framed; embed pages (YouTube etc.) never are. */
function canFrameVideo(item) {
const direct = /\.(mp4|webm|m4v)(\?|$)/i;
if (item.playUrl && direct.test(item.playUrl)) return true;
return item.source !== "embed" && direct.test(item.url ?? "");
}
function playIcon(item) { function playIcon(item) {
const play = el("span", "vk-media-play", "▶"); const play = el("span", "vk-media-play", "▶");
play.title = "Play"; play.title = "Play";
@@ -470,7 +477,7 @@
} }
media.append(img); media.append(img);
} else if (item.kind === "video") { } else if (item.kind === "video" && canFrameVideo(item)) {
// No thumbnail: take the first frame through the same-origin stream endpoint, // No thumbnail: take the first frame through the same-origin stream endpoint,
// which refreshes expired URLs and avoids cross-origin blocking. // which refreshes expired URLs and avoids cross-origin blocking.
const video = document.createElement("video"); const video = document.createElement("video");
@@ -1160,14 +1167,14 @@
); );
media.append( media.append(
button("Open in new tab", () => button("Open in new tab", () =>
window.open(directVideo, "_blank", "noreferrer") window.open(item.playUrl || directVideo, "_blank", "noreferrer")
) )
); );
}, },
{ once: true } { once: true }
); );
media.append(video); media.append(video);
} else if (item.kind === "video") { } else if (item.kind === "video" && canFrameVideo(item)) {
// No still available: show the first frame through the stream endpoint. // No still available: show the first frame through the stream endpoint.
const video = document.createElement("video"); const video = document.createElement("video");
video.preload = "metadata"; video.preload = "metadata";
@@ -1183,7 +1190,7 @@
); );
media.append(video); media.append(video);
media.append(viewerPlayIcon(item)); media.append(viewerPlayIcon(item));
} else if (/^(https?:|\/)/.test(source)) { } else if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) {
const img = document.createElement("img"); const img = document.createElement("img");
img.src = source; img.src = source;
img.alt = item.kind; img.alt = item.kind;
+7
View File
@@ -807,6 +807,13 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
throw new HttpError(502, `upstream ${upstream.status}`); throw new HttpError(502, `upstream ${upstream.status}`);
} }
// Never pipe a page into a media element: embed URLs that are not direct files
// (YouTube, provider redirectors) answer with HTML.
const upstreamType = upstream.headers.get("content-type") ?? "";
if (!/^(image|video|audio|application\/octet-stream)/i.test(upstreamType)) {
throw new HttpError(415, `not a media resource (${upstreamType || "unknown"})`);
}
res.status(upstream.status); res.status(upstream.status);
for (const header of ["content-type", "content-length", "accept-ranges", "content-range"]) { for (const header of ["content-type", "content-length", "accept-ranges", "content-range"]) {
const value = upstream.headers.get(header); const value = upstream.headers.get(header);