Cache preview frames for Discord video attachments
This commit is contained in:
@@ -40,6 +40,10 @@ mobile draft at 390px.
|
||||
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
|
||||
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
|
||||
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
|
||||
- [x] C29 Attachment video previews. Discord's image proxy renders a frame for video files
|
||||
(`?width=320&height=320&format=webp`), so `previewUrlFor` now resizes attachment mp4/webm in
|
||||
links mode too and `pendingPreviews` includes non-embed videos. Live: paused-scrape videos
|
||||
backfilled to `image/webp` thumbs, pending queue drained to 0.
|
||||
- [x] C28 Media grid usability. Infinite scroll (240 px threshold) with a Load more fallback,
|
||||
`previewError` exposed so tiles show PRIVATE (403) or GONE (404/410) with the reason in the
|
||||
tooltip, and the scrape strip lists only queued/running jobs — paused scrapes move to History.
|
||||
|
||||
@@ -117,8 +117,11 @@ Downloads run after a finished walk when the scrape used `mode: "download"`:
|
||||
|
||||
Previews run after every walk:
|
||||
|
||||
- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
|
||||
are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
|
||||
- In both modes, Discord-hosted image, GIF, and **video** URLs (`cdn.discordapp.com`,
|
||||
`media.discordapp.net`) are transformed to the CDN resizer
|
||||
(`media.discordapp.net?...width=320&height=320&format=webp`). The proxy renders a frame for
|
||||
video files, so attachment mp4/webm get a cached poster in links mode without downloading the
|
||||
file. Download mode additionally keeps an ffmpeg poster.
|
||||
- In both modes, embed rows also get their stored `preview_url` (YouTube, X, TikTok thumbnails,
|
||||
and so on) cached. The embed's full media URL is never fetched; embed rows are excluded from
|
||||
`pendingDownloads`.
|
||||
|
||||
+3
-2
@@ -90,8 +90,9 @@ media-scrape/db/
|
||||
owns that path.
|
||||
- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
|
||||
other embed stays a link. Its thumbnail is cached (below); the video itself is never fetched.
|
||||
- Previews are cached in `thumbs/` after every walk, in both modes: Discord-hosted images and GIFs
|
||||
(resized to 320px WebP) and third-party embed thumbnails (YouTube, X, TikTok, and so on). The
|
||||
- Previews are cached in `thumbs/` after every walk, in both modes: Discord-hosted images, GIFs,
|
||||
and video files (the proxy renders a frame, resized to 320px WebP) plus third-party embed
|
||||
thumbnails (YouTube, X, TikTok, and so on). The
|
||||
stored bytes keep their real type (`preview_ext`), so JPEG provider thumbnails are served as
|
||||
`image/jpeg`, not mislabeled WebP. Embeds without a thumbnail in Discord's data are resolved
|
||||
through the provider's oEmbed/fxtwitter API and finally the page's `og:image`. Previews count
|
||||
|
||||
@@ -193,18 +193,17 @@ const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
|
||||
/**
|
||||
* The preview target for a media row.
|
||||
*
|
||||
* Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
|
||||
* Attachments and stickers: their Discord-hosted URL, resized to a small WebP. Discord's proxy
|
||||
* also renders a frame for video files, so attachment videos get a poster for free.
|
||||
* Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
|
||||
* Returns null when the row has no preview source.
|
||||
* A bare embed URL is a page, never a preview. Returns null when the row has no preview source.
|
||||
*/
|
||||
export function previewUrlFor(media: {
|
||||
url: string;
|
||||
kind: string;
|
||||
previewUrl?: string | null;
|
||||
}): string | null {
|
||||
const candidate =
|
||||
media.previewUrl ??
|
||||
(media.kind === "image" || media.kind === "gif" ? media.url : null);
|
||||
const candidate = media.previewUrl ?? media.url;
|
||||
if (!candidate) return null;
|
||||
|
||||
try {
|
||||
@@ -214,8 +213,13 @@ export function previewUrlFor(media: {
|
||||
url.searchParams.set("width", "320");
|
||||
url.searchParams.set("height", "320");
|
||||
url.searchParams.set("format", "webp");
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
if (media.previewUrl || media.kind === "image" || media.kind === "gif") {
|
||||
return url.toString();
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -342,7 +342,8 @@ export class ScrapeStore {
|
||||
* embed thumbnails only when `includeEmbeds` is set (download mode).
|
||||
*/
|
||||
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
|
||||
const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))";
|
||||
const local =
|
||||
"((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video'))";
|
||||
const condition = includeEmbeds
|
||||
? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
|
||||
: local;
|
||||
|
||||
@@ -522,6 +522,10 @@ void (async () => {
|
||||
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
|
||||
);
|
||||
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
|
||||
assert.equal(
|
||||
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4?ex=abc", kind: "video" }),
|
||||
"https://media.discordapp.net/attachments/1/2/clip.mp4?ex=abc&width=320&height=320&format=webp"
|
||||
);
|
||||
assert.equal(
|
||||
previewUrlFor({
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
@@ -599,7 +603,10 @@ void (async () => {
|
||||
{
|
||||
messages: [
|
||||
message("600", {
|
||||
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })],
|
||||
attachments: [
|
||||
attachment({ filename: "prev.png", contentType: "image/png" }),
|
||||
attachment({ filename: "clip.mp4", contentType: "video/mp4" })
|
||||
],
|
||||
embeds: [
|
||||
{
|
||||
type: "video",
|
||||
@@ -643,9 +650,10 @@ void (async () => {
|
||||
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.previewed, 2);
|
||||
assert.equal(meta?.previewed, 3);
|
||||
assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
|
||||
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
||||
assert.equal(fetched.some(url => url.includes("clip.mp4") && url.includes("format=webp")), true);
|
||||
|
||||
const store = new ScrapeStore(join(root, started.id));
|
||||
const row = store.queryMedia({ kind: "image" })[0];
|
||||
|
||||
Reference in New Issue
Block a user