Resolve external embed previews and keep their real MIME types

2026-09-16 12:52:44 +02:00
parent 7fa6118b6f
commit 1b5fcfed2f
11 changed files with 313 additions and 41 deletions
@@ -59,9 +59,25 @@ export interface DownloadOutcome {
    path?: string;
    hash?: string;
    bytes?: number;
+    /** File extension of the stored bytes, when known. */
+    ext?: string;
    error?: string;
 }

+const PREVIEW_EXTENSIONS: Record<string, string> = {
+    "image/webp": "webp",
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/gif": "gif",
+    "image/avif": "avif"
+};
+
+/** Extension for preview bytes from the response type, defaulting to webp. */
+export function previewExtension(contentType: string | null): string {
+    const type = (contentType ?? "").split(";")[0].trim().toLowerCase();
+    return PREVIEW_EXTENSIONS[type] ?? "webp";
+}
+
 /** 4xx that will not improve on retry: the signed URL expired or the file is gone. */
 const SKIP_STATUS = new Set([400, 401, 403, 404, 410]);

@@ -214,11 +230,13 @@ export async function downloadPreview(
            return { id: media.id, state: "skipped", error: "too large" };
        }

+        // Keep the real type: provider thumbnails are often JPEG or PNG, not WebP.
+        const ext = previewExtension(response.headers.get("content-type"));
        const hash = createHash("sha256").update(buffer).digest("hex");
-        const file = `${hash}.webp`;
+        const file = `${hash}.${ext}`;
        writeFileSync(join(dir, file), buffer);

-        return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
+        return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength, ext };
    } catch (error) {
        return {
            id: media.id,
@@ -0,0 +1,114 @@
+/*
+ * Resolve a preview image for embeds that carry none in Discord's data.
+ * Provider oEmbed/API first, then the page's og:image / twitter:image.
+ */
+
+import type { DownloadOptions } from "./download.js";
+import type { MediaRow } from "./store.js";
+
+const USER_AGENT = "veskslop-media/0.1";
+const MAX_HTML_BYTES = 512 * 1024;
+const TIMEOUT_MS = 10_000;
+
+function fetchOf(options: DownloadOptions): typeof fetch {
+    return options.fetchImpl ?? fetch;
+}
+
+async function getJson(url: string, options: DownloadOptions): Promise<any | null> {
+    try {
+        const response = await fetchOf(options)(url, {
+            headers: { "user-agent": USER_AGENT, accept: "application/json" },
+            signal: AbortSignal.timeout(TIMEOUT_MS)
+        });
+        if (!response.ok) return null;
+        return await response.json();
+    } catch {
+        return null;
+    }
+}
+
+async function getText(url: string, options: DownloadOptions): Promise<string | null> {
+    try {
+        const response = await fetchOf(options)(url, {
+            headers: { "user-agent": USER_AGENT, accept: "text/html" },
+            signal: AbortSignal.timeout(TIMEOUT_MS)
+        });
+        if (!response.ok) return null;
+        return (await response.text()).slice(0, MAX_HTML_BYTES);
+    } catch {
+        return null;
+    }
+}
+
+/** First og:image or twitter:image in the page head, either attribute order. */
+export function ogImageFromHtml(html: string): string | null {
+    for (const pattern of [
+        /<meta[^>]+(?:property|name)=["']og:image["'][^>]+content=["']([^"']+)["']/i,
+        /<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']og:image["']/i,
+        /<meta[^>]+(?:property|name)=["']twitter:image["'][^>]+content=["']([^"']+)["']/i,
+        /<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']twitter:image["']/i
+    ]) {
+        const match = pattern.exec(html);
+        if (match?.[1]) return match[1];
+    }
+    return null;
+}
+
+/**
+ * A preview URL for an embed. Uses the stored thumbnail when present, otherwise asks the
+ * provider (oEmbed, fxtwitter) and finally falls back to the page's og:image.
+ */
+export async function resolveEmbedPreview(
+    media: Pick<MediaRow, "url" | "source" | "preview_url">,
+    options: DownloadOptions = {}
+): Promise<string | null> {
+    if (media.source !== "embed") return null;
+    if (media.preview_url) return media.preview_url;
+
+    let parsed: URL;
+    try {
+        parsed = new URL(media.url);
+    } catch {
+        return null;
+    }
+
+    const host = parsed.hostname.replace(/^www\./, "").toLowerCase();
+
+    if (host === "youtube.com" || host === "m.youtube.com" || host === "youtu.be") {
+        const data = await getJson(
+            `https://www.youtube.com/oembed?url=${encodeURIComponent(media.url)}&format=json`,
+            options
+        );
+        if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
+    }
+
+    if (host === "vimeo.com") {
+        const data = await getJson(
+            `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(media.url)}`,
+            options
+        );
+        if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
+    }
+
+    if (host === "tiktok.com") {
+        const data = await getJson(
+            `https://www.tiktok.com/oembed?url=${encodeURIComponent(media.url)}`,
+            options
+        );
+        if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
+    }
+
+    if (host === "fxtwitter.com" || host === "vxtwitter.com" || host === "twitter.com" || host === "x.com") {
+        const match = /\/status\/(\d+)/.exec(parsed.pathname);
+        if (match) {
+            const data = await getJson(`https://api.fxtwitter.com/status/${match[1]}`, options);
+            const video = data?.tweet?.media?.videos?.[0]?.thumbnail_url;
+            const photo = data?.tweet?.media?.photos?.[0]?.url;
+            if (typeof video === "string") return video;
+            if (typeof photo === "string") return photo;
+        }
+    }
+
+    const html = await getText(media.url, options);
+    return html ? ogImageFromHtml(html) : null;
+}
@@ -13,6 +13,7 @@ import {
    videoPoster,
    type DownloadOptions
 } from "./download.js";
+import { resolveEmbedPreview } from "./embed-preview.js";
 import {
    createScrapeDirs,
    listScrapes,
@@ -358,18 +359,38 @@ export class ScrapeEngine {
                return;
            }

-            const outcome = await downloadPreview(row, join(dir, "thumbs"), options);
+            // Some providers (fxtwitter, vxReddit, ...) ship no thumbnail in Discord's data;
+            // resolve one from the provider or the page before caching it.
+            let previewRow = row;
+            if (row.source === "embed" && !row.preview_url) {
+                const resolved = await resolveEmbedPreview(row, options);
+                if (!resolved) {
+                    store.markPreviewError(row.id, "no preview source");
+                    this.persist(dir, meta);
+                    await this.sleep(downloadGapMs(options));
+                    continue;
+                }
+                previewRow = { ...row, preview_url: resolved };
+            }
+
+            const outcome = await downloadPreview(previewRow, join(dir, "thumbs"), options);

            if (outcome.state === "downloaded" && outcome.hash) {
-                store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
+                const ext = outcome.ext ?? "webp";
+                store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0, ext });
                meta.previewed = (meta.previewed ?? 0) + 1;

                // Embed thumbnails also land in the custom destination, so the download has a
                // viewable poster for embeds while the media tab keeps its local copy.
                if (options.destinationDir && row.source === "embed") {
-                    const target = join(options.destinationDir, meta.id, "thumbnails", `${outcome.hash}.webp`);
+                    const target = join(
+                        options.destinationDir,
+                        meta.id,
+                        "thumbnails",
+                        `${outcome.hash}.${ext}`
+                    );
                    mkdirSync(dirname(target), { recursive: true });
-                    copyFileSync(join(dir, "thumbs", `${outcome.hash}.webp`), target);
+                    copyFileSync(join(dir, "thumbs", `${outcome.hash}.${ext}`), target);
                }
            } else {
                store.markPreviewError(row.id, outcome.error ?? outcome.state);
@@ -4,6 +4,7 @@
 */

 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
 export {
@@ -133,6 +133,7 @@ export interface MediaRow {
    preview_url: string | null;
    file_hash: string | null;
    preview_hash: string | null;
+    preview_ext: string | null;
    file_bytes: number | null;
    preview_bytes: number | null;
    file_path: string | null;
@@ -169,6 +170,7 @@ CREATE TABLE IF NOT EXISTS media (
    preview_url TEXT,
    file_hash TEXT,
    preview_hash TEXT,
+    preview_ext TEXT,
    file_bytes INTEGER,
    preview_bytes INTEGER,
    file_path TEXT,
@@ -212,6 +214,7 @@ export class ScrapeStore {
        if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
        if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
        if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
+        if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT");
        if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
    }

@@ -347,10 +350,12 @@ export class ScrapeStore {
            .all(limit) as unknown as MediaRow[];
    }

-    markPreview(id: number, preview: { hash: string; bytes: number }): void {
+    markPreview(id: number, preview: { hash: string; bytes: number; ext?: string }): void {
        this.db
-            .prepare("UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_error = NULL WHERE id = ?")
-            .run(preview.hash, preview.bytes, id);
+            .prepare(
+                "UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?"
+            )
+            .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
    }

    markPreviewError(id: number, error: string): void {
@@ -10,6 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { downloadOne, downloadPreview, previewUrlFor, videoPoster } from "../src/download.js";
+import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
@@ -549,6 +550,17 @@ void (async () => {
        assert.equal(outcome.state, "downloaded");
        assert.equal(outcome.path, `${outcome.hash}.webp`);
        assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
+
+        const jpeg = await downloadPreview(
+            { id: 10, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image", source: "attachment", preview_url: null },
+            dir,
+            {
+                fetchImpl: async () =>
+                    new Response("jpeg-bytes", { status: 200, headers: { "content-type": "image/jpeg" } })
+            }
+        );
+        assert.equal(jpeg.path, `${jpeg.hash}.jpg`);
+        assert.equal(existsSync(join(dir, `${jpeg.hash}.jpg`)), true);
    });

    await test("embed thumbnails download only when enabled", async () => {
@@ -635,6 +647,7 @@ void (async () => {
        const row = store.queryMedia({ kind: "image" })[0];
        assert.ok(row.preview_hash);
        assert.ok((row.preview_bytes ?? 0) > 0);
+        assert.equal(row.preview_ext, "webp");
        store.close();
        assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true);
    });
@@ -704,14 +717,82 @@ void (async () => {

        const embed = store.queryMedia({ kind: "video" })[0];
        assert.ok(embed.preview_hash);
-        assert.equal(
-            existsSync(join(dest, started.id, "thumbnails", `${embed.preview_hash}.webp`)),
-            true
-        );
-        assert.equal(existsSync(join(root, started.id, "thumbs", `${embed.preview_hash}.webp`)), true);
+        const previewFile = `${embed.preview_hash}.${embed.preview_ext ?? "webp"}`;
+        assert.equal(existsSync(join(dest, started.id, "thumbnails", previewFile)), true);
+        assert.equal(existsSync(join(root, started.id, "thumbs", previewFile)), true);
        store.close();
    });

+    await test("embed previews resolve from providers and og:image", async () => {
+        const fetchImpl = async (input: any) => {
+            const url = String(input);
+            if (url.includes("youtube.com/oembed")) {
+                return new Response(JSON.stringify({ thumbnail_url: "https://i.ytimg.com/ok.jpg" }), {
+                    status: 200
+                });
+            }
+            if (url.includes("api.fxtwitter.com")) {
+                return new Response(
+                    JSON.stringify({ tweet: { media: { videos: [{ thumbnail_url: "https://pbs.twimg.com/ok.jpg" }] } } }),
+                    { status: 200 }
+                );
+            }
+            if (url.includes("vxreddit.com")) {
+                return new Response(
+                    '<html><head><meta property="og:image" content="https://i.redd.it/ok.jpg"></head></html>',
+                    { status: 200 }
+                );
+            }
+            return new Response("no", { status: 404 });
+        };
+
+        assert.equal(
+            await resolveEmbedPreview(
+                { url: "https://youtu.be/abc", source: "embed", preview_url: null },
+                { fetchImpl }
+            ),
+            "https://i.ytimg.com/ok.jpg"
+        );
+        assert.equal(
+            await resolveEmbedPreview(
+                { url: "https://fxtwitter.com/u/status/12345", source: "embed", preview_url: null },
+                { fetchImpl }
+            ),
+            "https://pbs.twimg.com/ok.jpg"
+        );
+        assert.equal(
+            await resolveEmbedPreview(
+                { url: "https://vxreddit.com/r/x/comments/1", source: "embed", preview_url: null },
+                { fetchImpl }
+            ),
+            "https://i.redd.it/ok.jpg"
+        );
+        assert.equal(
+            await resolveEmbedPreview(
+                { url: "https://x.com/u/status/1", source: "embed", preview_url: "https://stored/ok.jpg" },
+                { fetchImpl }
+            ),
+            "https://stored/ok.jpg"
+        );
+        assert.equal(
+            await resolveEmbedPreview(
+                { url: "https://cdn.discordapp.com/x.png", source: "attachment", preview_url: null },
+                { fetchImpl }
+            ),
+            null
+        );
+
+        assert.equal(
+            ogImageFromHtml('<meta content="https://a/1.jpg" property="og:image">'),
+            "https://a/1.jpg"
+        );
+        assert.equal(
+            ogImageFromHtml('<meta name="twitter:image" content="https://a/2.jpg">'),
+            "https://a/2.jpg"
+        );
+        assert.equal(ogImageFromHtml("<html></html>"), null);
+    });
+
    await test("videoPoster extracts a cached frame", async () => {
        const dir = join(root, "poster-test");
        mkdirSync(dir, { recursive: true });