Resolve thumbnail-less embeds, including Discord CDN unfurls

docs/media-scraper.md

@@ -138,7 +138,11 @@ Previews run after every walk:
   `pendingDownloads`.
 - Embeds without a thumbnail in Discord's data (fxtwitter, vxReddit, ...) are resolved first:
   provider oEmbed/fxtwitter API, then the page's `og:image`/`twitter:image` (512 KiB read cap,
-  10 s timeout, one attempt).
+  10 s timeout, one attempt). The page read never consumes media bytes: a non-HTML content type
+  stops it. An embed that unfurls a pasted Discord CDN attachment link resolves through the Discord
+  proxy instead, so it gets the same still (and video frame) as a plain attachment. Every
+  thumbnail-less embed enters preview passes; one with no resolvable image is closed as
+  `no preview source`.
 - Animated GIF previews are stored as a static first frame (`ffmpeg`, 480px WebP); the original
   animated GIF URL stays in `preview_url` for hover playback. Tenor embeds without a thumbnail are
   resolved through Tenor's oEmbed. An animated GIF over the size cap is framed by ffmpeg reading

media-scrape/src/backfill.ts

@@ -392,7 +392,8 @@ export class BackfillQueue {
             job.processed += progressed + result.ratelimited;
 
             const elapsed = Math.max(1, this.now() - (job.startedAt ?? this.now()));
-            job.perMinute = job.processed / (elapsed / 60_000);
+            // Early in a run the measured rate is noise; report null until it settles.
+            job.perMinute = elapsed >= 30_000 ? job.processed / (elapsed / 60_000) : null;
 
             const after = this.engine.rateLimitStatus();
             if (after.active) this.limitedUntil = after.until ?? this.now() + after.remainingMs;

media-scrape/src/embed-preview.ts

@@ -34,6 +34,11 @@ async function getText(url: string, options: DownloadOptions): Promise<string |
             signal: AbortSignal.timeout(TIMEOUT_MS)
         });
         if (!response.ok) return null;
+
+        // Never pull media bytes into memory just to look for an og:image.
+        const type = response.headers.get("content-type") ?? "";
+        if (type && !/text\/html|application\/xhtml|text\/plain/i.test(type)) return null;
+
         return (await response.text()).slice(0, MAX_HTML_BYTES);
     } catch {
         return null;
@@ -74,6 +79,16 @@ export async function resolveEmbedPreview(
 
     const host = parsed.hostname.replace(/^www\./, "").toLowerCase();
 
+    // Discord unfurls a pasted attachment link into an embed. The URL is still a CDN file, so
+    // the proxy can render a still (and a video frame) exactly like a plain attachment.
+    if ((host === "cdn.discordapp.com" || host === "media.discordapp.net") && parsed.pathname.startsWith("/attachments/")) {
+        parsed.hostname = "media.discordapp.net";
+        parsed.searchParams.set("width", "320");
+        parsed.searchParams.set("height", "320");
+        parsed.searchParams.set("format", "webp");
+        return parsed.toString();
+    }
+
     if (host === "youtube.com" || host === "m.youtube.com" || host === "youtu.be") {
         const data = await getJson(
             `https://www.youtube.com/oembed?url=${encodeURIComponent(media.url)}&format=json`,

media-scrape/src/store.ts

@@ -356,9 +356,9 @@ export class ScrapeStore {
     pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
         const local =
             "((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))";
-        const condition = includeEmbeds
+        // With embeds on, a thumbnail-less embed still needs a pass: the engine resolves one
-            ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
+        // from the provider or the page (or the Discord proxy for CDN attachment embeds).
-            : local;
+        const condition = includeEmbeds ? `(${local} OR source = 'embed')` : local;
 
         return this.db
             .prepare(

media-scrape/test/run.ts

@@ -612,6 +612,68 @@ void (async () => {
         assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
     });
 
+    await test("pendingPreviews includes embeds without a stored thumbnail", () => {
+        const dir = join(root, "pending-embeds");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+        store.insertPage([
+            message("670", {
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
+                        title: null,
+                        provider: "cdn.discordapp.com",
+                        videoUrl: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            })
+        ]);
+
+        assert.equal(store.pendingPreviews(10, false).length, 0);
+        assert.equal(store.pendingPreviews(10, true).length, 1);
+        store.close();
+    });
+
+    await test("CDN attachment embeds resolve through the Discord proxy", async () => {
+        const resolved = await resolveEmbedPreview({
+            url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4",
+            source: "embed",
+            preview_url: null
+        });
+
+        assert.ok(resolved);
+        const parsed = new URL(String(resolved));
+        assert.equal(parsed.hostname, "media.discordapp.net");
+        assert.equal(parsed.searchParams.get("width"), "320");
+        assert.equal(parsed.searchParams.get("format"), "webp");
+    });
+
+    await test("the og:image fallback never reads media bytes", async () => {
+        const seen: string[] = [];
+        const resolved = await resolveEmbedPreview(
+            {
+                url: "https://example.com/clip.mp4",
+                source: "embed",
+                preview_url: null
+            },
+            {
+                fetchImpl: async (input: any) => {
+                    seen.push(String(input));
+                    return new Response("binary", {
+                        status: 200,
+                        headers: { "content-type": "video/mp4" }
+                    });
+                }
+            }
+        );
+
+        assert.equal(resolved, null);
+        assert.equal(seen.length, 1);
+    });
+
     await test("engine caches previews in links mode, including embed thumbnails", async () => {
         const backend = new FakeBackend([
             {