Cache embed thumbnails in download mode

CURRENT.md

@@ -40,6 +40,11 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL;
+    the full third-party media is never fetched. Live run `...-20260916091458` on the test
+    channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also
+    fixed the state transition: the job stays `running` through download and preview passes so
+    clients do not see `done` early.
   - [x] C12 Media tab filters and full-window layout: kind chips, source filter
     (attachment/sticker/embed), source-channel filter, and the view now fills the window below the
     header and call strip. API/MCP gained `source` and `channelId` on `query_media`. Smoke-tested

docs/media-scraper.md

@@ -101,14 +101,18 @@ Downloads run after a finished walk when the scrape used `mode: "download"`:
   (`source != 'embed'`).
 - Progress counters: `downloaded`, `downloadErrors`.
 
-Previews run after every walk, in both modes:
+Previews run after every walk:
 
-- Only Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) are
+- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
-  transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
+  are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
-  Third-party embeds are skipped.
+- In `download` mode only, embed rows also get their stored `preview_url` (YouTube, X, TikTok
+  thumbnails, and so on) cached. The embed's full media URL is never fetched; embed rows are
+  excluded from `pendingDownloads`.
 - One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
 - Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and
   the counter is `previewed`. Previews count toward the storage limit.
+- The job stays `running` through the download and preview passes, so a client never sees `done`
+  on a half-archived scrape.
 
 ## Job states
 

docs/media.md

@@ -80,14 +80,12 @@ media-scrape/db/
 - `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows
   are never download targets. The Media tab prefers the local file when one exists, so the archive
   survives CDN URL expiry.
-- Discord-hosted image and GIF previews are cached small (WebP, 320px) in `thumbs/` after every
+- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
-  walk, in both modes, so the grid does not depend on expiring CDN links. Third-party embed
+  other embed stays a link. In `download` mode their thumbnail is cached (below); the video itself
-  thumbnails are never fetched.
+  is never fetched.
-- Third-party embeds are **never downloaded**: YouTube, Instagram, X, TikTok, and every other
+- Previews are cached small (WebP, 320px) in `thumbs/` after every walk. In both modes this covers
-  embed stays a link with metadata.
+  Discord-hosted images and GIFs; in `download` mode it also covers embed thumbnails, so the grid
-- Discord-hosted image and sticker previews are cached small (`thumbs/`, WebP, via the CDN
+  does not depend on expiring CDN links. Previews count toward the storage limit.
-  resizer) in both modes, so the Media grid does not depend on expiring CDN links. Previews count
-  toward the storage limit.
 - Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is
   refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry.
 

media-scrape/src/download.ts

@@ -19,6 +19,8 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Cache third-party embed thumbnails. Download mode only. */
+    embedPreviews?: boolean;
     sleep?: (ms: number) => Promise<void>;
     random?: () => number;
     log?: (line: string) => void;
@@ -125,20 +127,30 @@ export async function downloadOne(
 const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
 
 /**
- * A small WebP preview URL for a Discord-hosted image or GIF.
+ * The preview target for a media row.
- * Third-party embeds return null: they are never fetched.
+ *
+ * Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
+ * Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
+ * Returns null when the row has no preview source.
  */
-export function previewUrlFor(media: { url: string; kind: string }): string | null {
+export function previewUrlFor(media: {
-    if (media.kind !== "image" && media.kind !== "gif") return null;
+    url: string;
+    kind: string;
+    previewUrl?: string | null;
+}): string | null {
+    const candidate =
+        media.previewUrl ??
+        (media.kind === "image" || media.kind === "gif" ? media.url : null);
+    if (!candidate) return null;
 
     try {
-        const url = new URL(media.url);
+        const url = new URL(candidate);
-        if (!DISCORD_HOSTS.has(url.hostname)) return null;
+        if (DISCORD_HOSTS.has(url.hostname)) {
-
             url.hostname = "media.discordapp.net";
             url.searchParams.set("width", "320");
             url.searchParams.set("height", "320");
             url.searchParams.set("format", "webp");
+        }
         return url.toString();
     } catch {
         return null;
@@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu
 
 /** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
 export async function downloadPreview(
-    media: Pick<MediaRow, "id" | "url" | "kind">,
+    media: Pick<MediaRow, "id" | "url" | "kind" | "source" | "preview_url">,
     dir: string,
     options: DownloadOptions = {}
 ): Promise<DownloadOutcome> {
-    const target = previewUrlFor(media);
+    // Third-party thumbnails only in download mode; the full embed is never fetched.
+    if (media.source === "embed" && options.embedPreviews !== true) {
+        return { id: media.id, state: "skipped", error: "embed previews off" };
+    }
+
+    const target = previewUrlFor({
+        url: media.url,
+        kind: media.kind,
+        previewUrl: media.preview_url
+    });
     if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
 
     const fetchImpl = options.fetchImpl ?? fetch;

media-scrape/src/engine.ts

@@ -220,13 +220,25 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && meta.mode === "download") {
+            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+                // Keep the job running until post-processing finishes, so a client never
+                // treats a half-archived scrape as complete.
+                meta.state = "running";
+                this.persist(dir, meta);
+
+                if (meta.mode === "download") {
                     await this.runDownloads(store, dir, meta);
                 }
 
-            if (meta.state === "done" && this.options.previews !== false) {
+                if (this.options.previews !== false && !this.stopRequested.has(meta.id)) {
                     await this.runPreviews(store, dir, meta);
                 }
+
+                if (meta.state === "running") {
+                    meta.state = "done";
+                }
+                this.persist(dir, meta);
+            }
         } catch (error) {
             meta.state = "failed";
             meta.lastError = error instanceof Error ? error.message : String(error);
@@ -271,10 +283,11 @@ export class ScrapeEngine {
         }
     }
 
-    /** Cache small previews for Discord-hosted images. Runs in both modes. */
+    /** Cache small previews. In download mode this includes embed thumbnails. */
     private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
-        const options = this.options.download ?? {};
+        const embedPreviews = meta.mode === "download";
-        const pending = store.pendingPreviews();
+        const options = { ...(this.options.download ?? {}), embedPreviews };
+        const pending = store.pendingPreviews(5000, embedPreviews);
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {

media-scrape/src/store.ts

@@ -294,11 +294,19 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }
 
-    /** Image and GIF rows that still need a cached preview. */
+    /**
-    pendingPreviews(limit = 5000): MediaRow[] {
+     * Rows that still need a cached preview. Attachment/sticker images and GIFs always;
+     * embed thumbnails only when `includeEmbeds` is set (download mode).
+     */
+    pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
+        const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))";
+        const condition = includeEmbeds
+            ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
+            : local;
+
         return this.db
             .prepare(
-                "SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?"
+                `SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
             )
             .all(limit) as unknown as MediaRow[];
     }

media-scrape/test/run.ts

@@ -351,7 +351,7 @@ void (async () => {
                                 provider: "YouTube",
                                 videoUrl: "https://www.youtube.com/embed/x",
                                 imageUrl: null,
-                                thumbnailUrl: null
+                                thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
                             }
                         ]
                     })
@@ -359,6 +359,7 @@ void (async () => {
             }
         ]);
 
+        const fetched: string[] = [];
         const engine = new ScrapeEngine({
             rootDir: root,
             pageSize: 2,
@@ -367,10 +368,15 @@ void (async () => {
             sleep: async () => {},
             random: () => 0,
             log: () => {},
-            previews: false,
+            previews: true,
             download: {
-                fetchImpl: async () =>
+                fetchImpl: async (input: any) => {
-                    new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
+                    fetched.push(String(input));
+                    return new Response("png-bytes", {
+                        status: 200,
+                        headers: { "content-type": "image/png" }
+                    });
+                },
                 minGapMs: 0,
                 maxGapMs: 0,
                 sleep: async () => {}
@@ -383,6 +389,9 @@ void (async () => {
         const meta = engine.get(started.id);
         assert.equal(meta?.state, "done");
         assert.equal(meta?.downloaded, 1);
+        assert.equal(meta?.previewed, 2);
+        assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
+        assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
 
         const store = new ScrapeStore(join(root, started.id));
         const row = store.queryMedia({ kind: "image" })[0];
@@ -426,13 +435,20 @@ void (async () => {
         store.close();
     });
 
-    await test("preview urls are Discord-only resizes", () => {
+    await test("preview urls resize Discord and pass embed thumbnails through", () => {
         assert.equal(
             previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
             "https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
         );
         assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
-        assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null);
+        assert.equal(
+            previewUrlFor({
+                url: "https://www.youtube.com/watch?v=x",
+                kind: "video",
+                previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+            }),
+            "https://i.ytimg.com/vi/x/hqdefault.jpg"
+        );
     });
 
     await test("preview downloads are written by hash", async () => {
@@ -440,7 +456,13 @@ void (async () => {
         mkdirSync(dir, { recursive: true });
 
         const outcome = await downloadPreview(
-            { id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" },
+            {
+                id: 9,
+                url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
+                kind: "image",
+                source: "attachment",
+                preview_url: null
+            },
             dir,
             {
                 fetchImpl: async () =>
@@ -453,17 +475,57 @@ void (async () => {
         assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
     });
 
-    await test("engine caches previews in links mode", async () => {
+    await test("embed thumbnails download only when enabled", async () => {
+        const dir = join(root, "preview-embed");
+        mkdirSync(dir, { recursive: true });
+
+        const media = {
+            id: 11,
+            url: "https://www.youtube.com/watch?v=x",
+            kind: "video" as const,
+            source: "embed",
+            preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+        };
+        const calls: string[] = [];
+        const fetchImpl = async (input: any) => {
+            calls.push(String(input));
+            return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
+        };
+
+        const off = await downloadPreview(media, dir, { fetchImpl });
+        assert.equal(off.state, "skipped");
+        assert.equal(calls.length, 0);
+
+        const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
+        assert.equal(on.state, "downloaded");
+        assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
+        assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
+    });
+
+    await test("engine caches previews in links mode and skips embeds", async () => {
         const backend = new FakeBackend([
             {
                 messages: [
                     message("600", {
-                        attachments: [attachment({ filename: "prev.png", contentType: "image/png" })]
+                        attachments: [attachment({ filename: "prev.png", contentType: "image/png" })],
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://www.youtube.com/watch?v=x",
+                                title: null,
+                                provider: "YouTube",
+                                videoUrl: "https://www.youtube.com/embed/x",
+                                imageUrl: null,
+                                thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+                            }
+                        ]
                     })
                 ]
             }
         ]);
 
+        const fetched: string[] = [];
+
         const engine = new ScrapeEngine({
             rootDir: root,
             pageSize: 2,
@@ -474,8 +536,10 @@ void (async () => {
             log: () => {},
             previews: true,
             download: {
-                fetchImpl: async () =>
+                fetchImpl: async (input: any) => {
-                    new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }),
+                    fetched.push(String(input));
+                    return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
+                },
                 minGapMs: 0,
                 maxGapMs: 0,
                 sleep: async () => {}
@@ -488,6 +552,7 @@ void (async () => {
         const meta = engine.get(started.id);
         assert.equal(meta?.state, "done");
         assert.equal(meta?.previewed, 1);
+        assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
 
         const store = new ScrapeStore(join(root, started.id));
         const row = store.queryMedia({ kind: "image" })[0];