Cache embed thumbnails in download mode

media-scrape/src/download.ts

@@ -19,6 +19,8 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Cache third-party embed thumbnails. Download mode only. */
+    embedPreviews?: boolean;
     sleep?: (ms: number) => Promise<void>;
     random?: () => number;
     log?: (line: string) => void;
@@ -125,20 +127,30 @@ export async function downloadOne(
 const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
 
 /**
- * A small WebP preview URL for a Discord-hosted image or GIF.
- * Third-party embeds return null: they are never fetched.
+ * The preview target for a media row.
+ *
+ * Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
+ * Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
+ * Returns null when the row has no preview source.
  */
-export function previewUrlFor(media: { url: string; kind: string }): string | null {
-    if (media.kind !== "image" && media.kind !== "gif") return null;
+export function previewUrlFor(media: {
+    url: string;
+    kind: string;
+    previewUrl?: string | null;
+}): string | null {
+    const candidate =
+        media.previewUrl ??
+        (media.kind === "image" || media.kind === "gif" ? media.url : null);
+    if (!candidate) return null;
 
     try {
-        const url = new URL(media.url);
-        if (!DISCORD_HOSTS.has(url.hostname)) return null;
-
-        url.hostname = "media.discordapp.net";
-        url.searchParams.set("width", "320");
-        url.searchParams.set("height", "320");
-        url.searchParams.set("format", "webp");
+        const url = new URL(candidate);
+        if (DISCORD_HOSTS.has(url.hostname)) {
+            url.hostname = "media.discordapp.net";
+            url.searchParams.set("width", "320");
+            url.searchParams.set("height", "320");
+            url.searchParams.set("format", "webp");
+        }
         return url.toString();
     } catch {
         return null;
@@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu
 
 /** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
 export async function downloadPreview(
-    media: Pick<MediaRow, "id" | "url" | "kind">,
+    media: Pick<MediaRow, "id" | "url" | "kind" | "source" | "preview_url">,
     dir: string,
     options: DownloadOptions = {}
 ): Promise<DownloadOutcome> {
-    const target = previewUrlFor(media);
+    // Third-party thumbnails only in download mode; the full embed is never fetched.
+    if (media.source === "embed" && options.embedPreviews !== true) {
+        return { id: media.id, state: "skipped", error: "embed previews off" };
+    }
+
+    const target = previewUrlFor({
+        url: media.url,
+        kind: media.kind,
+        previewUrl: media.preview_url
+    });
     if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
 
     const fetchImpl = options.fetchImpl ?? fetch;

media-scrape/src/engine.ts

@@ -220,12 +220,24 @@ export class ScrapeEngine {
                 await this.sleep(this.gapMs());
             }
 
-            if (meta.state === "done" && meta.mode === "download") {
-                await this.runDownloads(store, dir, meta);
-            }
+            if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
+                // Keep the job running until post-processing finishes, so a client never
+                // treats a half-archived scrape as complete.
+                meta.state = "running";
+                this.persist(dir, meta);
 
-            if (meta.state === "done" && this.options.previews !== false) {
-                await this.runPreviews(store, dir, meta);
+                if (meta.mode === "download") {
+                    await this.runDownloads(store, dir, meta);
+                }
+
+                if (this.options.previews !== false && !this.stopRequested.has(meta.id)) {
+                    await this.runPreviews(store, dir, meta);
+                }
+
+                if (meta.state === "running") {
+                    meta.state = "done";
+                }
+                this.persist(dir, meta);
             }
         } catch (error) {
             meta.state = "failed";
@@ -271,10 +283,11 @@ export class ScrapeEngine {
         }
     }
 
-    /** Cache small previews for Discord-hosted images. Runs in both modes. */
+    /** Cache small previews. In download mode this includes embed thumbnails. */
     private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
-        const options = this.options.download ?? {};
-        const pending = store.pendingPreviews();
+        const embedPreviews = meta.mode === "download";
+        const options = { ...(this.options.download ?? {}), embedPreviews };
+        const pending = store.pendingPreviews(5000, embedPreviews);
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {

media-scrape/src/store.ts

@@ -294,11 +294,19 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }
 
-    /** Image and GIF rows that still need a cached preview. */
-    pendingPreviews(limit = 5000): MediaRow[] {
+    /**
+     * Rows that still need a cached preview. Attachment/sticker images and GIFs always;
+     * embed thumbnails only when `includeEmbeds` is set (download mode).
+     */
+    pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
+        const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))";
+        const condition = includeEmbeds
+            ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
+            : local;
+
         return this.db
             .prepare(
-                "SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?"
+                `SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
             )
             .all(limit) as unknown as MediaRow[];
     }