Cache embed thumbnails in download mode

media-scrape/src/download.ts

@@ -19,6 +19,8 @@ export interface DownloadOptions {
     maxRetries?: number;
     /** Preview size cap. Default 5 MiB. */
     previewMaxBytes?: number;
+    /** Cache third-party embed thumbnails. Download mode only. */
+    embedPreviews?: boolean;
     sleep?: (ms: number) => Promise<void>;
     random?: () => number;
     log?: (line: string) => void;
@@ -125,20 +127,30 @@ export async function downloadOne(
 const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
 
 /**
- * A small WebP preview URL for a Discord-hosted image or GIF.
- * Third-party embeds return null: they are never fetched.
+ * The preview target for a media row.
+ *
+ * Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
+ * Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
+ * Returns null when the row has no preview source.
  */
-export function previewUrlFor(media: { url: string; kind: string }): string | null {
-    if (media.kind !== "image" && media.kind !== "gif") return null;
+export function previewUrlFor(media: {
+    url: string;
+    kind: string;
+    previewUrl?: string | null;
+}): string | null {
+    const candidate =
+        media.previewUrl ??
+        (media.kind === "image" || media.kind === "gif" ? media.url : null);
+    if (!candidate) return null;
 
     try {
-        const url = new URL(media.url);
-        if (!DISCORD_HOSTS.has(url.hostname)) return null;
-
-        url.hostname = "media.discordapp.net";
-        url.searchParams.set("width", "320");
-        url.searchParams.set("height", "320");
-        url.searchParams.set("format", "webp");
+        const url = new URL(candidate);
+        if (DISCORD_HOSTS.has(url.hostname)) {
+            url.hostname = "media.discordapp.net";
+            url.searchParams.set("width", "320");
+            url.searchParams.set("height", "320");
+            url.searchParams.set("format", "webp");
+        }
         return url.toString();
     } catch {
         return null;
@@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu
 
 /** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
 export async function downloadPreview(
-    media: Pick<MediaRow, "id" | "url" | "kind">,
+    media: Pick<MediaRow, "id" | "url" | "kind" | "source" | "preview_url">,
     dir: string,
     options: DownloadOptions = {}
 ): Promise<DownloadOutcome> {
-    const target = previewUrlFor(media);
+    // Third-party thumbnails only in download mode; the full embed is never fetched.
+    if (media.source === "embed" && options.embedPreviews !== true) {
+        return { id: media.id, state: "skipped", error: "embed previews off" };
+    }
+
+    const target = previewUrlFor({
+        url: media.url,
+        kind: media.kind,
+        previewUrl: media.preview_url
+    });
     if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
 
     const fetchImpl = options.fetchImpl ?? fetch;