Add Discord-hosted preview caching (C11)

media-scrape/src/download.ts

@@ -17,6 +17,8 @@ export interface DownloadOptions {
     maxGapMs?: number;
     /** Retries after the first attempt, for network and 5xx errors only. */
     maxRetries?: number;
+    /** Preview size cap. Default 5 MiB. */
+    previewMaxBytes?: number;
     sleep?: (ms: number) => Promise<void>;
     random?: () => number;
     log?: (line: string) => void;
@@ -120,6 +122,64 @@ export async function downloadOne(
     return { id: media.id, state: "failed", error: lastError };
 }
 
+const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
+
+/**
+ * A small WebP preview URL for a Discord-hosted image or GIF.
+ * Third-party embeds return null: they are never fetched.
+ */
+export function previewUrlFor(media: { url: string; kind: string }): string | null {
+    if (media.kind !== "image" && media.kind !== "gif") return null;
+
+    try {
+        const url = new URL(media.url);
+        if (!DISCORD_HOSTS.has(url.hostname)) return null;
+
+        url.hostname = "media.discordapp.net";
+        url.searchParams.set("width", "320");
+        url.searchParams.set("height", "320");
+        url.searchParams.set("format", "webp");
+        return url.toString();
+    } catch {
+        return null;
+    }
+}
+
+/** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
+export async function downloadPreview(
+    media: Pick<MediaRow, "id" | "url" | "kind">,
+    dir: string,
+    options: DownloadOptions = {}
+): Promise<DownloadOutcome> {
+    const target = previewUrlFor(media);
+    if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
+
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const maxBytes = options.previewMaxBytes ?? 5 * 1024 * 1024;
+
+    try {
+        const response = await fetchImpl(target);
+        if (!response.ok) return { id: media.id, state: "skipped", error: `http ${response.status}` };
+
+        const buffer = Buffer.from(await response.arrayBuffer());
+        if (buffer.byteLength > maxBytes) {
+            return { id: media.id, state: "skipped", error: "too large" };
+        }
+
+        const hash = createHash("sha256").update(buffer).digest("hex");
+        const file = `${hash}.webp`;
+        writeFileSync(join(dir, file), buffer);
+
+        return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
+    } catch (error) {
+        return {
+            id: media.id,
+            state: "failed",
+            error: error instanceof Error ? error.message : String(error)
+        };
+    }
+}
+
 /** The gap before the next file, in milliseconds. */
 export function downloadGapMs(options: DownloadOptions = {}): number {
     const min = options.minGapMs ?? 1000;