Add Discord-hosted preview caching (C11)

media-scrape/src/download.ts

@@ -17,6 +17,8 @@ export interface DownloadOptions {
     maxGapMs?: number;
     /** Retries after the first attempt, for network and 5xx errors only. */
     maxRetries?: number;
+    /** Preview size cap. Default 5 MiB. */
+    previewMaxBytes?: number;
     sleep?: (ms: number) => Promise<void>;
     random?: () => number;
     log?: (line: string) => void;
@@ -120,6 +122,64 @@ export async function downloadOne(
     return { id: media.id, state: "failed", error: lastError };
 }
 
+const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
+
+/**
+ * A small WebP preview URL for a Discord-hosted image or GIF.
+ * Third-party embeds return null: they are never fetched.
+ */
+export function previewUrlFor(media: { url: string; kind: string }): string | null {
+    if (media.kind !== "image" && media.kind !== "gif") return null;
+
+    try {
+        const url = new URL(media.url);
+        if (!DISCORD_HOSTS.has(url.hostname)) return null;
+
+        url.hostname = "media.discordapp.net";
+        url.searchParams.set("width", "320");
+        url.searchParams.set("height", "320");
+        url.searchParams.set("format", "webp");
+        return url.toString();
+    } catch {
+        return null;
+    }
+}
+
+/** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
+export async function downloadPreview(
+    media: Pick<MediaRow, "id" | "url" | "kind">,
+    dir: string,
+    options: DownloadOptions = {}
+): Promise<DownloadOutcome> {
+    const target = previewUrlFor(media);
+    if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
+
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const maxBytes = options.previewMaxBytes ?? 5 * 1024 * 1024;
+
+    try {
+        const response = await fetchImpl(target);
+        if (!response.ok) return { id: media.id, state: "skipped", error: `http ${response.status}` };
+
+        const buffer = Buffer.from(await response.arrayBuffer());
+        if (buffer.byteLength > maxBytes) {
+            return { id: media.id, state: "skipped", error: "too large" };
+        }
+
+        const hash = createHash("sha256").update(buffer).digest("hex");
+        const file = `${hash}.webp`;
+        writeFileSync(join(dir, file), buffer);
+
+        return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
+    } catch (error) {
+        return {
+            id: media.id,
+            state: "failed",
+            error: error instanceof Error ? error.message : String(error)
+        };
+    }
+}
+
 /** The gap before the next file, in milliseconds. */
 export function downloadGapMs(options: DownloadOptions = {}): number {
     const min = options.minGapMs ?? 1000;

media-scrape/src/engine.ts

@@ -4,7 +4,7 @@
  */
 
 import { join } from "node:path";
-import { downloadGapMs, downloadOne, type DownloadOptions } from "./download.js";
+import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
 import {
     createScrapeDirs,
     listScrapes,
@@ -48,6 +48,8 @@ export interface ScrapeEngineOptions {
     log?: (line: string) => void;
     /** Download settings for `mode: "download"`. */
     download?: DownloadOptions;
+    /** Cache Discord-hosted image previews after every walk. Default true. */
+    previews?: boolean;
 }
 
 interface QueuedJob {
@@ -221,6 +223,10 @@ export class ScrapeEngine {
             if (meta.state === "done" && meta.mode === "download") {
                 await this.runDownloads(store, dir, meta);
             }
+
+            if (meta.state === "done" && this.options.previews !== false) {
+                await this.runPreviews(store, dir, meta);
+            }
         } catch (error) {
             meta.state = "failed";
             meta.lastError = error instanceof Error ? error.message : String(error);
@@ -265,6 +271,31 @@ export class ScrapeEngine {
         }
     }
 
+    /** Cache small previews for Discord-hosted images. Runs in both modes. */
+    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+        const options = this.options.download ?? {};
+        const pending = store.pendingPreviews();
+
+        for (const row of pending) {
+            if (this.stopRequested.has(meta.id)) {
+                meta.state = "paused";
+                return;
+            }
+
+            const outcome = await downloadPreview(row, join(dir, "thumbs"), options);
+
+            if (outcome.state === "downloaded" && outcome.hash) {
+                store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
+                meta.previewed = (meta.previewed ?? 0) + 1;
+            } else {
+                store.markPreviewError(row.id, outcome.error ?? outcome.state);
+            }
+
+            this.persist(dir, meta);
+            await this.sleep(downloadGapMs(options));
+        }
+    }
+
     private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise<FetchPage> {
         const pageSize = this.options.pageSize ?? 100;
         const maxRetries = this.options.maxRetries ?? 8;

media-scrape/src/index.ts

@@ -5,7 +5,14 @@
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
-export { downloadGapMs, downloadOne, type DownloadOptions, type DownloadOutcome } from "./download.js";
+export {
+    downloadGapMs,
+    downloadOne,
+    downloadPreview,
+    previewUrlFor,
+    type DownloadOptions,
+    type DownloadOutcome
+} from "./download.js";
 export {
     ScrapeEngine,
     type FetchPage,

media-scrape/src/store.ts

@@ -114,6 +114,7 @@ export interface MediaRow {
     preview_bytes: number | null;
     file_path: string | null;
     download_error: string | null;
+    preview_error: string | null;
     created_at: string;
 }
 
@@ -145,6 +146,7 @@ CREATE TABLE IF NOT EXISTS media (
     preview_bytes INTEGER,
     file_path TEXT,
     download_error TEXT,
+    preview_error TEXT,
     created_at TEXT NOT NULL
 );
 
@@ -178,6 +180,7 @@ export class ScrapeStore {
         if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
         if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
         if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
+        if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
     }
 
     close(): void {
@@ -291,6 +294,25 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }
 
+    /** Image and GIF rows that still need a cached preview. */
+    pendingPreviews(limit = 5000): MediaRow[] {
+        return this.db
+            .prepare(
+                "SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?"
+            )
+            .all(limit) as unknown as MediaRow[];
+    }
+
+    markPreview(id: number, preview: { hash: string; bytes: number }): void {
+        this.db
+            .prepare("UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_error = NULL WHERE id = ?")
+            .run(preview.hash, preview.bytes, id);
+    }
+
+    markPreviewError(id: number, error: string): void {
+        this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
+    }
+
     getMedia(id: number): MediaRow | null {
         const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
             | MediaRow

media-scrape/src/types.ts

@@ -87,6 +87,8 @@ export interface ScrapeMeta {
     downloaded?: number;
     /** Files that failed after retries; skipped URLs are not counted here. */
     downloadErrors?: number;
+    /** Cached image previews. */
+    previewed?: number;
 }
 
 export function emptyMediaCounts(): Record<MediaKind, number> {

media-scrape/test/run.ts

@@ -8,7 +8,7 @@ import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { downloadOne } from "../src/download.js";
+import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
@@ -85,6 +85,7 @@ function engineFor(root: string, backend: ScrapeBackend, extra: Record<string, u
         sleep: async () => {},
         random: () => 0,
         log: () => {},
+        previews: false,
         ...extra
     });
 }
@@ -366,6 +367,7 @@ void (async () => {
             sleep: async () => {},
             random: () => 0,
             log: () => {},
+            previews: false,
             download: {
                 fetchImpl: async () =>
                     new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
@@ -394,6 +396,77 @@ void (async () => {
         assert.equal(existsSync(join(root, started.id, row.file_path)), true);
     });
 
+    await test("preview urls are Discord-only resizes", () => {
+        assert.equal(
+            previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
+            "https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
+        );
+        assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
+        assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null);
+    });
+
+    await test("preview downloads are written by hash", async () => {
+        const dir = join(root, "preview-test");
+        mkdirSync(dir, { recursive: true });
+
+        const outcome = await downloadPreview(
+            { id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" },
+            dir,
+            {
+                fetchImpl: async () =>
+                    new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } })
+            }
+        );
+
+        assert.equal(outcome.state, "downloaded");
+        assert.equal(outcome.path, `${outcome.hash}.webp`);
+        assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
+    });
+
+    await test("engine caches previews in links mode", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("600", {
+                        attachments: [attachment({ filename: "prev.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: true,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const meta = engine.get(started.id);
+        assert.equal(meta?.state, "done");
+        assert.equal(meta?.previewed, 1);
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "image" })[0];
+        assert.ok(row.preview_hash);
+        assert.ok((row.preview_bytes ?? 0) > 0);
+        store.close();
+        assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true);
+    });
+
     await test("retention marks the oldest completed scrapes up to the limit", () => {
         const scrapes = [
             { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },