Add Discord-hosted preview caching (C11)

media-scrape/src/engine.ts

@@ -4,7 +4,7 @@
  */
 
 import { join } from "node:path";
-import { downloadGapMs, downloadOne, type DownloadOptions } from "./download.js";
+import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
 import {
     createScrapeDirs,
     listScrapes,
@@ -48,6 +48,8 @@ export interface ScrapeEngineOptions {
     log?: (line: string) => void;
     /** Download settings for `mode: "download"`. */
     download?: DownloadOptions;
+    /** Cache Discord-hosted image previews after every walk. Default true. */
+    previews?: boolean;
 }
 
 interface QueuedJob {
@@ -221,6 +223,10 @@ export class ScrapeEngine {
             if (meta.state === "done" && meta.mode === "download") {
                 await this.runDownloads(store, dir, meta);
             }
+
+            if (meta.state === "done" && this.options.previews !== false) {
+                await this.runPreviews(store, dir, meta);
+            }
         } catch (error) {
             meta.state = "failed";
             meta.lastError = error instanceof Error ? error.message : String(error);
@@ -265,6 +271,31 @@ export class ScrapeEngine {
         }
     }
 
+    /** Cache small previews for Discord-hosted images. Runs in both modes. */
+    private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+        const options = this.options.download ?? {};
+        const pending = store.pendingPreviews();
+
+        for (const row of pending) {
+            if (this.stopRequested.has(meta.id)) {
+                meta.state = "paused";
+                return;
+            }
+
+            const outcome = await downloadPreview(row, join(dir, "thumbs"), options);
+
+            if (outcome.state === "downloaded" && outcome.hash) {
+                store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
+                meta.previewed = (meta.previewed ?? 0) + 1;
+            } else {
+                store.markPreviewError(row.id, outcome.error ?? outcome.state);
+            }
+
+            this.persist(dir, meta);
+            await this.sleep(downloadGapMs(options));
+        }
+    }
+
     private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise<FetchPage> {
         const pageSize = this.options.pageSize ?? 100;
         const maxRetries = this.options.maxRetries ?? 8;