Add the media downloader with hash storage and local file serving

media-scrape/src/engine.ts

@@ -4,6 +4,7 @@
  */
 
 import { join } from "node:path";
+import { downloadGapMs, downloadOne, type DownloadOptions } from "./download.js";
 import {
     createScrapeDirs,
     listScrapes,
@@ -45,6 +46,8 @@ export interface ScrapeEngineOptions {
     random?: () => number;
     now?: () => number;
     log?: (line: string) => void;
+    /** Download settings for `mode: "download"`. */
+    download?: DownloadOptions;
 }
 
 interface QueuedJob {
@@ -214,6 +217,10 @@ export class ScrapeEngine {
 
                 await this.sleep(this.gapMs());
             }
+
+            if (meta.state === "done" && meta.mode === "download") {
+                await this.runDownloads(store, dir, meta);
+            }
         } catch (error) {
             meta.state = "failed";
             meta.lastError = error instanceof Error ? error.message : String(error);
@@ -225,6 +232,39 @@ export class ScrapeEngine {
         }
     }
 
+    /** Fetch attachment bytes for a finished walk. Serial, hash-addressed, resumable. */
+    private async runDownloads(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
+        const options = this.options.download ?? {};
+        const pending = store.pendingDownloads();
+
+        for (const row of pending) {
+            if (this.stopRequested.has(meta.id)) {
+                meta.state = "paused";
+                return;
+            }
+
+            const outcome = await downloadOne(row, join(dir, "media"), {
+                ...options,
+                log: line => this.log(`${meta.id}: ${line}`)
+            });
+
+            if (outcome.state === "downloaded" && outcome.path && outcome.hash) {
+                store.markDownloaded(row.id, {
+                    path: outcome.path,
+                    hash: outcome.hash,
+                    bytes: outcome.bytes ?? 0
+                });
+                meta.downloaded = (meta.downloaded ?? 0) + 1;
+            } else {
+                store.markDownloadError(row.id, outcome.error ?? outcome.state);
+                if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
+            }
+
+            this.persist(dir, meta);
+            await this.sleep(downloadGapMs(options));
+        }
+    }
+
     private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise<FetchPage> {
         const pageSize = this.options.pageSize ?? 100;
         const maxRetries = this.options.maxRetries ?? 8;