Add the media downloader with hash storage and local file serving

media-scrape/src/download.ts

@@ -0,0 +1,129 @@
+/*
+ * Attachment downloader for the media archive. Serial, hash-addressed, conservative.
+ * Rules: docs/media-scraper.md. Third-party embeds are never targets here.
+ */
+
+import { createHash } from "node:crypto";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { MediaRow } from "./store.js";
+
+export interface DownloadOptions {
+    fetchImpl?: typeof fetch;
+    /** Skip files larger than this. Default 50 MiB. */
+    maxFileBytes?: number;
+    /** Gap between files. Default 1000-3000 ms. */
+    minGapMs?: number;
+    maxGapMs?: number;
+    /** Retries after the first attempt, for network and 5xx errors only. */
+    maxRetries?: number;
+    sleep?: (ms: number) => Promise<void>;
+    random?: () => number;
+    log?: (line: string) => void;
+}
+
+export interface DownloadOutcome {
+    id: number;
+    state: "downloaded" | "skipped" | "failed";
+    path?: string;
+    hash?: string;
+    bytes?: number;
+    error?: string;
+}
+
+/** 4xx that will not improve on retry: the signed URL expired or the file is gone. */
+const SKIP_STATUS = new Set([400, 401, 403, 404, 410]);
+
+const TYPE_EXTENSIONS: Record<string, string> = {
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/gif": "gif",
+    "image/webp": "webp",
+    "image/avif": "avif",
+    "video/mp4": "mp4",
+    "video/webm": "webm",
+    "video/quicktime": "mov",
+    "video/x-matroska": "mkv",
+    "audio/mpeg": "mp3",
+    "audio/ogg": "ogg",
+    "audio/wav": "wav",
+    "audio/x-wav": "wav",
+    "audio/mp4": "m4a",
+    "audio/flac": "flac"
+};
+
+function extensionFor(contentType: string | null, url: string): string {
+    const type = (contentType ?? "").split(";")[0].trim().toLowerCase();
+    if (TYPE_EXTENSIONS[type]) return TYPE_EXTENSIONS[type];
+
+    try {
+        const match = /\.([a-z0-9]{2,5})$/i.exec(new URL(url).pathname);
+        if (match) return match[1].toLowerCase();
+    } catch {
+        // fall through
+    }
+    return "bin";
+}
+
+const defaultSleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms));
+
+/** Download one media row into `dir`. The file name is the SHA-256 of the bytes. */
+export async function downloadOne(
+    media: Pick<MediaRow, "id" | "url" | "kind">,
+    dir: string,
+    options: DownloadOptions = {}
+): Promise<DownloadOutcome> {
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const maxBytes = options.maxFileBytes ?? 50 * 1024 * 1024;
+    const maxRetries = options.maxRetries ?? 2;
+    const sleep = options.sleep ?? defaultSleep;
+
+    let lastError = "download failed";
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const response = await fetchImpl(media.url);
+
+            if (!response.ok) {
+                if (SKIP_STATUS.has(response.status)) {
+                    return { id: media.id, state: "skipped", error: `http ${response.status}` };
+                }
+                lastError = `http ${response.status}`;
+            } else {
+                const declared = Number(response.headers.get("content-length") ?? 0);
+                if (declared > maxBytes) {
+                    return { id: media.id, state: "skipped", error: `too large (${declared} bytes)` };
+                }
+
+                const buffer = Buffer.from(await response.arrayBuffer());
+                if (buffer.byteLength > maxBytes) {
+                    return { id: media.id, state: "skipped", error: "too large" };
+                }
+
+                const hash = createHash("sha256").update(buffer).digest("hex");
+                const extension = extensionFor(response.headers.get("content-type"), media.url);
+                const file = `${hash}.${extension}`;
+                writeFileSync(join(dir, file), buffer);
+
+                return { id: media.id, state: "downloaded", path: join("media", file), hash, bytes: buffer.byteLength };
+            }
+        } catch (error) {
+            lastError = error instanceof Error ? error.message : String(error);
+        }
+
+        if (attempt < maxRetries) {
+            options.log?.(`download ${media.id} failed (${lastError}), retrying`);
+            await sleep(2 ** attempt * 1000 + 1000);
+        }
+    }
+
+    return { id: media.id, state: "failed", error: lastError };
+}
+
+/** The gap before the next file, in milliseconds. */
+export function downloadGapMs(options: DownloadOptions = {}): number {
+    const min = options.minGapMs ?? 1000;
+    const max = Math.max(min, options.maxGapMs ?? 3000);
+    const random = options.random ?? Math.random;
+    return min + Math.floor(random() * (max - min + 1));
+}