Add the media downloader with hash storage and local file serving

poc/public/media.js

@@ -170,15 +170,16 @@
     function renderGrid() {
         grid.replaceChildren();
         for (const item of state.items) {
+            const source = item.localUrl || item.url;
             const card = el("a", "vk-media-card");
-            card.href = item.url;
+            card.href = source;
             card.target = "_blank";
             card.rel = "noreferrer";
 
-            if ((item.kind === "image" || item.kind === "gif") && /^https?:/.test(item.url)) {
+            if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) {
                 const img = document.createElement("img");
                 img.loading = "lazy";
-                img.src = item.url;
+                img.src = source;
                 img.alt = item.kind;
                 card.append(img);
             } else {

poc/src/media.ts

@@ -6,6 +6,7 @@
  */
 
 import { rmSync } from "node:fs";
+import { join } from "node:path";
 import {
     dirSizeBytes,
     isNearLimit,
@@ -88,14 +89,34 @@ function formatBytes(bytes: number): string {
 
 export interface MediaItemResult {
     scrapeId: string;
+    itemId: number;
     channelId: string;
     kind: MediaKind;
     url: string;
+    localUrl: string | null;
     provider: string | null;
     downloaded: boolean;
     createdAt: string;
 }
 
+const FILE_CONTENT_TYPES: Record<string, string> = {
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    gif: "image/gif",
+    webp: "image/webp",
+    avif: "image/avif",
+    mp4: "video/mp4",
+    webm: "video/webm",
+    mov: "video/quicktime",
+    mkv: "video/x-matroska",
+    mp3: "audio/mpeg",
+    ogg: "audio/ogg",
+    wav: "audio/wav",
+    m4a: "audio/mp4",
+    flac: "audio/flac"
+};
+
 export interface MediaStatus {
     slopbot: SlopbotStatus;
     scrapes: ScrapeMeta[];
@@ -110,9 +131,19 @@ export class MediaService {
     private botStarting: Promise<void> | null = null;
 
     constructor(private readonly deps: MediaServiceDeps) {
+        const number = (value: string | undefined, fallback: number) => {
+            const parsed = Number(value);
+            return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+        };
+
         this.engine = new ScrapeEngine({
             rootDir: this.rootDir,
-            log: line => this.deps.log?.(line)
+            log: line => this.deps.log?.(line),
+            download: {
+                maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
+                minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
+                maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
+            }
         });
     }
 
@@ -280,6 +311,18 @@ export class MediaService {
         };
     }
 
+    /** Absolute path and content type for a downloaded file, or null. */
+    getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
+        const item = this.getMediaItem(scrapeId, itemId);
+        if (!item?.file_path) return null;
+
+        const extension = item.file_path.split(".").pop()?.toLowerCase() ?? "";
+        return {
+            path: join(this.rootDir, scrapeId, item.file_path),
+            contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
+        };
+    }
+
     /** One media row inside a scrape, with its scrape and channel for context. */
     getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
         const meta = this.engine.get(scrapeId);
@@ -315,9 +358,11 @@ export class MediaService {
                     if (results.length >= limit) break;
                     results.push({
                         scrapeId: meta.id,
+                        itemId: row.id,
                         channelId: meta.channelId,
                         kind: row.kind,
                         url: row.url,
+                        localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null,
                         provider: row.provider,
                         downloaded: row.downloaded === 1,
                         createdAt: row.created_at

poc/src/webui.ts

@@ -754,6 +754,15 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         })
     );
 
+    app.get(
+        "/api/media/:scrapeId/:itemId/file",
+        wrap(async (req, res) => {
+            const file = deps.media.getMediaFile(String(req.params.scrapeId), Number(req.params.itemId));
+            if (!file) throw new HttpError(404, "file not found");
+            res.type(file.contentType).sendFile(file.path);
+        })
+    );
+
     app.get(
         "/api/media/storage",
         wrap(async (_req, res) => {