Add links-mode embed thumbnails, item metadata, and media filters

media-scrape/src/engine.ts

@@ -4,7 +4,13 @@
  */
 
 import { join } from "node:path";
-import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
+import {
+    downloadGapMs,
+    downloadOne,
+    downloadPreview,
+    probeDimensions,
+    type DownloadOptions
+} from "./download.js";
 import {
     createScrapeDirs,
     listScrapes,
@@ -50,6 +56,8 @@ export interface ScrapeEngineOptions {
     download?: DownloadOptions;
     /** Cache Discord-hosted image previews after every walk. Default true. */
     previews?: boolean;
+    /** Read dimensions from a downloaded file. Defaults to ffprobe. */
+    probe?: (path: string) => Promise<{ width: number; height: number } | null>;
 }
 
 interface QueuedJob {
@@ -244,6 +252,9 @@ export class ScrapeEngine {
             meta.lastError = error instanceof Error ? error.message : String(error);
             this.log(`${meta.id}: failed: ${meta.lastError}`);
         } finally {
+            if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
+                meta.finishedAt = new Date(this.now()).toISOString();
+            }
             this.persist(dir, meta);
             store.close();
             this.active.set(meta.id, meta);
@@ -273,6 +284,11 @@ export class ScrapeEngine {
                     bytes: outcome.bytes ?? 0
                 });
                 meta.downloaded = (meta.downloaded ?? 0) + 1;
+
+                if (row.width === null || row.height === null) {
+                    const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
+                    if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
+                }
             } else {
                 store.markDownloadError(row.id, outcome.error ?? outcome.state);
                 if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
@@ -283,11 +299,10 @@ export class ScrapeEngine {
         }
     }
 
-    /** Cache small previews. In download mode this includes embed thumbnails. */
+    /** Cache small previews. Embed thumbnails are cached in both modes. */
     private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
-        const embedPreviews = meta.mode === "download";
-        const options = { ...(this.options.download ?? {}), embedPreviews };
-        const pending = store.pendingPreviews(5000, embedPreviews);
+        const options = { ...(this.options.download ?? {}), embedPreviews: true };
+        const pending = store.pendingPreviews(5000, true);
 
         for (const row of pending) {
             if (this.stopRequested.has(meta.id)) {