Add links-mode embed thumbnails, item metadata, and media filters

media-scrape/test/run.ts

@@ -12,6 +12,7 @@ import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
 import type { CoreAttachment, CoreMessage } from "../src/types.js";
@@ -196,6 +197,7 @@ void (async () => {
         assert.equal(meta?.messages, 3);
         assert.equal(meta?.cursor, "99");
         assert.equal(meta?.media.image, 1);
+        assert.ok(meta?.finishedAt);
         assert.equal(backend.calls[0].limit, 2);
         assert.equal(backend.calls[1].before, "100");
 
@@ -209,6 +211,9 @@ void (async () => {
         assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
         assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
         assert.equal(stored.storedBytesByKind().image, 0);
+        const row = stored.queryMedia({ kind: "image" })[0];
+        assert.equal(row.author_name, "tester");
+        assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
         stored.close();
 
         assert.equal(listScrapes(root).length, 1);
@@ -405,14 +410,30 @@ void (async () => {
         assert.equal(existsSync(join(root, started.id, row.file_path)), true);
     });
 
-    await test("store filters by source and kind", () => {
+    await test("aspect and resolution presets have fixed buckets", () => {
+        assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
+        assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
+        assert.equal(aspectCondition("nope"), null);
+        assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
+        assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
+        assert.equal(resolutionCondition("nope"), null);
+    });
+
+    await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
         const dir = join(root, "store-source");
         mkdirSync(dir, { recursive: true });
 
         const store = new ScrapeStore(dir);
         store.insertPage([
             message("700", {
-                attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
+                attachments: [
+                    attachment({
+                        filename: "pic.png",
+                        contentType: "image/png",
+                        width: 640,
+                        height: 360
+                    })
+                ],
                 embeds: [
                     {
                         type: "video",
@@ -432,6 +453,59 @@ void (async () => {
         assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
         assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
         assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
+
+        assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
+        assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
+        assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
+        assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
+        assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
+        assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
+        assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
+        assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
+
+        const [wide] = store.queryMedia({ kind: "image" });
+        assert.equal(wide.aspect_ratio, 1.778);
+        store.close();
+    });
+
+    await test("engine probes dimensions when the metadata lacks them", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("701", {
+                        attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            probe: async () => ({ width: 1080, height: 1920 }),
+            download: {
+                fetchImpl: async () =>
+                    new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
+        await engine.processQueue();
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "image" })[0];
+        assert.equal(row.width, 1080);
+        assert.equal(row.height, 1920);
+        assert.equal(row.aspect_ratio, 0.563);
         store.close();
     });
 
@@ -502,7 +576,7 @@ void (async () => {
         assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
     });
 
-    await test("engine caches previews in links mode and skips embeds", async () => {
+    await test("engine caches previews in links mode, including embed thumbnails", async () => {
         const backend = new FakeBackend([
             {
                 messages: [
@@ -551,8 +625,9 @@ void (async () => {
 
         const meta = engine.get(started.id);
         assert.equal(meta?.state, "done");
-        assert.equal(meta?.previewed, 1);
-        assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
+        assert.equal(meta?.previewed, 2);
+        assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
+        assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
 
         const store = new ScrapeStore(join(root, started.id));
         const row = store.queryMedia({ kind: "image" })[0];