Add sort and exclusion filters to the media grid

2026-09-17 11:01:50 +02:00
parent cc14140f34
commit 515af01e84
13 changed files with 561 additions and 96 deletions
@@ -22,7 +22,7 @@ import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.
 import { BackfillQueue } from "../src/backfill.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
-import { betterCandidate } from "../src/dedupe.js";
+import { betterCandidate, sortCandidates } from "../src/dedupe.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -1347,6 +1347,91 @@ void (async () => {
        store.close();
    });

+    await test("store exclusion, size, and length filters", () => {
+        const dir = join(root, "exclude-test");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("900", {
+                attachments: [
+                    attachment({ filename: "keep.png", contentType: "image/png", size: 1_000 }),
+                    attachment({ filename: "pixel.jpg", contentType: "image/jpeg", size: 400 } )
+                ],
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=abc",
+                        title: null,
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/embed/abc",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://clips.twitch.tv/xyz",
+                        title: null,
+                        provider: "Twitch",
+                        videoUrl: "https://clips.twitch.tv/xyz",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            }),
+            message("901", {
+                attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 9_000, durationSecs: 42 })]
+            })
+        ]);
+
+        // Insert order decides the ids: 1 keep.png, 2 pixel.jpg, 3 YouTube, 4 Twitch, 5 clip.mp4.
+        const ids = (options: Record<string, unknown>) =>
+            store
+                .queryMedia({ limit: 50, ...options })
+                .map(row => row.id)
+                .sort((a, b) => a - b);
+
+        assert.deepEqual(ids({}), [1, 2, 3, 4, 5]);
+        assert.deepEqual(ids({ excludeProviders: ["youtube"] }), [1, 2, 4, 5]);
+        assert.deepEqual(ids({ excludeSources: ["embed"] }), [1, 2, 5]);
+        assert.deepEqual(ids({ excludeExts: ["png"] }), [2, 3, 4, 5]);
+        assert.deepEqual(ids({ sizeGreaterThan: 9000 }), []);
+        assert.deepEqual(ids({ sizeLessThan: 600 }), [2]);
+        assert.deepEqual(ids({ lengthGreaterThan: 10 }), [5]);
+        // Unknown lengths drop out while a length range is set.
+        assert.deepEqual(ids({ lengthLessThan: 10 }), []);
+        store.close();
+    });
+
+    await test("sortCandidates orders newest, oldest, user, size, and length", () => {
+        const make = (name: string | null, posted: string, size: number | null, duration: number | null, id: number) => ({
+            meta: {} as never,
+            row: {
+                id,
+                posted_at: posted,
+                author_name: name,
+                file_bytes: null,
+                size,
+                duration_secs: duration
+            }
+        });
+
+        const items = [
+            make("zoe", "2026-09-01T00:00:00.000Z", 100, 5, 1),
+            make("amy", "2026-09-03T00:00:00.000Z", 300, 1, 2),
+            make(null, "2026-09-02T00:00:00.000Z", null, null, 3)
+        ];
+
+        const ids = (sort: "newest" | "oldest" | "user" | "size" | "length") =>
+            sortCandidates([...items], sort).map(candidate => candidate.row.id);
+
+        assert.deepEqual(ids("newest"), [2, 3, 1]);
+        assert.deepEqual(ids("oldest"), [1, 3, 2]);
+        assert.deepEqual(ids("user"), [2, 1, 3]);
+        assert.deepEqual(ids("size"), [2, 1, 3]);
+        assert.deepEqual(ids("length"), [1, 2, 3]);
+    });
+
    await test("backfill queue groups scrapes per chat and estimates an ETA", async () => {
        const backend = new FakeBackend([
            {