Add sort and exclusion filters to the media grid

2026-09-17 11:01:50 +02:00
parent cc14140f34
commit 515af01e84
13 changed files with 561 additions and 96 deletions
@@ -17,3 +17,41 @@ export function candidateScore(candidate: DedupeCandidate): number {
 export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
    return candidateScore(candidate) > candidateScore(current);
 }
+
+/** Ordering for merged candidates. Nulls sort last for the descending keys. */
+export function sortCandidates<T extends { row: { posted_at: string | null; author_name: string | null; file_bytes: number | null; size: number | null; duration_secs: number | null } }>(
+    candidates: T[],
+    sort: "newest" | "oldest" | "user" | "size" | "length"
+): T[] {
+    const number = (value: number | null | undefined) => (value === null || value === undefined ? -1 : value);
+    const text = (value: string | null | undefined) => (value ?? "").toLowerCase();
+
+    // Missing values sort last in both directions.
+    const compareText = (
+        left: string | null | undefined,
+        right: string | null | undefined,
+        direction: 1 | -1
+    ) => {
+        const a = text(left);
+        const b = text(right);
+        if (!a && !b) return 0;
+        if (!a) return 1;
+        if (!b) return -1;
+        return direction * a.localeCompare(b);
+    };
+
+    switch (sort) {
+        case "oldest":
+            return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, 1));
+        case "user":
+            return candidates.sort((a, b) => compareText(a.row.author_name, b.row.author_name, 1));
+        case "size":
+            return candidates.sort(
+                (a, b) => number(b.row.file_bytes ?? b.row.size) - number(a.row.file_bytes ?? a.row.size)
+            );
+        case "length":
+            return candidates.sort((a, b) => number(b.row.duration_secs) - number(a.row.duration_secs));
+        default:
+            return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, -1));
+    }
+}
@@ -4,7 +4,7 @@
 */

 export { extractMedia, type ExtractedMedia } from "./extract.js";
-export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
+export { betterCandidate, candidateScore, sortCandidates, type DedupeCandidate } from "./dedupe.js";
 export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
@@ -53,6 +53,8 @@ export {
    scrapeDir,
    uniqueScrapeId,
    writeScrapeMeta,
-    type MediaRow
+    type MediaFilterOptions,
+    type MediaRow,
+    type MediaSort
 } from "./store.js";
 export * from "./types.js";
@@ -102,6 +102,9 @@ export function uniqueScrapeId(root: string, channelId: string, now = new Date()
    return id;
 }

+/** Global ordering applied once candidates from every scrape are merged. */
+export type MediaSort = "newest" | "oldest" | "user" | "size" | "length";
+
 export interface MediaFilterOptions {
    kind?: MediaKind;
    source?: string;
@@ -110,12 +113,25 @@ export interface MediaFilterOptions {
    cachedOnly?: boolean;
    exts?: string[];
    excludeHosts?: string[];
+    /** URL extension or content type to hide. */
+    excludeExts?: string[];
+    /** Provider hostnames to hide (youtube, twitch, ...). */
+    excludeProviders?: string[];
+    /** Row sources to hide (attachment, sticker, embed). */
+    excludeSources?: string[];
+    /** Bytes; rows with no known size drop out while this is set. */
+    sizeLessThan?: number;
+    sizeGreaterThan?: number;
+    /** Seconds; rows with no known duration drop out while this is set. */
+    lengthLessThan?: number;
+    lengthGreaterThan?: number;
    aspect?: string;
    resolution?: string;
    after?: string;
    before?: string;
    limit?: number;
    offset?: number;
+    sort?: MediaSort;
 }

 export interface MediaRow {
@@ -564,7 +580,9 @@ export class ScrapeStore {
            conditions.push("preview_hash IS NOT NULL");
        }
        if (options.exts?.length) {
-            const clauses = options.exts.map(() => "(lower(url) LIKE ? OR lower(content_type) LIKE ?)");
+            const clauses = options.exts.map(
+                () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
+            );
            conditions.push(`(${clauses.join(" OR ")})`);
            for (const ext of options.exts) {
                values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
@@ -574,6 +592,39 @@ export class ScrapeStore {
            conditions.push("lower(url) NOT LIKE ?");
            values.push(`%${host.toLowerCase()}%`);
        }
+        if (options.excludeExts?.length) {
+            const clauses = options.excludeExts.map(
+                () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
+            );
+            conditions.push(`NOT (${clauses.join(" OR ")})`);
+            for (const ext of options.excludeExts) {
+                values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
+            }
+        }
+        for (const provider of options.excludeProviders ?? []) {
+            conditions.push("lower(COALESCE(provider, '')) NOT LIKE ?");
+            values.push(`%${provider.toLowerCase()}%`);
+        }
+        if (options.excludeSources?.length) {
+            conditions.push(`COALESCE(source, '') NOT IN (${options.excludeSources.map(() => "?").join(", ")})`);
+            values.push(...options.excludeSources);
+        }
+        if (options.sizeLessThan !== undefined) {
+            conditions.push("COALESCE(file_bytes, size) < ?");
+            values.push(options.sizeLessThan);
+        }
+        if (options.sizeGreaterThan !== undefined) {
+            conditions.push("COALESCE(file_bytes, size) > ?");
+            values.push(options.sizeGreaterThan);
+        }
+        if (options.lengthLessThan !== undefined) {
+            conditions.push("duration_secs < ?");
+            values.push(options.lengthLessThan);
+        }
+        if (options.lengthGreaterThan !== undefined) {
+            conditions.push("duration_secs > ?");
+            values.push(options.lengthGreaterThan);
+        }
        if (options.after) {
            conditions.push("posted_at >= ?");
            values.push(options.after);
@@ -22,7 +22,7 @@ import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.
 import { BackfillQueue } from "../src/backfill.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
-import { betterCandidate } from "../src/dedupe.js";
+import { betterCandidate, sortCandidates } from "../src/dedupe.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -1347,6 +1347,91 @@ void (async () => {
        store.close();
    });

+    await test("store exclusion, size, and length filters", () => {
+        const dir = join(root, "exclude-test");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("900", {
+                attachments: [
+                    attachment({ filename: "keep.png", contentType: "image/png", size: 1_000 }),
+                    attachment({ filename: "pixel.jpg", contentType: "image/jpeg", size: 400 } )
+                ],
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=abc",
+                        title: null,
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/embed/abc",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://clips.twitch.tv/xyz",
+                        title: null,
+                        provider: "Twitch",
+                        videoUrl: "https://clips.twitch.tv/xyz",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            }),
+            message("901", {
+                attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 9_000, durationSecs: 42 })]
+            })
+        ]);
+
+        // Insert order decides the ids: 1 keep.png, 2 pixel.jpg, 3 YouTube, 4 Twitch, 5 clip.mp4.
+        const ids = (options: Record<string, unknown>) =>
+            store
+                .queryMedia({ limit: 50, ...options })
+                .map(row => row.id)
+                .sort((a, b) => a - b);
+
+        assert.deepEqual(ids({}), [1, 2, 3, 4, 5]);
+        assert.deepEqual(ids({ excludeProviders: ["youtube"] }), [1, 2, 4, 5]);
+        assert.deepEqual(ids({ excludeSources: ["embed"] }), [1, 2, 5]);
+        assert.deepEqual(ids({ excludeExts: ["png"] }), [2, 3, 4, 5]);
+        assert.deepEqual(ids({ sizeGreaterThan: 9000 }), []);
+        assert.deepEqual(ids({ sizeLessThan: 600 }), [2]);
+        assert.deepEqual(ids({ lengthGreaterThan: 10 }), [5]);
+        // Unknown lengths drop out while a length range is set.
+        assert.deepEqual(ids({ lengthLessThan: 10 }), []);
+        store.close();
+    });
+
+    await test("sortCandidates orders newest, oldest, user, size, and length", () => {
+        const make = (name: string | null, posted: string, size: number | null, duration: number | null, id: number) => ({
+            meta: {} as never,
+            row: {
+                id,
+                posted_at: posted,
+                author_name: name,
+                file_bytes: null,
+                size,
+                duration_secs: duration
+            }
+        });
+
+        const items = [
+            make("zoe", "2026-09-01T00:00:00.000Z", 100, 5, 1),
+            make("amy", "2026-09-03T00:00:00.000Z", 300, 1, 2),
+            make(null, "2026-09-02T00:00:00.000Z", null, null, 3)
+        ];
+
+        const ids = (sort: "newest" | "oldest" | "user" | "size" | "length") =>
+            sortCandidates([...items], sort).map(candidate => candidate.row.id);
+
+        assert.deepEqual(ids("newest"), [2, 3, 1]);
+        assert.deepEqual(ids("oldest"), [1, 3, 2]);
+        assert.deepEqual(ids("user"), [2, 1, 3]);
+        assert.deepEqual(ids("size"), [2, 1, 3]);
+        assert.deepEqual(ids("length"), [1, 2, 3]);
+    });
+
    await test("backfill queue groups scrapes per chat and estimates an ETA", async () => {
        const backend = new FakeBackend([
            {