Add sort and exclusion filters to the media grid

media-scrape/src/dedupe.ts

@@ -17,3 +17,41 @@ export function candidateScore(candidate: DedupeCandidate): number {
 export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
     return candidateScore(candidate) > candidateScore(current);
 }
+
+/** Ordering for merged candidates. Nulls sort last for the descending keys. */
+export function sortCandidates<T extends { row: { posted_at: string | null; author_name: string | null; file_bytes: number | null; size: number | null; duration_secs: number | null } }>(
+    candidates: T[],
+    sort: "newest" | "oldest" | "user" | "size" | "length"
+): T[] {
+    const number = (value: number | null | undefined) => (value === null || value === undefined ? -1 : value);
+    const text = (value: string | null | undefined) => (value ?? "").toLowerCase();
+
+    // Missing values sort last in both directions.
+    const compareText = (
+        left: string | null | undefined,
+        right: string | null | undefined,
+        direction: 1 | -1
+    ) => {
+        const a = text(left);
+        const b = text(right);
+        if (!a && !b) return 0;
+        if (!a) return 1;
+        if (!b) return -1;
+        return direction * a.localeCompare(b);
+    };
+
+    switch (sort) {
+        case "oldest":
+            return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, 1));
+        case "user":
+            return candidates.sort((a, b) => compareText(a.row.author_name, b.row.author_name, 1));
+        case "size":
+            return candidates.sort(
+                (a, b) => number(b.row.file_bytes ?? b.row.size) - number(a.row.file_bytes ?? a.row.size)
+            );
+        case "length":
+            return candidates.sort((a, b) => number(b.row.duration_secs) - number(a.row.duration_secs));
+        default:
+            return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, -1));
+    }
+}

media-scrape/src/index.ts

@@ -4,7 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
-export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
+export { betterCandidate, candidateScore, sortCandidates, type DedupeCandidate } from "./dedupe.js";
 export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
@@ -53,6 +53,8 @@ export {
     scrapeDir,
     uniqueScrapeId,
     writeScrapeMeta,
-    type MediaRow
+    type MediaFilterOptions,
+    type MediaRow,
+    type MediaSort
 } from "./store.js";
 export * from "./types.js";

media-scrape/src/store.ts

@@ -102,6 +102,9 @@ export function uniqueScrapeId(root: string, channelId: string, now = new Date()
     return id;
 }
 
+/** Global ordering applied once candidates from every scrape are merged. */
+export type MediaSort = "newest" | "oldest" | "user" | "size" | "length";
+
 export interface MediaFilterOptions {
     kind?: MediaKind;
     source?: string;
@@ -110,12 +113,25 @@ export interface MediaFilterOptions {
     cachedOnly?: boolean;
     exts?: string[];
     excludeHosts?: string[];
+    /** URL extension or content type to hide. */
+    excludeExts?: string[];
+    /** Provider hostnames to hide (youtube, twitch, ...). */
+    excludeProviders?: string[];
+    /** Row sources to hide (attachment, sticker, embed). */
+    excludeSources?: string[];
+    /** Bytes; rows with no known size drop out while this is set. */
+    sizeLessThan?: number;
+    sizeGreaterThan?: number;
+    /** Seconds; rows with no known duration drop out while this is set. */
+    lengthLessThan?: number;
+    lengthGreaterThan?: number;
     aspect?: string;
     resolution?: string;
     after?: string;
     before?: string;
     limit?: number;
     offset?: number;
+    sort?: MediaSort;
 }
 
 export interface MediaRow {
@@ -564,7 +580,9 @@ export class ScrapeStore {
             conditions.push("preview_hash IS NOT NULL");
         }
         if (options.exts?.length) {
-            const clauses = options.exts.map(() => "(lower(url) LIKE ? OR lower(content_type) LIKE ?)");
+            const clauses = options.exts.map(
+                () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
+            );
             conditions.push(`(${clauses.join(" OR ")})`);
             for (const ext of options.exts) {
                 values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
@@ -574,6 +592,39 @@ export class ScrapeStore {
             conditions.push("lower(url) NOT LIKE ?");
             values.push(`%${host.toLowerCase()}%`);
         }
+        if (options.excludeExts?.length) {
+            const clauses = options.excludeExts.map(
+                () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)"
+            );
+            conditions.push(`NOT (${clauses.join(" OR ")})`);
+            for (const ext of options.excludeExts) {
+                values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`);
+            }
+        }
+        for (const provider of options.excludeProviders ?? []) {
+            conditions.push("lower(COALESCE(provider, '')) NOT LIKE ?");
+            values.push(`%${provider.toLowerCase()}%`);
+        }
+        if (options.excludeSources?.length) {
+            conditions.push(`COALESCE(source, '') NOT IN (${options.excludeSources.map(() => "?").join(", ")})`);
+            values.push(...options.excludeSources);
+        }
+        if (options.sizeLessThan !== undefined) {
+            conditions.push("COALESCE(file_bytes, size) < ?");
+            values.push(options.sizeLessThan);
+        }
+        if (options.sizeGreaterThan !== undefined) {
+            conditions.push("COALESCE(file_bytes, size) > ?");
+            values.push(options.sizeGreaterThan);
+        }
+        if (options.lengthLessThan !== undefined) {
+            conditions.push("duration_secs < ?");
+            values.push(options.lengthLessThan);
+        }
+        if (options.lengthGreaterThan !== undefined) {
+            conditions.push("duration_secs > ?");
+            values.push(options.lengthGreaterThan);
+        }
         if (options.after) {
             conditions.push("posted_at >= ?");
             values.push(options.after);