From 515af01e84eef29dded4d0dfd7be7952c6f575b0 Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 11:01:50 +0200 Subject: [PATCH] Add sort and exclusion filters to the media grid --- CURRENT.md | 9 ++ docs/media.md | 7 +- docs/webui.md | 10 +- media-scrape/src/dedupe.ts | 38 +++++++ media-scrape/src/index.ts | 6 +- media-scrape/src/store.ts | 53 ++++++++- media-scrape/test/run.ts | 87 +++++++++++++- poc/public/media.css | 23 ++++ poc/public/media.js | 219 +++++++++++++++++++++++++++++++++++- poc/src/control-bridge.ts | 33 ++++++ poc/src/media.ts | 35 +++++- poc/src/webui.ts | 134 +++++++++------------- poc/test/media-ui-smoke.mjs | 3 + 13 files changed, 561 insertions(+), 96 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index a78b94f..d7650ed 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,15 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C46 Sort and exclusion filters. The Advanced drawer gained a sort select (newest, oldest, + user, size, length; global over the merged URL-dedupe stream) and a **Do not include** group + with nested collapsibles: sources (Twitch, YouTube, Discord, Tenor, Giphy, embeds, attachments, + stickers), file types, size (less/greater than with B/KB/MB/GB units), and length (seconds). + Store, `/api/media` + `/api/media/stats` (shared parser), and MCP `query_media` all carry the + new params; size and length ranges drop rows with unknown values, and the ext matching now + coalesces NULL content types so a `NOT (...)` cannot swallow rows. Also hid the media filter bar + and drawer on the Bot tab. Unit tests for every exclusion, all five sorts (nulls last), and + smoke checks for the hidden bar on the Bot tab. - [x] C45 Streaming media grid. The grid rebuilt every card on each appended page, which made filtered browsing flash, and the `Load more` trigger only worked once the grid could already scroll, so short filtered pages dead-stopped at ~11 cards. Now pages append incrementally diff --git a/docs/media.md b/docs/media.md index 2f73a39..240d63b 100644 --- a/docs/media.md +++ b/docs/media.md @@ -147,8 +147,11 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun - **Media** tab: the bar carries the common filters — kind chips, source kind, server/DM origin, and the multi-select People popover. The **Advanced search** drawer has aspect ratio, resolution, - source channel, posted-date range, file-type chips, cached-only, and a default-on hide for 4chan - links (they 404 almost always), with **Apply filters** / **Reset** in the bar. + source channel, posted-date range, file-type chips, cached-only, a default-on hide for 4chan + links (they 404 almost always), a sort select (newest, oldest, user, size, length), and a **Do not + include** group with nested collapsibles for sources, file types, size (with B/KB/MB/GB units), + and length (seconds), with **Apply filters** / **Reset** in the bar. Size and length ranges drop + rows with no known value. The bar and drawer show only on the Media tab. Nothing renders until **Load media** is pressed. A **Metadata** dropdown toggles which fields show in the small banner under each tile (User, Type, Date, Source, Resolution, Length). Video tiles carry a translucent play icon top-right. Discord serves no thumbnails for uploaded videos, diff --git a/docs/webui.md b/docs/webui.md index ba0f374..6bd6980 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -130,8 +130,12 @@ box live in Chat, not Call, and the connection stats are not chips yet. (attachments/stickers/embeds), origin (server channels or DMs), and a multi-select **People** popover. The **Advanced search** drawer holds aspect ratio, resolution, source channel, posted-date range, file-type chips (video mp4/webm/mov/mkv, image png/jpg/gif/webp/avif, audio), - **Show only cached previews**, and a default-on **Hide 4chan links (usually dead)**, plus the - card-size slider. Filters do not + **Show only cached previews**, a default-on **Hide 4chan links (usually dead)**, and a + **Do not include** group with nested collapsibles for sources (Twitch, YouTube, Discord, Tenor, + Giphy, embeds, attachments, stickers), file types, size (less/greater than, with B/KB/MB/GB + units), and length (less/greater than, seconds). A sort select orders the grid by newest, + oldest, user, size, or length. Size and length ranges drop rows with no known value. The filter + bar and the drawer are hidden on the Bot tab. Filters do not apply until **Apply filters** is pressed; **Reset** clears every filter and the zoom, unloads the grid, and brings back the initial Load media prompt. - A **Card size** slider scales tiles from two-thirds of the old size up to one image per row; the @@ -281,7 +285,7 @@ replaceable. | POST | `/api/audio/stop` | none | Stop speech and clear the queue | | POST | `/api/audio/capture` | `start` | Start or stop capture | | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | -| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?` (comma list), `origin?` (`guild`/`dm`), `cachedOnly?`, `ext?` (comma list), `excludeHosts?` (comma list), `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`, `previewError`, `animUrl`; `localUrl`/`thumbUrl` when a file or preview exists | +| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?` (comma list), `origin?` (`guild`/`dm`), `cachedOnly?`, `ext?` (comma list), `excludeHosts?` (comma list), `excludeExt?`, `excludeProviders?`, `excludeSources?` (comma lists), `sizeLt?`/`sizeGt?` (bytes), `lenLt?`/`lenGt?` (seconds), `sort?` (`newest` default, `oldest`, `user`, `size`, `length`), `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`, `previewError`, `animUrl`; `localUrl`/`thumbUrl` when a file or preview exists | | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | | GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) | | GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range`. Non-media upstream responses (provider pages) return `415` | diff --git a/media-scrape/src/dedupe.ts b/media-scrape/src/dedupe.ts index ffbb1f7..6348de8 100644 --- a/media-scrape/src/dedupe.ts +++ b/media-scrape/src/dedupe.ts @@ -17,3 +17,41 @@ export function candidateScore(candidate: DedupeCandidate): number { export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean { return candidateScore(candidate) > candidateScore(current); } + +/** Ordering for merged candidates. Nulls sort last for the descending keys. */ +export function sortCandidates( + candidates: T[], + sort: "newest" | "oldest" | "user" | "size" | "length" +): T[] { + const number = (value: number | null | undefined) => (value === null || value === undefined ? -1 : value); + const text = (value: string | null | undefined) => (value ?? "").toLowerCase(); + + // Missing values sort last in both directions. + const compareText = ( + left: string | null | undefined, + right: string | null | undefined, + direction: 1 | -1 + ) => { + const a = text(left); + const b = text(right); + if (!a && !b) return 0; + if (!a) return 1; + if (!b) return -1; + return direction * a.localeCompare(b); + }; + + switch (sort) { + case "oldest": + return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, 1)); + case "user": + return candidates.sort((a, b) => compareText(a.row.author_name, b.row.author_name, 1)); + case "size": + return candidates.sort( + (a, b) => number(b.row.file_bytes ?? b.row.size) - number(a.row.file_bytes ?? a.row.size) + ); + case "length": + return candidates.sort((a, b) => number(b.row.duration_secs) - number(a.row.duration_secs)); + default: + return candidates.sort((a, b) => compareText(a.row.posted_at, b.row.posted_at, -1)); + } +} diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index e7a0e20..f9e4406 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -4,7 +4,7 @@ */ export { extractMedia, type ExtractedMedia } from "./extract.js"; -export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js"; +export { betterCandidate, candidateScore, sortCandidates, type DedupeCandidate } from "./dedupe.js"; export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js"; @@ -53,6 +53,8 @@ export { scrapeDir, uniqueScrapeId, writeScrapeMeta, - type MediaRow + type MediaFilterOptions, + type MediaRow, + type MediaSort } from "./store.js"; export * from "./types.js"; diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index f7b0b24..b0b0348 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -102,6 +102,9 @@ export function uniqueScrapeId(root: string, channelId: string, now = new Date() return id; } +/** Global ordering applied once candidates from every scrape are merged. */ +export type MediaSort = "newest" | "oldest" | "user" | "size" | "length"; + export interface MediaFilterOptions { kind?: MediaKind; source?: string; @@ -110,12 +113,25 @@ export interface MediaFilterOptions { cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; + /** URL extension or content type to hide. */ + excludeExts?: string[]; + /** Provider hostnames to hide (youtube, twitch, ...). */ + excludeProviders?: string[]; + /** Row sources to hide (attachment, sticker, embed). */ + excludeSources?: string[]; + /** Bytes; rows with no known size drop out while this is set. */ + sizeLessThan?: number; + sizeGreaterThan?: number; + /** Seconds; rows with no known duration drop out while this is set. */ + lengthLessThan?: number; + lengthGreaterThan?: number; aspect?: string; resolution?: string; after?: string; before?: string; limit?: number; offset?: number; + sort?: MediaSort; } export interface MediaRow { @@ -564,7 +580,9 @@ export class ScrapeStore { conditions.push("preview_hash IS NOT NULL"); } if (options.exts?.length) { - const clauses = options.exts.map(() => "(lower(url) LIKE ? OR lower(content_type) LIKE ?)"); + const clauses = options.exts.map( + () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)" + ); conditions.push(`(${clauses.join(" OR ")})`); for (const ext of options.exts) { values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`); @@ -574,6 +592,39 @@ export class ScrapeStore { conditions.push("lower(url) NOT LIKE ?"); values.push(`%${host.toLowerCase()}%`); } + if (options.excludeExts?.length) { + const clauses = options.excludeExts.map( + () => "(lower(url) LIKE ? OR lower(COALESCE(content_type, '')) LIKE ?)" + ); + conditions.push(`NOT (${clauses.join(" OR ")})`); + for (const ext of options.excludeExts) { + values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`); + } + } + for (const provider of options.excludeProviders ?? []) { + conditions.push("lower(COALESCE(provider, '')) NOT LIKE ?"); + values.push(`%${provider.toLowerCase()}%`); + } + if (options.excludeSources?.length) { + conditions.push(`COALESCE(source, '') NOT IN (${options.excludeSources.map(() => "?").join(", ")})`); + values.push(...options.excludeSources); + } + if (options.sizeLessThan !== undefined) { + conditions.push("COALESCE(file_bytes, size) < ?"); + values.push(options.sizeLessThan); + } + if (options.sizeGreaterThan !== undefined) { + conditions.push("COALESCE(file_bytes, size) > ?"); + values.push(options.sizeGreaterThan); + } + if (options.lengthLessThan !== undefined) { + conditions.push("duration_secs < ?"); + values.push(options.lengthLessThan); + } + if (options.lengthGreaterThan !== undefined) { + conditions.push("duration_secs > ?"); + values.push(options.lengthGreaterThan); + } if (options.after) { conditions.push("posted_at >= ?"); values.push(options.after); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 13dbb82..84f1ca7 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -22,7 +22,7 @@ import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine. import { BackfillQueue } from "../src/backfill.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; -import { betterCandidate } from "../src/dedupe.js"; +import { betterCandidate, sortCandidates } from "../src/dedupe.js"; import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; @@ -1347,6 +1347,91 @@ void (async () => { store.close(); }); + await test("store exclusion, size, and length filters", () => { + const dir = join(root, "exclude-test"); + mkdirSync(dir, { recursive: true }); + const store = new ScrapeStore(dir); + + store.insertPage([ + message("900", { + attachments: [ + attachment({ filename: "keep.png", contentType: "image/png", size: 1_000 }), + attachment({ filename: "pixel.jpg", contentType: "image/jpeg", size: 400 } ) + ], + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=abc", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/embed/abc", + imageUrl: null, + thumbnailUrl: null + }, + { + type: "video", + url: "https://clips.twitch.tv/xyz", + title: null, + provider: "Twitch", + videoUrl: "https://clips.twitch.tv/xyz", + imageUrl: null, + thumbnailUrl: null + } + ] + }), + message("901", { + attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 9_000, durationSecs: 42 })] + }) + ]); + + // Insert order decides the ids: 1 keep.png, 2 pixel.jpg, 3 YouTube, 4 Twitch, 5 clip.mp4. + const ids = (options: Record) => + store + .queryMedia({ limit: 50, ...options }) + .map(row => row.id) + .sort((a, b) => a - b); + + assert.deepEqual(ids({}), [1, 2, 3, 4, 5]); + assert.deepEqual(ids({ excludeProviders: ["youtube"] }), [1, 2, 4, 5]); + assert.deepEqual(ids({ excludeSources: ["embed"] }), [1, 2, 5]); + assert.deepEqual(ids({ excludeExts: ["png"] }), [2, 3, 4, 5]); + assert.deepEqual(ids({ sizeGreaterThan: 9000 }), []); + assert.deepEqual(ids({ sizeLessThan: 600 }), [2]); + assert.deepEqual(ids({ lengthGreaterThan: 10 }), [5]); + // Unknown lengths drop out while a length range is set. + assert.deepEqual(ids({ lengthLessThan: 10 }), []); + store.close(); + }); + + await test("sortCandidates orders newest, oldest, user, size, and length", () => { + const make = (name: string | null, posted: string, size: number | null, duration: number | null, id: number) => ({ + meta: {} as never, + row: { + id, + posted_at: posted, + author_name: name, + file_bytes: null, + size, + duration_secs: duration + } + }); + + const items = [ + make("zoe", "2026-09-01T00:00:00.000Z", 100, 5, 1), + make("amy", "2026-09-03T00:00:00.000Z", 300, 1, 2), + make(null, "2026-09-02T00:00:00.000Z", null, null, 3) + ]; + + const ids = (sort: "newest" | "oldest" | "user" | "size" | "length") => + sortCandidates([...items], sort).map(candidate => candidate.row.id); + + assert.deepEqual(ids("newest"), [2, 3, 1]); + assert.deepEqual(ids("oldest"), [1, 3, 2]); + assert.deepEqual(ids("user"), [2, 1, 3]); + assert.deepEqual(ids("size"), [2, 1, 3]); + assert.deepEqual(ids("length"), [1, 2, 3]); + }); + await test("backfill queue groups scrapes per chat and estimates an ETA", async () => { const backend = new FakeBackend([ { diff --git a/poc/public/media.css b/poc/public/media.css index 5d62a14..38644a7 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -768,3 +768,26 @@ font: 400 11px/1.5 ui-monospace, Menlo, monospace; color: var(--color-neutral-500); } + +/* Advanced drawer: the "Do not include" group. */ +.vk-media-exclude-title { + cursor: pointer; + font: 600 11px/1.6 ui-monospace, Menlo, monospace; + color: var(--color-neutral-300); +} + +.vk-media-exclude-group { + margin-left: 12px; +} + +.vk-media-exclude-group summary { + cursor: pointer; +} + +.vk-media-small-input { + width: 90px; +} + +.vk-chip-strike { + text-decoration: line-through; +} diff --git a/poc/public/media.js b/poc/public/media.js index b253ff2..e4ce2ec 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -251,6 +251,12 @@ renderScrapes(); } + const SIZE_UNITS = { B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3 }; + + function sizeUnit(unit) { + return SIZE_UNITS[unit] ?? 1; + } + function filterQuery() { const kind = state.kind === "all" ? "" : `kind=${state.kind}&`; const source = state.filters.source ? `source=${state.filters.source}&` : ""; @@ -264,11 +270,34 @@ const excludeHosts = state.filters.excludeHosts.length ? `excludeHosts=${state.filters.excludeHosts.join(",")}&` : ""; + const sort = state.filters.sort && state.filters.sort !== "newest" ? `sort=${state.filters.sort}&` : ""; + const excludeProviders = state.filters.excludeProviders.length + ? `excludeProviders=${state.filters.excludeProviders.join(",")}&` + : ""; + const excludeSources = state.filters.excludeSources.length + ? `excludeSources=${state.filters.excludeSources.join(",")}&` + : ""; + const excludeExts = state.filters.excludeExts.length + ? `excludeExt=${state.filters.excludeExts.join(",")}&` + : ""; + const amount = (value, unit) => { + const number = Number(value); + if (value === "" || !Number.isFinite(number) || number < 0) return ""; + return String(Math.round(number * (unit ? sizeUnit(unit) : 1))); + }; + const sizeLt = state.filters.sizeLt !== "" ? `sizeLt=${amount(state.filters.sizeLt, state.filters.sizeLtUnit)}&` : ""; + const sizeGt = state.filters.sizeGt !== "" ? `sizeGt=${amount(state.filters.sizeGt, state.filters.sizeGtUnit)}&` : ""; + const lenLt = state.filters.lenLt !== "" ? `lenLt=${amount(state.filters.lenLt)}&` : ""; + const lenGt = state.filters.lenGt !== "" ? `lenGt=${amount(state.filters.lenGt)}&` : ""; const aspect = state.filters.aspect ? `aspect=${state.filters.aspect}&` : ""; const resolution = state.filters.resolution ? `resolution=${state.filters.resolution}&` : ""; const after = state.filters.after ? `after=${state.filters.after}T00:00:00.000Z&` : ""; const before = state.filters.before ? `before=${state.filters.before}T23:59:59.999Z&` : ""; - return `${kind}${source}${channel}${authors}${origin}${cached}${exts}${excludeHosts}${aspect}${resolution}${after}${before}`; + return ( + `${kind}${source}${channel}${authors}${origin}${cached}${exts}${excludeHosts}` + + `${sort}${excludeProviders}${excludeSources}${excludeExts}${sizeLt}${sizeGt}${lenLt}${lenGt}` + + `${aspect}${resolution}${after}${before}` + ); } function kindTotals(field) { @@ -1137,6 +1166,16 @@ cachedOnly: false, exts: [], excludeHosts: ["4chan.org", "i.4cdn.org"], + sort: "newest", + excludeProviders: [], + excludeSources: [], + excludeExts: [], + sizeLt: "", + sizeGt: "", + sizeLtUnit: "MB", + sizeGtUnit: "MB", + lenLt: "", + lenGt: "", aspect: "", resolution: "", after: "", @@ -1146,7 +1185,16 @@ function ensureDraft() { if (!state.draft) { - state.draft = { kind: state.kind, ...state.filters, authorIds: [...state.filters.authorIds] }; + state.draft = { + ...state.filters, + kind: state.kind, + authorIds: [...state.filters.authorIds], + exts: [...state.filters.exts], + excludeHosts: [...state.filters.excludeHosts], + excludeProviders: [...state.filters.excludeProviders], + excludeSources: [...state.filters.excludeSources], + excludeExts: [...state.filters.excludeExts] + }; } } @@ -1160,6 +1208,16 @@ cachedOnly: state.draft.cachedOnly, exts: [...state.draft.exts], excludeHosts: [...state.draft.excludeHosts], + sort: state.draft.sort, + excludeProviders: [...state.draft.excludeProviders], + excludeSources: [...state.draft.excludeSources], + excludeExts: [...state.draft.excludeExts], + sizeLt: state.draft.sizeLt, + sizeGt: state.draft.sizeGt, + sizeLtUnit: state.draft.sizeLtUnit, + sizeGtUnit: state.draft.sizeGtUnit, + lenLt: state.draft.lenLt, + lenGt: state.draft.lenGt, aspect: state.draft.aspect, resolution: state.draft.resolution, after: state.draft.after, @@ -1291,6 +1349,19 @@ const filters = section("Filters"); filters.append( + filterSelect( + [ + ["newest", "Newest first"], + ["oldest", "Oldest first"], + ["user", "User (A\u2192Z)"], + ["size", "Largest size"], + ["length", "Longest length"] + ], + draft.sort, + value => { + draft.sort = value; + } + ), filterSelect( [ ["", "Any ratio"], @@ -1403,6 +1474,146 @@ }); deadRow.append(deadBox, document.createTextNode(" Hide 4chan links (usually dead)")); filters.append(deadRow); + + // "Do not include": nested collapsibles, so exclusions stay out of the way. + const exclude = document.createElement("details"); + exclude.className = "vk-media-exclude"; + exclude.append(el("summary", "vk-media-exclude-title", "Do not include")); + + const group = title => { + const detailsNode = document.createElement("details"); + detailsNode.className = "vk-media-exclude-group"; + detailsNode.append(el("summary", "vk-media-status", title)); + return detailsNode; + }; + + const checkRow = (label, checked, onToggle) => { + const row = el("label", "vk-media-check"); + const box = document.createElement("input"); + box.type = "checkbox"; + box.checked = checked; + box.addEventListener("change", () => { + onToggle(box.checked); + renderFilterDrawer(); + }); + row.append(box, document.createTextNode(` ${label}`)); + return row; + }; + + const toggle = (list, value, on) => + on ? [...new Set([...list, value])] : list.filter(entry => entry !== value); + + const sourceGroup = group("Sources"); + for (const [value, label] of [ + ["twitch", "Twitch"], + ["youtube", "YouTube"], + ["discord", "Discord"], + ["tenor", "Tenor"], + ["giphy", "Giphy"] + ]) { + sourceGroup.append( + checkRow(label, draft.excludeProviders.includes(value), on => { + draft.excludeProviders = toggle(draft.excludeProviders, value, on); + }) + ); + } + for (const [value, label] of [ + ["embed", "Embeds"], + ["attachment", "Attachments"], + ["sticker", "Stickers"] + ]) { + sourceGroup.append( + checkRow(label, draft.excludeSources.includes(value), on => { + draft.excludeSources = toggle(draft.excludeSources, value, on); + }) + ); + } + exclude.append(sourceGroup); + + const extGroup = group("File types"); + const extRow = el("div", "vk-media-toolbar"); + for (const extension of [ + "mp4", + "webm", + "mov", + "mkv", + "png", + "jpg", + "gif", + "webp", + "avif", + "mp3", + "ogg", + "wav", + "m4a", + "flac" + ]) { + const chip = el( + "button", + `vk-chip ${draft.excludeExts.includes(extension) ? "vk-chip-off vk-chip-strike" : "vk-chip-on"}`, + extension + ); + chip.type = "button"; + chip.title = draft.excludeExts.includes(extension) + ? `Excluding ${extension}` + : `${extension} included`; + chip.addEventListener("click", () => { + draft.excludeExts = toggle(draft.excludeExts, extension, !draft.excludeExts.includes(extension)); + renderFilterDrawer(); + }); + extRow.append(chip); + } + extGroup.append(extRow); + exclude.append(extGroup); + + const amountRow = (label, key, unitKey) => { + const row = el("div", "vk-media-toolbar"); + row.append(el("span", "vk-media-status", label)); + + const input = document.createElement("input"); + input.type = "number"; + input.min = "0"; + input.step = "any"; + input.className = "vk-media-input vk-media-small-input"; + input.value = draft[key]; + input.addEventListener("input", () => { + draft[key] = input.value; + }); + row.append(input); + + if (unitKey) { + const unit = document.createElement("select"); + unit.className = "vk-chip vk-chip-off"; + for (const value of Object.keys(SIZE_UNITS)) { + const option = document.createElement("option"); + option.value = value; + option.textContent = value; + unit.append(option); + } + unit.value = draft[unitKey]; + unit.addEventListener("change", () => { + draft[unitKey] = unit.value; + }); + row.append(unit); + } else { + row.append(el("span", "vk-media-status", "seconds")); + } + + return row; + }; + + const sizeGroup = group("Size"); + sizeGroup.append( + amountRow("Less than", "sizeLt", "sizeLtUnit"), + amountRow("Greater than", "sizeGt", "sizeGtUnit") + ); + exclude.append(sizeGroup); + + const lengthGroup = group("Length"); + lengthGroup.append(amountRow("Less than", "lenLt"), amountRow("Greater than", "lenGt")); + exclude.append(lengthGroup); + + filters.append(exclude); filterDrawer.append(filters); // Zoom applies live; it is not part of the filter draft. @@ -1480,7 +1691,9 @@ function render() { title.textContent = state.view === "bot" ? "Slopbot" : "Media archive"; - toolbar.hidden = false; + // The filter bar and its drawer belong to the Media tab only. + toolbar.hidden = state.view === "bot"; + if (state.view === "bot") filterDrawer.hidden = true; grid.hidden = state.view === "bot"; botPanel.hidden = state.view !== "bot"; scrapeList.hidden = state.view === "bot"; diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index 84fabaf..e9da262 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -887,6 +887,23 @@ function buildServer(): McpServer { cachedOnly: z.boolean().optional().describe("Only items with a cached preview"), exts: z.array(z.string()).optional().describe("Only these file extensions (mp4, webm, png, ...)"), excludeHosts: z.array(z.string()).optional().describe("Hide URLs from these hosts"), + excludeExts: z.array(z.string()).optional().describe("Hide these file extensions"), + excludeProviders: z + .array(z.string()) + .optional() + .describe("Hide these providers (youtube, twitch, discord, tenor, giphy)"), + excludeSources: z + .array(z.string()) + .optional() + .describe("Hide these row sources (attachment, sticker, embed)"), + sizeLessThan: z.number().optional().describe("Only items smaller than this many bytes"), + sizeGreaterThan: z.number().optional().describe("Only items larger than this many bytes"), + lengthLessThan: z.number().optional().describe("Only items shorter than this many seconds"), + lengthGreaterThan: z.number().optional().describe("Only items longer than this many seconds"), + sort: z + .enum(["newest", "oldest", "user", "size", "length"]) + .optional() + .describe("Order: newest (default), oldest, user, size, length"), aspect: z .enum(["vertical", "square", "wide43", "wide169", "ultrawide", "other"]) .optional() @@ -911,6 +928,14 @@ function buildServer(): McpServer { cachedOnly, exts, excludeHosts, + excludeExts, + excludeProviders, + excludeSources, + sizeLessThan, + sizeGreaterThan, + lengthLessThan, + lengthGreaterThan, + sort, aspect, resolution, after, @@ -929,6 +954,14 @@ function buildServer(): McpServer { cachedOnly, exts, excludeHosts, + excludeExts, + excludeProviders, + excludeSources, + sizeLessThan, + sizeGreaterThan, + lengthLessThan, + lengthGreaterThan, + sort, aspect, resolution, after, diff --git a/poc/src/media.ts b/poc/src/media.ts index dc3b31e..04905dd 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -10,6 +10,7 @@ import { isAbsolute, join } from "node:path"; import { BackfillQueue, betterCandidate, + sortCandidates, dirSizeBytes, isNearLimit, planRetention, @@ -22,6 +23,7 @@ import { type MediaRow, type BackfillSettings, type BackfillStatus, + type MediaSort, type ScrapeBackend, type ScrapeMeta } from "../../media-scrape/src/index.js"; @@ -708,6 +710,14 @@ export class MediaService { cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; + excludeExts?: string[]; + excludeProviders?: string[]; + excludeSources?: string[]; + sizeLessThan?: number; + sizeGreaterThan?: number; + lengthLessThan?: number; + lengthGreaterThan?: number; + sort?: MediaSort; aspect?: string; resolution?: string; after?: string; @@ -829,10 +839,18 @@ export class MediaService { cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; + excludeExts?: string[]; + excludeProviders?: string[]; + excludeSources?: string[]; + sizeLessThan?: number; + sizeGreaterThan?: number; + lengthLessThan?: number; + lengthGreaterThan?: number; aspect?: string; resolution?: string; after?: string; before?: string; + sort?: MediaSort; }): Array<{ meta: ScrapeMeta; row: MediaRow }> { const candidates = new Map(); @@ -856,6 +874,13 @@ export class MediaService { cachedOnly: options.cachedOnly, exts: options.exts, excludeHosts: options.excludeHosts, + excludeExts: options.excludeExts, + excludeProviders: options.excludeProviders, + excludeSources: options.excludeSources, + sizeLessThan: options.sizeLessThan, + sizeGreaterThan: options.sizeGreaterThan, + lengthLessThan: options.lengthLessThan, + lengthGreaterThan: options.lengthGreaterThan, aspect: options.aspect, resolution: options.resolution, after: options.after, @@ -893,7 +918,7 @@ export class MediaService { } } - return [...candidates.values()]; + return sortCandidates([...candidates.values()], options.sort ?? "newest"); } /** Read media rows across every scrape, richest per URL, newest scrape first. */ @@ -909,6 +934,14 @@ export class MediaService { cachedOnly?: boolean; exts?: string[]; excludeHosts?: string[]; + excludeExts?: string[]; + excludeProviders?: string[]; + excludeSources?: string[]; + sizeLessThan?: number; + sizeGreaterThan?: number; + lengthLessThan?: number; + lengthGreaterThan?: number; + sort?: MediaSort; aspect?: string; resolution?: string; after?: string; diff --git a/poc/src/webui.ts b/poc/src/webui.ts index a30fc32..8359edc 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -177,6 +177,54 @@ function startAudioSocket(server: http.Server | https.Server, deps: WebUiDeps): }); } +/** Shared parsing for the media grid filters, used by /api/media and /api/media/stats. */ +function parseMediaFilters(query: Record) { + const list = (value: unknown) => + value + ? String(value) + .split(",") + .map(entry => entry.trim().toLowerCase()) + .filter(Boolean) + : undefined; + const amount = (value: unknown) => { + if (value === undefined || value === "") return undefined; + const number = Number(value); + return Number.isFinite(number) && number >= 0 ? number : undefined; + }; + + return { + kind: ["image", "gif", "video", "audio"].includes(String(query.kind ?? "")) + ? (String(query.kind) as "image" | "gif" | "video" | "audio") + : undefined, + source: ["attachment", "sticker", "embed"].includes(String(query.source ?? "")) + ? String(query.source) + : undefined, + channelId: query.channelId ? String(query.channelId) : undefined, + authorIds: list(query.authorId), + origin: + query.origin === "dm" || query.origin === "guild" + ? (String(query.origin) as "dm" | "guild") + : undefined, + cachedOnly: query.cachedOnly === "1" || query.cachedOnly === "true", + exts: list(query.ext), + excludeHosts: list(query.excludeHosts), + excludeExts: list(query.excludeExt), + excludeProviders: list(query.excludeProviders), + excludeSources: list(query.excludeSources), + sizeLessThan: amount(query.sizeLt), + sizeGreaterThan: amount(query.sizeGt), + lengthLessThan: amount(query.lenLt), + lengthGreaterThan: amount(query.lenGt), + aspect: query.aspect ? String(query.aspect) : undefined, + resolution: query.resolution ? String(query.resolution) : undefined, + after: query.after ? String(query.after) : undefined, + before: query.before ? String(query.before) : undefined, + sort: ["newest", "oldest", "user", "size", "length"].includes(String(query.sort ?? "")) + ? (String(query.sort) as "newest" | "oldest" | "user" | "size" | "length") + : undefined + }; +} + export function startWebUi(deps: WebUiDeps): WebUiHandle { const app = express(); app.use(express.json({ limit: "64kb" })); @@ -901,58 +949,14 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { app.get( "/api/media", wrap(async (req, res) => { - const kind = ["image", "gif", "video", "audio"].includes(String(req.query.kind ?? "")) - ? (String(req.query.kind) as "image" | "gif" | "video" | "audio") - : undefined; - const source = ["attachment", "sticker", "embed"].includes(String(req.query.source ?? "")) - ? String(req.query.source) - : undefined; - const channelId = req.query.channelId ? String(req.query.channelId) : undefined; - const authorIds = req.query.authorId - ? String(req.query.authorId) - .split(",") - .map(value => value.trim()) - .filter(Boolean) - : undefined; - const origin = - req.query.origin === "dm" || req.query.origin === "guild" - ? (String(req.query.origin) as "dm" | "guild") - : undefined; - const cachedOnly = req.query.cachedOnly === "1" || req.query.cachedOnly === "true"; - const exts = req.query.ext - ? String(req.query.ext) - .split(",") - .map(value => value.trim().toLowerCase()) - .filter(Boolean) - : undefined; - const excludeHosts = req.query.excludeHosts - ? String(req.query.excludeHosts) - .split(",") - .map(value => value.trim().toLowerCase()) - .filter(Boolean) - : undefined; - const aspect = req.query.aspect ? String(req.query.aspect) : undefined; - const resolution = req.query.resolution ? String(req.query.resolution) : undefined; - const after = req.query.after ? String(req.query.after) : undefined; - const before = req.query.before ? String(req.query.before) : undefined; + const filters = parseMediaFilters(req.query); const limit = clampLimit(req.query.limit, 60, 500); const offset = Math.max(0, Number(req.query.offset ?? 0) || 0); res.json({ ok: true, result: deps.media.queryMedia({ - kind, - source, - channelId, - authorIds, - origin, - cachedOnly, - exts, - excludeHosts, - aspect, - resolution, - after, - before, + ...filters, limit, offset }) @@ -970,46 +974,10 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { app.get( "/api/media/stats", wrap(async (req, res) => { - const kind = ["image", "gif", "video", "audio"].includes(String(req.query.kind ?? "")) - ? (String(req.query.kind) as "image" | "gif" | "video" | "audio") - : undefined; - - const authorIds = req.query.authorId - ? String(req.query.authorId) - .split(",") - .map(value => value.trim()) - .filter(Boolean) - : undefined; - const origin = - req.query.origin === "dm" || req.query.origin === "guild" - ? (String(req.query.origin) as "dm" | "guild") - : undefined; - const exts = req.query.ext - ? String(req.query.ext) - .split(",") - .map(value => value.trim().toLowerCase()) - .filter(Boolean) - : undefined; - const excludeHosts = req.query.excludeHosts - ? String(req.query.excludeHosts) - .split(",") - .map(value => value.trim().toLowerCase()) - .filter(Boolean) - : undefined; - res.json({ ok: true, result: deps.media.stats({ - kind, - source: req.query.source ? String(req.query.source) : undefined, - channelId: req.query.channelId ? String(req.query.channelId) : undefined, - authorIds, - origin, - cachedOnly: req.query.cachedOnly === "1" || req.query.cachedOnly === "true", - exts, - excludeHosts, - aspect: req.query.aspect ? String(req.query.aspect) : undefined, - resolution: req.query.resolution ? String(req.query.resolution) : undefined, + ...parseMediaFilters(req.query), after: req.query.after ? String(req.query.after) : undefined, before: req.query.before ? String(req.query.before) : undefined }) diff --git a/poc/test/media-ui-smoke.mjs b/poc/test/media-ui-smoke.mjs index c413ef5..b6c7c40 100644 --- a/poc/test/media-ui-smoke.mjs +++ b/poc/test/media-ui-smoke.mjs @@ -161,6 +161,9 @@ try { { timeout: 20000 } ); + check("media toolbar hidden on the bot tab", !(await page.locator("#media-toolbar").isVisible())); + check("media filter drawer hidden on the bot tab", !(await page.locator(".vk-media-drawer").isVisible())); + const summaryLines = await page.locator(".vk-bot-summary-line").count(); step("backfill summary lines", summaryLines); check("backfill summary renders", summaryLines >= 3, summaryLines);