From 0b72422afe3cd2125216db5a0c40f7d57ef953d3 Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 08:39:41 +0200 Subject: [PATCH] Add file-type and host filters with a 4chan default --- CURRENT.md | 7 ++++++ docs/media-mcp.md | 3 ++- docs/media.md | 4 ++-- docs/webui.md | 6 +++-- media-scrape/src/store.ts | 13 +++++++++++ poc/public/media.js | 48 ++++++++++++++++++++++++++++++++++++++- poc/src/control-bridge.ts | 6 +++++ poc/src/media.ts | 8 +++++++ poc/src/webui.ts | 28 +++++++++++++++++++++++ 9 files changed, 117 insertions(+), 6 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index e78d5a6..360b78c 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,13 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C39 File-type filter and the 4chan default. The Advanced drawer gained extension chips + (video mp4/webm/mov/mkv, image png/jpg/gif/webp/avif, audio mp3/ogg/wav/m4a/flac) matched on + URL or content type, plus a default-on **Hide 4chan links (usually dead)** toggle. Backend, + API (`ext`, `excludeHosts`) and MCP support; live: video count 7696 → 7383 with the default + exclusion, `ext=mp4`/`webm` return only those files. Also confirmed the mr-game-memorial group + has 924 attachment videos with 772 previews cached and 0 preview errors — the rest are just + pending in the 2,140-item backfill queue. - [x] C38 Filter layout swap. The common filters (filetype chips, source kind, server/DM origin, multi-select People popover) now sit in the always-visible bar; the Advanced search drawer holds aspect, resolution, channel, date range, cached-only, and the card-size slider. diff --git a/docs/media-mcp.md b/docs/media-mcp.md index ab3a051..ad73991 100644 --- a/docs/media-mcp.md +++ b/docs/media-mcp.md @@ -24,7 +24,8 @@ interface MediaService { ``` `MediaQuery` (implemented): `kind`, `source`, `scrapeId`, `channelId`, `authorIds`, `origin` -(`guild`/`dm`), `cachedOnly`, `aspect`, `resolution`, `after`, `before`, `limit`, `offset`. The +(`guild`/`dm`), `cachedOnly`, `exts`, `excludeHosts`, `aspect`, `resolution`, `after`, `before`, +`limit`, `offset`. The `aspect` and `resolution` values are the preset keys from `filter-presets.ts`; `after`/`before` are ISO times on `posted_at`. Planned: `guildId`, `text`. diff --git a/docs/media.md b/docs/media.md index 8c9b09e..5551fb9 100644 --- a/docs/media.md +++ b/docs/media.md @@ -147,8 +147,8 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun - **Media** tab: the bar carries the common filters — kind chips, source kind, server/DM origin, and the multi-select People popover. The **Advanced search** drawer has aspect ratio, resolution, - source channel, posted-date range, and cached-only, with **Apply filters** / **Reset** in the - bar. + source channel, posted-date range, file-type chips, cached-only, and a default-on hide for 4chan + links (they 404 almost always), with **Apply filters** / **Reset** in the bar. Nothing renders until **Load media** is pressed. A **Metadata** dropdown toggles which fields show in the small banner under each tile (User, Type, Date, Source, Resolution, Length). Video tiles carry a translucent play icon top-right. Discord serves no thumbnails for uploaded videos, diff --git a/docs/webui.md b/docs/webui.md index 5f4d34a..180f8fd 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -129,7 +129,9 @@ box live in Chat, not Call, and the connection stats are not chips yet. - Media lists scraped items. The bar shows the common filters: filetype chips, source kind (attachments/stickers/embeds), origin (server channels or DMs), and a multi-select **People** popover. The **Advanced search** drawer holds aspect ratio, resolution, source channel, - posted-date range, and **Show only cached previews**, plus the card-size slider. Filters do not + posted-date range, file-type chips (video mp4/webm/mov/mkv, image png/jpg/gif/webp/avif, audio), + **Show only cached previews**, and a default-on **Hide 4chan links (usually dead)**, plus the + card-size slider. Filters do not apply until **Apply filters** is pressed; **Reset** clears every filter and the zoom, unloads the grid, and brings back the initial Load media prompt. - A **Card size** slider scales tiles from two-thirds of the old size up to one image per row; the @@ -265,7 +267,7 @@ replaceable. | POST | `/api/audio/stop` | none | Stop speech and clear the queue | | POST | `/api/audio/capture` | `start` | Start or stop capture | | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | -| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?` (comma list), `origin?` (`guild`/`dm`), `cachedOnly?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`, `previewError`, `animUrl`; `localUrl`/`thumbUrl` when a file or preview exists | +| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?` (comma list), `origin?` (`guild`/`dm`), `cachedOnly?`, `ext?` (comma list), `excludeHosts?` (comma list), `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`, `contentType`, `durationSecs`, `previewError`, `animUrl`; `localUrl`/`thumbUrl` when a file or preview exists | | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | | GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) | | GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range`. Non-media upstream responses (provider pages) return `415` | diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index f71b2b2..1f04cf0 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -108,6 +108,8 @@ export interface MediaFilterOptions { authorId?: string; authorIds?: string[]; cachedOnly?: boolean; + exts?: string[]; + excludeHosts?: string[]; aspect?: string; resolution?: string; after?: string; @@ -493,6 +495,17 @@ export class ScrapeStore { if (options.cachedOnly) { conditions.push("preview_hash IS NOT NULL"); } + if (options.exts?.length) { + const clauses = options.exts.map(() => "(lower(url) LIKE ? OR lower(content_type) LIKE ?)"); + conditions.push(`(${clauses.join(" OR ")})`); + for (const ext of options.exts) { + values.push(`%.${ext.toLowerCase()}%`, `%/${ext.toLowerCase()}%`); + } + } + for (const host of options.excludeHosts ?? []) { + conditions.push("lower(url) NOT LIKE ?"); + values.push(`%${host.toLowerCase()}%`); + } if (options.after) { conditions.push("posted_at >= ?"); values.push(options.after); diff --git a/poc/public/media.js b/poc/public/media.js index 984dbc3..c3a7a7c 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -195,11 +195,15 @@ : ""; const origin = state.filters.origin ? `origin=${state.filters.origin}&` : ""; const cached = state.filters.cachedOnly ? "cachedOnly=1&" : ""; + const exts = state.filters.exts.length ? `ext=${state.filters.exts.join(",")}&` : ""; + const excludeHosts = state.filters.excludeHosts.length + ? `excludeHosts=${state.filters.excludeHosts.join(",")}&` + : ""; const aspect = state.filters.aspect ? `aspect=${state.filters.aspect}&` : ""; const resolution = state.filters.resolution ? `resolution=${state.filters.resolution}&` : ""; const after = state.filters.after ? `after=${state.filters.after}T00:00:00.000Z&` : ""; const before = state.filters.before ? `before=${state.filters.before}T23:59:59.999Z&` : ""; - return `${kind}${source}${channel}${authors}${origin}${cached}${aspect}${resolution}${after}${before}`; + return `${kind}${source}${channel}${authors}${origin}${cached}${exts}${excludeHosts}${aspect}${resolution}${after}${before}`; } function kindTotals(field) { @@ -629,6 +633,8 @@ authorIds: [], origin: "", cachedOnly: false, + exts: [], + excludeHosts: ["4chan.org", "i.4cdn.org"], aspect: "", resolution: "", after: "", @@ -650,6 +656,8 @@ authorIds: [...state.draft.authorIds], origin: state.draft.origin, cachedOnly: state.draft.cachedOnly, + exts: [...state.draft.exts], + excludeHosts: [...state.draft.excludeHosts], aspect: state.draft.aspect, resolution: state.draft.resolution, after: state.draft.after, @@ -846,6 +854,34 @@ }) ); + const typeRow = (label, extensions) => { + const row = el("div", "vk-media-toolbar"); + row.append(el("span", "vk-media-status", label)); + for (const extension of extensions) { + const chip = el( + "button", + `vk-chip ${draft.exts.includes(extension) ? "vk-chip-on" : "vk-chip-off"}`, + extension + ); + chip.type = "button"; + chip.addEventListener("click", () => { + draft.exts = draft.exts.includes(extension) + ? draft.exts.filter(value => value !== extension) + : [...draft.exts, extension]; + renderFilterDrawer(); + }); + row.append(chip); + } + return row; + }; + + filters.append( + el("div", "vk-media-status", "File types"), + typeRow("Video", ["mp4", "webm", "mov", "mkv"]), + typeRow("Image", ["png", "jpg", "gif", "webp", "avif"]), + typeRow("Audio", ["mp3", "ogg", "wav", "m4a", "flac"]) + ); + const cachedRow = el("label", "vk-media-check"); const cachedBox = document.createElement("input"); cachedBox.type = "checkbox"; @@ -855,6 +891,16 @@ }); cachedRow.append(cachedBox, document.createTextNode(" Show only cached previews")); filters.append(cachedRow); + + const deadRow = el("label", "vk-media-check"); + const deadBox = document.createElement("input"); + deadBox.type = "checkbox"; + deadBox.checked = draft.excludeHosts.includes("4chan.org"); + deadBox.addEventListener("change", () => { + draft.excludeHosts = deadBox.checked ? ["4chan.org", "i.4cdn.org"] : []; + }); + deadRow.append(deadBox, document.createTextNode(" Hide 4chan links (usually dead)")); + filters.append(deadRow); filterDrawer.append(filters); // Zoom applies live; it is not part of the filter draft. diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index 84526f1..fe08f0e 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -883,6 +883,8 @@ function buildServer(): McpServer { authorIds: z.array(z.string()).optional().describe("Restrict to these posters"), origin: z.enum(["dm", "guild"]).optional().describe("Server channels or DMs"), cachedOnly: z.boolean().optional().describe("Only items with a cached preview"), + exts: z.array(z.string()).optional().describe("Only these file extensions (mp4, webm, png, ...)"), + excludeHosts: z.array(z.string()).optional().describe("Hide URLs from these hosts"), aspect: z .enum(["vertical", "square", "wide43", "wide169", "ultrawide", "other"]) .optional() @@ -905,6 +907,8 @@ function buildServer(): McpServer { authorIds, origin, cachedOnly, + exts, + excludeHosts, aspect, resolution, after, @@ -921,6 +925,8 @@ function buildServer(): McpServer { authorIds, origin, cachedOnly, + exts, + excludeHosts, aspect, resolution, after, diff --git a/poc/src/media.ts b/poc/src/media.ts index 20ea818..6087df6 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -638,6 +638,8 @@ export class MediaService { authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; + exts?: string[]; + excludeHosts?: string[]; aspect?: string; resolution?: string; after?: string; @@ -757,6 +759,8 @@ export class MediaService { authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; + exts?: string[]; + excludeHosts?: string[]; aspect?: string; resolution?: string; after?: string; @@ -782,6 +786,8 @@ export class MediaService { authorId: options.authorId, authorIds: options.authorIds, cachedOnly: options.cachedOnly, + exts: options.exts, + excludeHosts: options.excludeHosts, aspect: options.aspect, resolution: options.resolution, after: options.after, @@ -833,6 +839,8 @@ export class MediaService { authorIds?: string[]; origin?: "dm" | "guild"; cachedOnly?: boolean; + exts?: string[]; + excludeHosts?: string[]; aspect?: string; resolution?: string; after?: string; diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 45d30b1..806e2b2 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -893,6 +893,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { ? (String(req.query.origin) as "dm" | "guild") : undefined; const cachedOnly = req.query.cachedOnly === "1" || req.query.cachedOnly === "true"; + const exts = req.query.ext + ? String(req.query.ext) + .split(",") + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + : undefined; + const excludeHosts = req.query.excludeHosts + ? String(req.query.excludeHosts) + .split(",") + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + : undefined; const aspect = req.query.aspect ? String(req.query.aspect) : undefined; const resolution = req.query.resolution ? String(req.query.resolution) : undefined; const after = req.query.after ? String(req.query.after) : undefined; @@ -909,6 +921,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { authorIds, origin, cachedOnly, + exts, + excludeHosts, aspect, resolution, after, @@ -944,6 +958,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { req.query.origin === "dm" || req.query.origin === "guild" ? (String(req.query.origin) as "dm" | "guild") : undefined; + const exts = req.query.ext + ? String(req.query.ext) + .split(",") + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + : undefined; + const excludeHosts = req.query.excludeHosts + ? String(req.query.excludeHosts) + .split(",") + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + : undefined; res.json({ ok: true, @@ -954,6 +980,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { authorIds, origin, cachedOnly: req.query.cachedOnly === "1" || req.query.cachedOnly === "true", + exts, + excludeHosts, aspect: req.query.aspect ? String(req.query.aspect) : undefined, resolution: req.query.resolution ? String(req.query.resolution) : undefined, after: req.query.after ? String(req.query.after) : undefined,