From 8befc05591fda393f2bc410977fe1eabf7ba1f78 Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 13:43:27 +0200 Subject: [PATCH] Support wildcards in the title search --- CURRENT.md | 5 ++++ TODO.md | 5 ---- docs/media-scraper.md | 4 ++- docs/media.md | 2 +- docs/webui.md | 3 ++- media-scrape/src/store.ts | 32 ++++++++++++++++++++---- media-scrape/test/run.ts | 51 +++++++++++++++++++++++++++++++++++++++ poc/public/media.js | 4 +-- 8 files changed, 91 insertions(+), 15 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 39bf036..39d9aeb 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,11 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C52 Wildcard title search. `*` now maps to a LIKE wildcard in both title modes: a leading + `*` drops the word anchor (`*car` finds *scar* and *discard*), in-token stars match between + letters (`c*r*`), and a trailing star matches the plain prefix. `%` and `_` are escaped with an + `ESCAPE` clause so they only match themselves. Unit tests cover prefix vs wildcard vs in-word + matches; the drawer hint documents the syntax. - [x] C51 YouTube Shorts normalization. Shorts links broke the embedded player and View full because the viewer needs `watch?v=`. Extraction now rewrites `youtube.com/shorts/` (and `m.`) to the watch form, the viewer also accepts `/shorts/` defensively, and a one-time diff --git a/TODO.md b/TODO.md index fb72d81..434cbaa 100644 --- a/TODO.md +++ b/TODO.md @@ -64,11 +64,6 @@ Ordered backlog. `CURRENT.md` tracks the live session state. ## Media archive follow-ups -1. **Wildcard title search.** Support `*` in the title field: a leading `*` means substring anywhere - (`*car` finds *scar*, *discard*), `c*r` matches inside a word, and a plain token keeps today's - word-prefix rule. Map `*` to SQL `%`, escape literal `%`/`_` in the input, drop the word-boundary - anchor for tokens that start with `*`, and document it in the Advanced drawer hint. - 2. **Title work in its own lane.** Title resolution talks straight to `youtube.com/oembed` (daemon fetch; no Discord API, no bot account), but it currently shares the media-traffic lock and the preview batch pacing, so it serializes behind walks and previews. Move it to its own diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 8dbc6c1..1f35dbd 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -103,7 +103,9 @@ indexed watch URL folds into it (preview/file fields merge, message links move o Discord's `title` (YouTube and friends) plus `title_key`, the lowercased, punctuation-collapsed form used by the title search: multi-word prefix tokens by default ("Ma Ca Ju" finds *Man who jumps over car*), one phrase with the exact-match toggle, and rows without a title drop out while a -title query is set. Filter presets live in +title query is set. `*` is a wildcard anywhere in a token: `*car` matches inside words (*scar*, +*discard*), `c*r*` matches between letters, `car*` behaves like the plain prefix. Literal `%` and +`_` are escaped, so they only match themselves. Filter presets live in `media-scrape/src/filter-presets.ts`: aspect buckets (vertical, square, 4:3, 16:9, 21:9, other) and resolution buckets by the shortest side (SD, HD, FHD, QHD, 4K+, unknown). A missing resolution is filled with `ffprobe` after a download when `VESKSLOP_FFPROBE` is available. diff --git a/docs/media.md b/docs/media.md index 63ce859..ac30f14 100644 --- a/docs/media.md +++ b/docs/media.md @@ -150,7 +150,7 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun source channel, posted-date range, file-type chips, cached-only, a default-on hide for 4chan links (they 404 almost always), a sort select (newest, oldest, user, size, length), and a **Do not include** and **Must include** groups with nested collapsibles, a title search for embed titles - (multi-word prefix matching, with an **Exact match (quotes)** toggle) for sources (the common + (multi-word prefix matching with `*` wildcards, and an **Exact match (quotes)** toggle) for sources (the common providers in the archive plus embeds, attachments, and stickers), file types, size (with B/KB/MB/GB units), and length (seconds), with **Apply filters** / **Reset** in the bar. Size and length ranges drop diff --git a/docs/webui.md b/docs/webui.md index 0a73bd0..f282a29 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -139,7 +139,8 @@ box live in Chat, not Call, and the connection stats are not chips yet. bounds with B/KB/MB/GB units or seconds). A provider entry may carry several substrings, shown as `Giphy/Tenor`. A title field searches embed titles (YouTube): default is multi-word prefix matching (each token must start a word of the title, order-free), and **Exact match (quotes)** - treats the input as one phrase. A sort select orders the grid by newest, oldest, user, size, or + treats the input as one phrase. `*` acts as a wildcard in both modes (`*car` matches *scar*, + `c*r*` between letters); literal `%` and `_` are escaped. A sort select orders the grid by newest, oldest, user, size, or length. Size and length bounds drop rows with no known value. The filter bar and the drawer are hidden on the Bot tab. Filters do not diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 2dd02f3..c4ca2a0 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -187,6 +187,20 @@ export function normalizeTitleKey(title: string): string { .replace(/\s+/g, " "); } +/** Lowercase a title query and collapse punctuation, but keep `*` as a wildcard marker. */ +export function normalizeTitleQuery(query: string): string { + return query + .toLowerCase() + .replace(/[^a-z0-9*]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +/** Escape LIKE metacharacters, then turn `*` into `%`. */ +function titleLikePattern(value: string): string { + return value.replace(/[\\%_]/g, match => `\\${match}`).replace(/\*/g, "%"); +} + const SCHEMA = ` CREATE TABLE IF NOT EXISTS messages ( id TEXT PRIMARY KEY, @@ -803,14 +817,22 @@ export class ScrapeStore { values.push(options.lengthGreaterThan); } if (options.titleQuery?.trim()) { - const query = normalizeTitleKey(options.titleQuery); + const query = normalizeTitleQuery(options.titleQuery); if (options.titleExact) { - conditions.push("title_key LIKE ?"); - values.push(`%${query}%`); + // One phrase, substring of the title; `*` still acts as a wildcard. + conditions.push("title_key LIKE ? ESCAPE '\\'"); + values.push(`%${titleLikePattern(query)}%`); } else { for (const token of query.split(" ").filter(Boolean)) { - conditions.push("(' ' || COALESCE(title_key, '')) LIKE ?"); - values.push(`% ${token}%`); + const pattern = titleLikePattern(token); + if (token.startsWith("*")) { + // Leading `*`: anywhere in a word, not just the start. + conditions.push("COALESCE(title_key, '') LIKE ? ESCAPE '\\'"); + values.push(`%${pattern}%`); + } else { + conditions.push("(' ' || COALESCE(title_key, '')) LIKE ? ESCAPE '\\'"); + values.push(`% ${pattern}%`); + } } } } diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index fc75c95..7561881 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -31,6 +31,7 @@ import { dirSizeBytes, listScrapes, normalizeTitleKey, + normalizeTitleQuery, readScrapeMeta, ScrapeStore, writeScrapeMeta @@ -1619,6 +1620,56 @@ void (async () => { store.close(); }); + await test("title wildcards match anywhere and stay literal for % and _", () => { + const dir = join(root, "title-wildcards"); + mkdirSync(dir, { recursive: true }); + const store = new ScrapeStore(dir); + const embed = (id: string, title: string) => ({ + type: "video", + url: `https://www.youtube.com/watch?v=${id}`, + title, + provider: "YouTube", + videoUrl: `https://www.youtube.com/watch?v=${id}`, + imageUrl: null, + thumbnailUrl: null + }); + + store.insertPage([ + message("1300", { + embeds: [ + embed("w1", "Scar Tissue"), + embed("w2", "Discard the rest"), + embed("w3", "Car wash"), + embed("w4", "Cartoon physics"), + embed("w5", "Carly and the r") + ] + }) + ]); + + const ids = (options: Record) => + store + .queryMedia({ limit: 50, ...options }) + .map(row => row.url.split("v=")[1]) + .sort(); + + // Word-prefix default: car starts a word. + assert.deepEqual(ids({ titleQuery: "car" }), ["w3", "w4", "w5"]); + // Leading *: anywhere, so scar and discard join in. + assert.deepEqual(ids({ titleQuery: "*car" }), ["w1", "w2", "w3", "w4", "w5"]); + // Trailing * behaves like the prefix form. + assert.deepEqual(ids({ titleQuery: "car*" }), ["w3", "w4", "w5"]); + // In-word wildcard after a word start. + assert.deepEqual(ids({ titleQuery: "c*r" }), ["w3", "w4", "w5"]); + assert.deepEqual(ids({ titleQuery: "c*r*" }), ["w3", "w4", "w5"]); + assert.deepEqual(ids({ titleQuery: "*c*r*" }), ["w1", "w2", "w3", "w4", "w5"]); + store.close(); + }); + + await test("normalizeTitleQuery keeps stars and drops punctuation", () => { + assert.equal(normalizeTitleQuery(" *CaR, C*r* "), "*car c*r*"); + assert.equal(normalizeTitleQuery("Boat"), "boat"); + }); + await test("normalizeTitleKey collapses punctuation and case", () => { assert.equal(normalizeTitleKey("Woman's Day: 3 Things!"), "woman s day 3 things"); assert.equal(normalizeTitleKey(" A---B "), "a b"); diff --git a/poc/public/media.js b/poc/public/media.js index a4cf228..5c7dce3 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -1400,7 +1400,7 @@ const input = document.createElement("input"); input.type = "text"; input.className = "vk-media-input"; - input.placeholder = "Video title (Ma Ca Ju finds Man ... car jumps)"; + input.placeholder = "Video title (Ma Ca Ju; *car, c*r* wildcards)"; input.value = draft.title; input.addEventListener("input", () => { draft.title = input.value; @@ -1411,7 +1411,7 @@ const box = document.createElement("input"); box.type = "checkbox"; box.checked = draft.titleExact; - box.title = "Match the whole phrase instead of per-word prefixes"; + box.title = "Match the whole phrase instead of per-word prefixes; * still works as a wildcard"; box.addEventListener("change", () => { draft.titleExact = box.checked; });