diff --git a/CURRENT.md b/CURRENT.md index d7650ed..e6f5c36 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,13 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C47 Provider list from the dataset. Aggregated the `provider` column over all 11 scraper + databases (distinct URLs): discord 13631 (folding cdn.discordapp.com and media.discordapp.net), + youtube 4840, tenor 352, twitch 196, twitter 111, steam 83, medal 59, sketchfab 31, gfycat 30, + imgur 23. Those ten now live in `PROVIDER_FILTERS` (media-scrape), are served in + `/api/media/status` as `providerFilters`, and the Advanced drawer renders the source checkboxes + from that list (4chan stays out; the default URL host exclusion covers it). Unit test for the + list shape and a smoke check that the checkbox count matches the API list (13 = 10 + 3 kinds). - [x] C46 Sort and exclusion filters. The Advanced drawer gained a sort select (newest, oldest, user, size, length; global over the merged URL-dedupe stream) and a **Do not include** group with nested collapsibles: sources (Twitch, YouTube, Discord, Tenor, Giphy, embeds, attachments, diff --git a/TODO.md b/TODO.md index b581183..fd271c0 100644 --- a/TODO.md +++ b/TODO.md @@ -62,16 +62,6 @@ Ordered backlog. `CURRENT.md` tracks the live session state. optional `VOX_CLEAN`, and a help line in the TTS panel. Requested in `TTS-DESIGN.md`; the wrapper side is ready and verified. Inline `/rate` and `/pitch` need no UI. -## Media archive follow-ups - -1. **Provider exclusion list from real data.** The Advanced `Do not include → Sources` checkboxes - use a hand-picked list (Twitch, YouTube, Discord, Tenor, Giphy). Replace it with the providers - that actually dominate the archive: aggregate the `provider` column across the scraper - databases (`SELECT lower(provider), COUNT(*) FROM media WHERE provider IS NOT NULL GROUP BY 1 - ORDER BY 2 DESC`), take the top ~8-10, turn that into the checkbox list, and unit-test that the - UI list matches the shared constant. Keep the source-kind boxes (embed/attachment/sticker) - separate from the provider boxes. - ## Web UI follow-ups Ordered after the voice backend unless the owner asks first. diff --git a/docs/media.md b/docs/media.md index 240d63b..12a6ac6 100644 --- a/docs/media.md +++ b/docs/media.md @@ -149,8 +149,9 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun and the multi-select People popover. The **Advanced search** drawer has aspect ratio, resolution, source channel, posted-date range, file-type chips, cached-only, a default-on hide for 4chan links (they 404 almost always), a sort select (newest, oldest, user, size, length), and a **Do not - include** group with nested collapsibles for sources, file types, size (with B/KB/MB/GB units), - and length (seconds), with **Apply filters** / **Reset** in the bar. Size and length ranges drop + include** group with nested collapsibles for sources (the ten most common providers plus embeds, + attachments, and stickers), file types, size (with B/KB/MB/GB units), and length (seconds), with + **Apply filters** / **Reset** in the bar. Size and length ranges drop rows with no known value. The bar and drawer show only on the Media tab. Nothing renders until **Load media** is pressed. A **Metadata** dropdown toggles which fields show in the small banner under each tile (User, Type, Date, Source, Resolution, Length). Video diff --git a/docs/webui.md b/docs/webui.md index 6bd6980..8a2239a 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -131,9 +131,10 @@ box live in Chat, not Call, and the connection stats are not chips yet. popover. The **Advanced search** drawer holds aspect ratio, resolution, source channel, posted-date range, file-type chips (video mp4/webm/mov/mkv, image png/jpg/gif/webp/avif, audio), **Show only cached previews**, a default-on **Hide 4chan links (usually dead)**, and a - **Do not include** group with nested collapsibles for sources (Twitch, YouTube, Discord, Tenor, - Giphy, embeds, attachments, stickers), file types, size (less/greater than, with B/KB/MB/GB - units), and length (less/greater than, seconds). A sort select orders the grid by newest, + **Do not include** group with nested collapsibles for sources (the ten most common providers in + the archive — Discord, YouTube, Tenor, Twitch, Twitter, Steam, Medal, Sketchfab, Gfycat, Imgur — + plus embeds, attachments, stickers), file types, size (less/greater than, with B/KB/MB/GB units), + and length (less/greater than, seconds). A sort select orders the grid by newest, oldest, user, size, or length. Size and length ranges drop rows with no known value. The filter bar and the drawer are hidden on the Bot tab. Filters do not apply until **Apply filters** is pressed; **Reset** clears every filter and the zoom, unloads the @@ -290,7 +291,7 @@ replaceable. | GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached preview (type from `preview_ext`) | | GET | `/api/media/:scrapeId/:itemId/stream` | none | Playback/full-view source: the local file, or the remote URL refreshed from the source message. Supports `Range`. Non-media upstream responses (provider pages) return `415` | | GET | `/api/media/scrapes` | none | Every scrape with state and counters | -| GET | `/api/media/status` | none | Slopbot connection state, the scrape list, and `limitedUntil` while a Discord cooldown holds media traffic | +| GET | `/api/media/status` | none | Slopbot connection state, the scrape list, `limitedUntil` while a Discord cooldown holds media traffic, and `providerFilters` (the curated provider list for the Advanced drawer) | | GET | `/api/media/backfill` | none | Backfill jobs per chat with pending/cached/failed, per-kind pending, progress, throughput, `etaStartMs`/`etaRunMs`/`etaTotalMs`, totals with a Bot/Puppet split, the live settings, and `limitedUntil` | | POST | `/api/media/backfill/start` | `chatId?` | Start every job with pending rows, or one chat's job | | POST | `/api/media/backfill/pause` | `chatId?` | Pause all jobs, or one chat's job, after the current batch | diff --git a/media-scrape/src/filter-presets.ts b/media-scrape/src/filter-presets.ts index 3223204..7db43eb 100644 --- a/media-scrape/src/filter-presets.ts +++ b/media-scrape/src/filter-presets.ts @@ -48,3 +48,23 @@ export function resolutionCondition(preset: string): string | null { return null; } } + +/** + * The providers that dominate the archive, for the "Do not include" shortcuts. + * Aggregated 2026-09-17 over media-scrape/db (`provider` column, distinct URLs; discord folds in + * cdn.discordapp.com and media.discordapp.net). Matching is a case-insensitive substring on + * `provider`, so "twitter" also covers fxtwitter.com. The 4chan hosts stay out: the default URL + * host exclusion already hides them. + */ +export const PROVIDER_FILTERS = [ + "discord", + "youtube", + "tenor", + "twitch", + "twitter", + "steam", + "medal", + "sketchfab", + "gfycat", + "imgur" +] as const; diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index f9e4406..9b6fec6 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -10,6 +10,7 @@ export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js"; export { ASPECT_PRESETS, + PROVIDER_FILTERS, RESOLUTION_PRESETS, aspectCondition, resolutionCondition, diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 84f1ca7..7812d52 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -23,7 +23,7 @@ import { BackfillQueue } from "../src/backfill.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; import { betterCandidate, sortCandidates } from "../src/dedupe.js"; -import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; +import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; import { @@ -1347,6 +1347,20 @@ void (async () => { store.close(); }); + await test("provider filters are the curated archive list", () => { + assert.ok(PROVIDER_FILTERS.length >= 8 && PROVIDER_FILTERS.length <= 12, PROVIDER_FILTERS.length); + assert.equal(new Set(PROVIDER_FILTERS).size, PROVIDER_FILTERS.length); + + for (const value of PROVIDER_FILTERS) { + assert.equal(value, value.toLowerCase()); + assert.ok(!value.includes(" ") && !value.includes("."), `host-like entry: ${value}`); + } + + for (const required of ["discord", "youtube", "twitch", "tenor"]) { + assert.ok((PROVIDER_FILTERS as readonly string[]).includes(required), required); + } + }); + await test("store exclusion, size, and length filters", () => { const dir = join(root, "exclude-test"); mkdirSync(dir, { recursive: true }); diff --git a/poc/public/media.js b/poc/public/media.js index e4ce2ec..d4ea060 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -1332,7 +1332,14 @@ function toggleFilterDrawer() { filterDrawer.hidden = !filterDrawer.hidden; - if (!filterDrawer.hidden) renderFilterDrawer(); + if (!filterDrawer.hidden) { + renderFilterDrawer(); + if (!state.status?.providerFilters?.length) { + void loadScrapes().then(() => { + if (!filterDrawer.hidden) renderFilterDrawer(); + }); + } + } } /** The drawer holds the less common filters; the basic ones live in the bar. */ @@ -1504,13 +1511,8 @@ on ? [...new Set([...list, value])] : list.filter(entry => entry !== value); const sourceGroup = group("Sources"); - for (const [value, label] of [ - ["twitch", "Twitch"], - ["youtube", "YouTube"], - ["discord", "Discord"], - ["tenor", "Tenor"], - ["giphy", "Giphy"] - ]) { + for (const value of state.status?.providerFilters ?? []) { + const label = value.charAt(0).toUpperCase() + value.slice(1); sourceGroup.append( checkRow(label, draft.excludeProviders.includes(value), on => { draft.excludeProviders = toggle(draft.excludeProviders, value, on); diff --git a/poc/src/media.ts b/poc/src/media.ts index 04905dd..b0ebf62 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -21,6 +21,7 @@ import { resolveDbRoot, type MediaKind, type MediaRow, + PROVIDER_FILTERS, type BackfillSettings, type BackfillStatus, type MediaSort, @@ -160,6 +161,8 @@ export interface MediaStatus { scrapes: ScrapeMeta[]; /** ISO time while a Discord rate limit holds media traffic. */ limitedUntil: string | null; + /** The common providers, for the "Do not include" shortcuts in the Advanced drawer. */ + providerFilters: string[]; } export class MediaService { @@ -491,7 +494,8 @@ export class MediaService { scrapes: this.engine.list(), limitedUntil: limit.active ? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString() - : null + : null, + providerFilters: [...PROVIDER_FILTERS] }; } diff --git a/poc/test/media-ui-smoke.mjs b/poc/test/media-ui-smoke.mjs index b6c7c40..8a61463 100644 --- a/poc/test/media-ui-smoke.mjs +++ b/poc/test/media-ui-smoke.mjs @@ -116,8 +116,26 @@ try { step("people checkboxes", await page.locator(".vk-media-people input").count()); await page.getByRole("button", { name: /^People/ }).click(); + const providerFilters = await page.evaluate(async () => { + const response = await fetch("/api/media/status"); + const body = await response.json(); + return body?.result?.providerFilters ?? []; + }); + step("provider filters from the API", providerFilters); + await page.getByRole("button", { name: "Advanced search" }).click(); await page.waitForSelector(".vk-media-drawer:not([hidden])", { timeout: 5000 }); + const sourceBoxes = await page + .locator(".vk-media-exclude-group") + .first() + .locator(".vk-media-check") + .count(); + step("source checkboxes", sourceBoxes); + check( + "provider checkboxes match the API list", + sourceBoxes === providerFilters.length + 3, + { sourceBoxes, providers: providerFilters.length } + ); await page.locator(".vk-media-check", { hasText: "Show only cached previews" }).locator("input").check(); await page.getByRole("button", { name: "Apply filters" }).click(); await page.waitForTimeout(2500);