From 774183b6fa2695986b21425271a2380f3795c33e Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 13:19:22 +0200 Subject: [PATCH] Backfill YouTube titles through the queue --- CURRENT.md | 9 ++ TODO.md | 9 -- docs/media-scraper.md | 9 +- docs/media.md | 3 +- media-scrape/src/backfill.ts | 13 ++- media-scrape/src/embed-preview.ts | 24 ++++++ media-scrape/src/engine.ts | 51 +++++++++-- media-scrape/src/index.ts | 2 +- media-scrape/src/store.ts | 60 ++++++++++++- media-scrape/test/run.ts | 135 ++++++++++++++++++++++++++++++ poc/public/media.js | 1 + 11 files changed, 295 insertions(+), 21 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 61aefe3..bbe16a8 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,15 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C50 Title backfill in the queue. YouTube-like rows (provider or URL matching + youtube/youtu.be) without a title are now a second work type in the backfill queue: resolved + through the provider oEmbed, stored as `title`/`title_key`, with `title_attempts` and + `title_error` so deleted videos close out after the tries setting. Titles and previews share a + batch (up to half for titles while both remain); `previewCounts().titlePending`, + `BackfillMember.pendingTitles`, and the job totals carry the work, and the Bot tab member line + shows `+ N titles`. Unit tests cover selection, resolution, and the give-up cap. Live: the + user's queue (batch 40, 30 s, auto off) started once to verify — 133 titles resolved in two + minutes, dead videos counting toward the cap. - [x] C49 Searchable video titles. Embed rows now store Discord's `title` plus a normalized `title_key` (lowercase, punctuation collapsed). The Advanced drawer has a title field with an **Exact match (quotes)** toggle: default is order-free multi-word prefix search ("Ma Ca Ju" diff --git a/TODO.md b/TODO.md index c817340..fd271c0 100644 --- a/TODO.md +++ b/TODO.md @@ -62,15 +62,6 @@ Ordered backlog. `CURRENT.md` tracks the live session state. optional `VOX_CLEAN`, and a help line in the TTS panel. Requested in `TTS-DESIGN.md`; the wrapper side is ready and verified. Inline `/rate` and `/pitch` need no UI. -## Media archive follow-ups - -1. **Backfill titles for existing rows.** New scrapes store embed titles, but the ~20k rows - already indexed have `title IS NULL`. Add a paced pass that resolves titles through the provider - oEmbed endpoints (YouTube first) for rows where `title IS NULL AND lower(provider) LIKE - '%youtube%'`, with the same batch/cooldown settings as previews, an attempt cap so dead videos - close out, progress in the Bot tab, and tests. Consider folding it into the backfill queue as a - second work type. - ## Web UI follow-ups Ordered after the voice backend unless the owner asks first. diff --git a/docs/media-scraper.md b/docs/media-scraper.md index e122b58..d1f042a 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -88,7 +88,7 @@ One SQLite file per scrape (`index.db`, `node:sqlite`). | Table | Columns | | --- | --- | | `messages` | `id`, `author_id`, `author_name`, `timestamp`, `content` | -| `media` | `id`, `kind`, `source`, `author_id`, `author_name`, `posted_at`, `url`, `provider`, `title`, `title_key`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `aspect_ratio`, `duration_secs`, `downloaded`, `file_path`, `created_at` | +| `media` | `id`, `kind`, `source`, `author_id`, `author_name`, `posted_at`, `url`, `provider`, `title`, `title_key`, `title_error`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `aspect_ratio`, `duration_secs`, `downloaded`, `file_path`, `created_at` | | `media_sources` | `media_id`, `message_id` | | `downloads` | `media_id`, `state`, `attempts`, `error` | @@ -163,7 +163,12 @@ Previews run after every walk: - The job stays `running` through the download and preview passes, so a client never sees `done` on a half-archived scrape. A paused walk still gets these passes for the pages it collected. - The **backfill queue** (`media-scrape/src/backfill.ts`) fills in scrapes that paused or finished - before their previews were cached. One job per chat groups every paused, done, or failed scrape + before their previews were cached, and resolves missing titles. **Titles**: YouTube-like rows + (`provider` or URL matching youtube/youtu.be) whose Discord embed carried no title are resolved + through the provider oEmbed and stored as `title` + `title_key`; failures count toward + `title_attempts` and close out as `title_error` after the tries-per-item setting, so deleted + videos stop rotating. Titles and previews share a batch: up to half of it goes to titles while + both kinds of work remain. One job per chat groups every paused, done, or failed scrape of that chat, so a paused-and-resumed scrape stays in one job. Jobs run one at a time, in batches, and report pending/cached/failed counts, per-kind pending, throughput, and an ETA. The Bot tab drives them: **Start backfill**, per-job Start/Pause, and **Pause all**. Jobs with the diff --git a/docs/media.md b/docs/media.md index 7a5798b..63ce859 100644 --- a/docs/media.md +++ b/docs/media.md @@ -200,7 +200,8 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun that disappears when the tab is opened and a Clear button. - **Bot** tab: slopbot connection state, the backfill queue, per-backend stats, and the tunables. **Backfill** shows totals (pending, cached, failed, downloaded, stored bytes), an ETA at the - current settings, and a Bot/Puppet split of pending/cached/failed. One job card per chat lists its + current settings, and a Bot/Puppet split of pending/cached/failed. Pending includes missing + YouTube titles; each member line shows its share as `+ N titles`. One job card per chat lists its member scrapes (paused and resumed scrapes of the same chat are combined), per-kind pending counts, a cached/total progress bar, a measured ETA, and **Start**/**Pause**; the section header carries **Start backfill** and **Pause all**. A red banner reads `API timed out, increase timers` diff --git a/media-scrape/src/backfill.ts b/media-scrape/src/backfill.ts index d9b5f4e..e03333c 100644 --- a/media-scrape/src/backfill.ts +++ b/media-scrape/src/backfill.ts @@ -29,6 +29,8 @@ export interface BackfillMember { backend: BackendName; mode: ScrapeMode; pending: number; + /** YouTube-like rows still missing a title. */ + pendingTitles: number; cached: number; failed: number; total: number; @@ -152,6 +154,7 @@ export class BackfillQueue { backend: meta.backend, mode: meta.mode, pending: 0, + pendingTitles: 0, cached: 0, failed: 0, total: 0, @@ -165,6 +168,7 @@ export class BackfillQueue { try { const counts = store.previewCounts(); member.pending = counts.pending; + member.pendingTitles = counts.titlePending; member.cached = counts.cached; member.failed = counts.failed; member.total = counts.total; @@ -205,7 +209,10 @@ export class BackfillQueue { for (const [chatId, group] of groups) { // Hide chats that are fully backfilled unless their job did work this session. const tracked = this.jobs.get(chatId); - const pending = group.members.reduce((sum, member) => sum + member.pending, 0); + const pending = group.members.reduce( + (sum, member) => sum + member.pending + member.pendingTitles, + 0 + ); if (!tracked && pending === 0) continue; const perMinute = tracked?.perMinute ?? this.estimatedPerMinute(settings, gaps); @@ -411,7 +418,9 @@ export class BackfillQueue { while (!job.stop) { const group = this.plan(false).get(job.chatId); - const member = group?.members.find(candidate => candidate.pending > 0); + const member = group?.members.find( + candidate => candidate.pending > 0 || candidate.pendingTitles > 0 + ); if (!member) break; const limit = this.engine.rateLimitStatus(); diff --git a/media-scrape/src/embed-preview.ts b/media-scrape/src/embed-preview.ts index 8f21e11..31d9748 100644 --- a/media-scrape/src/embed-preview.ts +++ b/media-scrape/src/embed-preview.ts @@ -59,6 +59,30 @@ export function ogImageFromHtml(html: string): string | null { return null; } +/** + * The embed's title from the provider oEmbed, for YouTube-like rows. Null when the host is not + * YouTube or the provider does not answer; the backfill queue retries and then gives up. + */ +export async function resolveEmbedTitle( + url: string, + options: DownloadOptions = {} +): Promise { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + + const host = parsed.hostname.replace(/^www\./, "").toLowerCase(); + if (host !== "youtube.com" && host !== "m.youtube.com" && host !== "youtu.be" && host !== "youtube-nocookie.com") { + return null; + } + + const data = await getJson(`https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`, options); + return typeof data?.title === "string" && data.title.trim() ? data.title.trim() : null; +} + /** * A preview URL for an embed. Uses the stored thumbnail when present, otherwise asks the * provider (oEmbed, fxtwitter) and finally falls back to the page's og:image. diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 23d301e..6f311cd 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -16,7 +16,7 @@ import { videoPoster, type DownloadOptions } from "./download.js"; -import { resolveEmbedPreview } from "./embed-preview.js"; +import { resolveEmbedPreview, resolveEmbedTitle } from "./embed-preview.js"; import { createScrapeDirs, listScrapes, @@ -52,6 +52,8 @@ export interface PreviewBatchResult { skipped: number; failed: number; ratelimited: number; + /** Titles resolved in this batch. */ + titled: number; /** Rows still pending after the batch. */ remaining: number; /** True when a stop request interrupted the batch. */ @@ -547,7 +549,9 @@ export class ScrapeEngine { async previewBatch(scrapeId: string, limit: number): Promise { const dir = join(this.options.rootDir, scrapeId); const meta = this.active.get(scrapeId) ?? readScrapeMeta(dir); - if (!meta) return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, remaining: 0, stopped: false }; + if (!meta) { + return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, titled: 0, remaining: 0, stopped: false }; + } const releaseTraffic = await this.acquireTraffic(); const store = new ScrapeStore(dir); @@ -572,10 +576,47 @@ export class ScrapeEngine { skipped: 0, failed: 0, ratelimited: 0, + titled: 0, remaining: 0, stopped: false }; - const pending = store.pendingPreviews(limit, true); + + // Title work first, then previews with what is left of the batch. YouTube rows that + // never got a title share the queue with the preview pass. + const titleBudget = store.pendingTitles(1).length > 0 && limit > 1 ? Math.ceil(limit / 2) : limit; + for (const row of store.pendingTitles(titleBudget)) { + if (this.stopRequested.has(meta.id)) { + meta.state = "paused"; + result.stopped = true; + break; + } + + const title = await resolveEmbedTitle(row.url, options); + if (title) { + store.markTitle(row.id, title); + result.titled++; + } else { + const attempts = store.bumpTitleAttempt(row.id); + const maxAttempts = this.options.previewMaxAttempts ?? 3; + if (attempts >= maxAttempts) { + store.markTitleError(row.id, `no title after ${attempts} attempt(s)`); + this.log(`${meta.id}: title #${row.id} gave up after ${attempts} attempts (${row.url.slice(0, 60)})`); + } else { + this.log(`${meta.id}: title #${row.id} unresolved (${attempts}/${maxAttempts}) (${row.url.slice(0, 60)})`); + } + } + + this.persist(dir, meta); + await this.sleep(previewGapMs(options)); + } + + if (result.stopped) { + result.remaining = store.pendingPreviews(5000, true).length; + return result; + } + + const previewLimit = Math.max(1, limit - result.titled); + const pending = store.pendingPreviews(previewLimit, true); for (const row of pending) { if (this.stopRequested.has(meta.id)) { @@ -711,10 +752,10 @@ export class ScrapeEngine { } result.remaining = store.pendingPreviews(5000, true).length; - if (pending.length) { + if (pending.length || result.titled) { this.log( `${meta.id}: preview pass done: ${result.cached} cached, ${result.skipped} skipped, ` + - `${result.failed} failed, ${result.remaining} still pending` + `${result.failed} failed, ${result.titled} titled, ${result.remaining} still pending` ); } diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index 9b6fec6..70f3ac3 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -5,7 +5,7 @@ export { extractMedia, type ExtractedMedia } from "./extract.js"; export { betterCandidate, candidateScore, sortCandidates, type DedupeCandidate } from "./dedupe.js"; -export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js"; +export { ogImageFromHtml, resolveEmbedPreview, resolveEmbedTitle } from "./embed-preview.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js"; export { diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 34b5e09..d678e2a 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -171,6 +171,8 @@ export interface MediaRow { preview_attempts: number; /** Embed title from Discord (YouTube and friends). */ title: string | null; + title_attempts: number; + title_error: string | null; /** Lowercased, punctuation-collapsed title for token-prefix search. */ title_key: string | null; created_at: string; @@ -224,6 +226,8 @@ CREATE TABLE IF NOT EXISTS media ( preview_attempts INTEGER NOT NULL DEFAULT 0, title TEXT, title_key TEXT, + title_attempts INTEGER NOT NULL DEFAULT 0, + title_error TEXT, created_at TEXT NOT NULL ); @@ -273,6 +277,10 @@ export class ScrapeStore { this.db.exec("ALTER TABLE media ADD COLUMN title TEXT"); this.db.exec("ALTER TABLE media ADD COLUMN title_key TEXT"); } + if (!names.has("title_attempts")) { + this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0"); + this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT"); + } } close(): void { @@ -434,6 +442,8 @@ export class ScrapeStore { pending: number; cached: number; failed: number; + /** YouTube-like rows still missing a title. */ + titlePending: number; byKind: Record; } { const empty = () => ({ total: 0, pending: 0, cached: 0, failed: 0 }); @@ -477,10 +487,58 @@ export class ScrapeStore { totals.failed += row.failed ?? 0; } - return { ...totals, byKind }; + const titles = this.db + .prepare( + `SELECT COUNT(*) AS count FROM media + WHERE title IS NULL AND title_error IS NULL + AND (lower(COALESCE(provider, '')) LIKE '%youtube%' + OR lower(url) LIKE '%youtube.com/%' + OR lower(url) LIKE '%youtu.be/%')` + ) + .get() as { count: number | null } | undefined; + + return { ...totals, titlePending: titles?.count ?? 0, byKind }; } /** Count one transient preview failure and return the new total. */ + /** + * YouTube-like rows whose title Discord did not provide, oldest first. Resolved through the + * provider oEmbed in the backfill queue. + */ + pendingTitles(limit = 500): MediaRow[] { + return this.db + .prepare( + `SELECT * FROM media + WHERE title IS NULL AND title_error IS NULL + AND (lower(COALESCE(provider, '')) LIKE '%youtube%' + OR lower(url) LIKE '%youtube.com/%' + OR lower(url) LIKE '%youtu.be/%') + ORDER BY id LIMIT ?` + ) + .all(limit) as unknown as MediaRow[]; + } + + /** Store a resolved title and its search key. */ + markTitle(id: number, title: string): void { + this.db + .prepare("UPDATE media SET title = ?, title_key = ?, title_error = NULL WHERE id = ?") + .run(title.slice(0, 300), normalizeTitleKey(title), id); + } + + /** Count one transient title failure and return the new total. */ + bumpTitleAttempt(id: number): number { + this.db.prepare("UPDATE media SET title_attempts = title_attempts + 1 WHERE id = ?").run(id); + const row = this.db.prepare("SELECT title_attempts FROM media WHERE id = ?").get(id) as + | { title_attempts: number | null } + | undefined; + + return row?.title_attempts ?? 0; + } + + markTitleError(id: number, error: string): void { + this.db.prepare("UPDATE media SET title_error = ? WHERE id = ?").run(error.slice(0, 200), id); + } + /** Count one transient preview failure and return the new total. */ bumpPreviewAttempt(id: number): number { this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index cc2ab7b..8679a57 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -1348,6 +1348,141 @@ void (async () => { store.close(); }); + await test("pendingTitles selects YouTube rows without a title", () => { + const dir = join(root, "pending-titles"); + mkdirSync(dir, { recursive: true }); + const store = new ScrapeStore(dir); + + store.insertPage([ + message("1100", { + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=withtitle", + title: "Has a title", + provider: "YouTube", + videoUrl: "https://www.youtube.com/watch?v=withtitle", + imageUrl: null, + thumbnailUrl: null + }, + { + type: "video", + url: "https://www.youtube.com/watch?v=notitle", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/watch?v=notitle", + imageUrl: null, + thumbnailUrl: null + }, + { + type: "video", + url: "https://youtu.be/short", + title: null, + provider: "youtu.be", + videoUrl: "https://youtu.be/short", + imageUrl: null, + thumbnailUrl: null + }, + { + type: "video", + url: "https://clips.twitch.tv/notyoutube", + title: null, + provider: "Twitch", + videoUrl: "https://clips.twitch.tv/notyoutube", + imageUrl: null, + thumbnailUrl: null + } + ] + }) + ]); + + const label = (url: string) => url.split("=")[1] ?? url.split("/").pop(); + const pending = store.pendingTitles(10).map(row => label(row.url)); + assert.deepEqual(pending.sort(), ["notitle", "short"].sort()); + + const target = store.pendingTitles(10).find(row => row.url.includes("notitle")); + const dead = store.pendingTitles(10).find(row => row.url.includes("youtu.be")); + assert.ok(target); + assert.ok(dead); + + store.markTitle(target.id, "Resolved Title"); + assert.equal(store.getMedia(target.id)?.title, "Resolved Title"); + assert.equal(store.pendingTitles(10).some(row => row.id === target.id), false); + + store.markTitleError(dead.id, "no title after 1 attempt(s)"); + assert.equal(store.pendingTitles(10).some(row => row.id === dead.id), false); + assert.equal(store.previewCounts().titlePending, 0); + store.close(); + }); + + await test("previewBatch resolves titles and gives up after the cap", async () => { + const embed = { + type: "video", + url: "https://www.youtube.com/watch?v=titlepass", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/watch?v=titlepass", + imageUrl: null, + thumbnailUrl: null + }; + + const backend = new FakeBackend([{ messages: [message("1101", { embeds: [embed] })] }]); + const engine = engineFor(root, backend, { + previewMaxAttempts: 3, + download: { + fetchImpl: async (input: any) => { + if (String(input).includes("oembed")) { + return new Response(JSON.stringify({ title: "Real Title" }), { + status: 200, + headers: { "content-type": "application/json" } + }); + } + return new Response("no", { status: 404 }); + }, + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "135", guildId: null, mode: "links", backend }); + await engine.processQueue(); + await engine.previewBatch(started.id, 5); + + const store = new ScrapeStore(join(root, started.id)); + const row = store.queryMedia({ kind: "video" })[0]; + assert.equal(row.title, "Real Title"); + assert.equal(row.title_key, "real title"); + assert.equal(store.previewCounts().titlePending, 0); + store.close(); + + // A provider that never answers is closed out after the cap. + const deadBackend = new FakeBackend([ + { messages: [message("1102", { embeds: [{ ...embed, url: "https://youtu.be/deadtitle", videoUrl: "https://youtu.be/deadtitle" }] })] } + ]); + const deadEngine = engineFor(root, deadBackend, { + previewMaxAttempts: 1, + download: { + fetchImpl: async () => { + throw new Error("fetch failed"); + }, + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const dead = await deadEngine.start({ channelId: "136", guildId: null, mode: "links", backend: deadBackend }); + await deadEngine.processQueue(); + await deadEngine.previewBatch(dead.id, 5); + + const deadStore = new ScrapeStore(join(root, dead.id)); + const deadRow = deadStore.queryMedia({ kind: "video" })[0]; + assert.match(String(deadRow.title_error), /^no title after 1 attempt/); + assert.equal(deadStore.previewCounts().titlePending, 0); + deadStore.close(); + }); + await test("title search matches word prefixes and exact phrases", () => { const dir = join(root, "title-search"); mkdirSync(dir, { recursive: true }); diff --git a/poc/public/media.js b/poc/public/media.js index 18cee59..bce03d2 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -790,6 +790,7 @@ "vk-bot-job-member", `${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` + `${member.backend} \u00b7 ${member.pending} pending` + + (member.pendingTitles ? ` + ${member.pendingTitles} titles` : "") + (kinds ? ` (${kinds})` : "") ) );