Backfill YouTube titles through the queue

2026-09-17 13:19:22 +02:00
parent 557ba5ca7e
commit 774183b6fa
11 changed files with 295 additions and 21 deletions
@@ -29,6 +29,8 @@ export interface BackfillMember {
    backend: BackendName;
    mode: ScrapeMode;
    pending: number;
+    /** YouTube-like rows still missing a title. */
+    pendingTitles: number;
    cached: number;
    failed: number;
    total: number;
@@ -152,6 +154,7 @@ export class BackfillQueue {
                backend: meta.backend,
                mode: meta.mode,
                pending: 0,
+                pendingTitles: 0,
                cached: 0,
                failed: 0,
                total: 0,
@@ -165,6 +168,7 @@ export class BackfillQueue {
                try {
                    const counts = store.previewCounts();
                    member.pending = counts.pending;
+                    member.pendingTitles = counts.titlePending;
                    member.cached = counts.cached;
                    member.failed = counts.failed;
                    member.total = counts.total;
@@ -205,7 +209,10 @@ export class BackfillQueue {
        for (const [chatId, group] of groups) {
            // Hide chats that are fully backfilled unless their job did work this session.
            const tracked = this.jobs.get(chatId);
-            const pending = group.members.reduce((sum, member) => sum + member.pending, 0);
+            const pending = group.members.reduce(
+                (sum, member) => sum + member.pending + member.pendingTitles,
+                0
+            );
            if (!tracked && pending === 0) continue;

            const perMinute = tracked?.perMinute ?? this.estimatedPerMinute(settings, gaps);
@@ -411,7 +418,9 @@ export class BackfillQueue {

        while (!job.stop) {
            const group = this.plan(false).get(job.chatId);
-            const member = group?.members.find(candidate => candidate.pending > 0);
+            const member = group?.members.find(
+                candidate => candidate.pending > 0 || candidate.pendingTitles > 0
+            );
            if (!member) break;

            const limit = this.engine.rateLimitStatus();
@@ -59,6 +59,30 @@ export function ogImageFromHtml(html: string): string | null {
    return null;
 }

+/**
+ * The embed's title from the provider oEmbed, for YouTube-like rows. Null when the host is not
+ * YouTube or the provider does not answer; the backfill queue retries and then gives up.
+ */
+export async function resolveEmbedTitle(
+    url: string,
+    options: DownloadOptions = {}
+): Promise<string | null> {
+    let parsed: URL;
+    try {
+        parsed = new URL(url);
+    } catch {
+        return null;
+    }
+
+    const host = parsed.hostname.replace(/^www\./, "").toLowerCase();
+    if (host !== "youtube.com" && host !== "m.youtube.com" && host !== "youtu.be" && host !== "youtube-nocookie.com") {
+        return null;
+    }
+
+    const data = await getJson(`https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`, options);
+    return typeof data?.title === "string" && data.title.trim() ? data.title.trim() : null;
+}
+
 /**
 * A preview URL for an embed. Uses the stored thumbnail when present, otherwise asks the
 * provider (oEmbed, fxtwitter) and finally falls back to the page's og:image.
@@ -16,7 +16,7 @@ import {
    videoPoster,
    type DownloadOptions
 } from "./download.js";
-import { resolveEmbedPreview } from "./embed-preview.js";
+import { resolveEmbedPreview, resolveEmbedTitle } from "./embed-preview.js";
 import {
    createScrapeDirs,
    listScrapes,
@@ -52,6 +52,8 @@ export interface PreviewBatchResult {
    skipped: number;
    failed: number;
    ratelimited: number;
+    /** Titles resolved in this batch. */
+    titled: number;
    /** Rows still pending after the batch. */
    remaining: number;
    /** True when a stop request interrupted the batch. */
@@ -547,7 +549,9 @@ export class ScrapeEngine {
    async previewBatch(scrapeId: string, limit: number): Promise<PreviewBatchResult> {
        const dir = join(this.options.rootDir, scrapeId);
        const meta = this.active.get(scrapeId) ?? readScrapeMeta(dir);
-        if (!meta) return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, remaining: 0, stopped: false };
+        if (!meta) {
+            return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, titled: 0, remaining: 0, stopped: false };
+        }

        const releaseTraffic = await this.acquireTraffic();
        const store = new ScrapeStore(dir);
@@ -572,10 +576,47 @@ export class ScrapeEngine {
            skipped: 0,
            failed: 0,
            ratelimited: 0,
+            titled: 0,
            remaining: 0,
            stopped: false
        };
-        const pending = store.pendingPreviews(limit, true);
+
+        // Title work first, then previews with what is left of the batch. YouTube rows that
+        // never got a title share the queue with the preview pass.
+        const titleBudget = store.pendingTitles(1).length > 0 && limit > 1 ? Math.ceil(limit / 2) : limit;
+        for (const row of store.pendingTitles(titleBudget)) {
+            if (this.stopRequested.has(meta.id)) {
+                meta.state = "paused";
+                result.stopped = true;
+                break;
+            }
+
+            const title = await resolveEmbedTitle(row.url, options);
+            if (title) {
+                store.markTitle(row.id, title);
+                result.titled++;
+            } else {
+                const attempts = store.bumpTitleAttempt(row.id);
+                const maxAttempts = this.options.previewMaxAttempts ?? 3;
+                if (attempts >= maxAttempts) {
+                    store.markTitleError(row.id, `no title after ${attempts} attempt(s)`);
+                    this.log(`${meta.id}: title #${row.id} gave up after ${attempts} attempts (${row.url.slice(0, 60)})`);
+                } else {
+                    this.log(`${meta.id}: title #${row.id} unresolved (${attempts}/${maxAttempts}) (${row.url.slice(0, 60)})`);
+                }
+            }
+
+            this.persist(dir, meta);
+            await this.sleep(previewGapMs(options));
+        }
+
+        if (result.stopped) {
+            result.remaining = store.pendingPreviews(5000, true).length;
+            return result;
+        }
+
+        const previewLimit = Math.max(1, limit - result.titled);
+        const pending = store.pendingPreviews(previewLimit, true);

        for (const row of pending) {
            if (this.stopRequested.has(meta.id)) {
@@ -711,10 +752,10 @@ export class ScrapeEngine {
        }

        result.remaining = store.pendingPreviews(5000, true).length;
-        if (pending.length) {
+        if (pending.length || result.titled) {
            this.log(
                `${meta.id}: preview pass done: ${result.cached} cached, ${result.skipped} skipped, ` +
-                    `${result.failed} failed, ${result.remaining} still pending`
+                    `${result.failed} failed, ${result.titled} titled, ${result.remaining} still pending`
            );
        }

@@ -5,7 +5,7 @@

 export { extractMedia, type ExtractedMedia } from "./extract.js";
 export { betterCandidate, candidateScore, sortCandidates, type DedupeCandidate } from "./dedupe.js";
-export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
+export { ogImageFromHtml, resolveEmbedPreview, resolveEmbedTitle } from "./embed-preview.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
 export {
@@ -171,6 +171,8 @@ export interface MediaRow {
    preview_attempts: number;
    /** Embed title from Discord (YouTube and friends). */
    title: string | null;
+    title_attempts: number;
+    title_error: string | null;
    /** Lowercased, punctuation-collapsed title for token-prefix search. */
    title_key: string | null;
    created_at: string;
@@ -224,6 +226,8 @@ CREATE TABLE IF NOT EXISTS media (
    preview_attempts INTEGER NOT NULL DEFAULT 0,
    title TEXT,
    title_key TEXT,
+    title_attempts INTEGER NOT NULL DEFAULT 0,
+    title_error TEXT,
    created_at TEXT NOT NULL
 );

@@ -273,6 +277,10 @@ export class ScrapeStore {
            this.db.exec("ALTER TABLE media ADD COLUMN title TEXT");
            this.db.exec("ALTER TABLE media ADD COLUMN title_key TEXT");
        }
+        if (!names.has("title_attempts")) {
+            this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0");
+            this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
+        }
    }

    close(): void {
@@ -434,6 +442,8 @@ export class ScrapeStore {
        pending: number;
        cached: number;
        failed: number;
+        /** YouTube-like rows still missing a title. */
+        titlePending: number;
        byKind: Record<MediaKind, { total: number; pending: number; cached: number; failed: number }>;
    } {
        const empty = () => ({ total: 0, pending: 0, cached: 0, failed: 0 });
@@ -477,10 +487,58 @@ export class ScrapeStore {
            totals.failed += row.failed ?? 0;
        }

-        return { ...totals, byKind };
+        const titles = this.db
+            .prepare(
+                `SELECT COUNT(*) AS count FROM media
+                 WHERE title IS NULL AND title_error IS NULL
+                   AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
+                        OR lower(url) LIKE '%youtube.com/%'
+                        OR lower(url) LIKE '%youtu.be/%')`
+            )
+            .get() as { count: number | null } | undefined;
+
+        return { ...totals, titlePending: titles?.count ?? 0, byKind };
    }

    /** Count one transient preview failure and return the new total. */
+    /**
+     * YouTube-like rows whose title Discord did not provide, oldest first. Resolved through the
+     * provider oEmbed in the backfill queue.
+     */
+    pendingTitles(limit = 500): MediaRow[] {
+        return this.db
+            .prepare(
+                `SELECT * FROM media
+                 WHERE title IS NULL AND title_error IS NULL
+                   AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
+                        OR lower(url) LIKE '%youtube.com/%'
+                        OR lower(url) LIKE '%youtu.be/%')
+                 ORDER BY id LIMIT ?`
+            )
+            .all(limit) as unknown as MediaRow[];
+    }
+
+    /** Store a resolved title and its search key. */
+    markTitle(id: number, title: string): void {
+        this.db
+            .prepare("UPDATE media SET title = ?, title_key = ?, title_error = NULL WHERE id = ?")
+            .run(title.slice(0, 300), normalizeTitleKey(title), id);
+    }
+
+    /** Count one transient title failure and return the new total. */
+    bumpTitleAttempt(id: number): number {
+        this.db.prepare("UPDATE media SET title_attempts = title_attempts + 1 WHERE id = ?").run(id);
+        const row = this.db.prepare("SELECT title_attempts FROM media WHERE id = ?").get(id) as
+            | { title_attempts: number | null }
+            | undefined;
+
+        return row?.title_attempts ?? 0;
+    }
+
+    markTitleError(id: number, error: string): void {
+        this.db.prepare("UPDATE media SET title_error = ? WHERE id = ?").run(error.slice(0, 200), id);
+    }
+
    /** Count one transient preview failure and return the new total. */
    bumpPreviewAttempt(id: number): number {
        this.db.prepare("UPDATE media SET preview_attempts = preview_attempts + 1 WHERE id = ?").run(id);
@@ -1348,6 +1348,141 @@ void (async () => {
        store.close();
    });

+    await test("pendingTitles selects YouTube rows without a title", () => {
+        const dir = join(root, "pending-titles");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("1100", {
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=withtitle",
+                        title: "Has a title",
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/watch?v=withtitle",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=notitle",
+                        title: null,
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/watch?v=notitle",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://youtu.be/short",
+                        title: null,
+                        provider: "youtu.be",
+                        videoUrl: "https://youtu.be/short",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://clips.twitch.tv/notyoutube",
+                        title: null,
+                        provider: "Twitch",
+                        videoUrl: "https://clips.twitch.tv/notyoutube",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            })
+        ]);
+
+        const label = (url: string) => url.split("=")[1] ?? url.split("/").pop();
+        const pending = store.pendingTitles(10).map(row => label(row.url));
+        assert.deepEqual(pending.sort(), ["notitle", "short"].sort());
+
+        const target = store.pendingTitles(10).find(row => row.url.includes("notitle"));
+        const dead = store.pendingTitles(10).find(row => row.url.includes("youtu.be"));
+        assert.ok(target);
+        assert.ok(dead);
+
+        store.markTitle(target.id, "Resolved Title");
+        assert.equal(store.getMedia(target.id)?.title, "Resolved Title");
+        assert.equal(store.pendingTitles(10).some(row => row.id === target.id), false);
+
+        store.markTitleError(dead.id, "no title after 1 attempt(s)");
+        assert.equal(store.pendingTitles(10).some(row => row.id === dead.id), false);
+        assert.equal(store.previewCounts().titlePending, 0);
+        store.close();
+    });
+
+    await test("previewBatch resolves titles and gives up after the cap", async () => {
+        const embed = {
+            type: "video",
+            url: "https://www.youtube.com/watch?v=titlepass",
+            title: null,
+            provider: "YouTube",
+            videoUrl: "https://www.youtube.com/watch?v=titlepass",
+            imageUrl: null,
+            thumbnailUrl: null
+        };
+
+        const backend = new FakeBackend([{ messages: [message("1101", { embeds: [embed] })] }]);
+        const engine = engineFor(root, backend, {
+            previewMaxAttempts: 3,
+            download: {
+                fetchImpl: async (input: any) => {
+                    if (String(input).includes("oembed")) {
+                        return new Response(JSON.stringify({ title: "Real Title" }), {
+                            status: 200,
+                            headers: { "content-type": "application/json" }
+                        });
+                    }
+                    return new Response("no", { status: 404 });
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "135", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        await engine.previewBatch(started.id, 5);
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.title, "Real Title");
+        assert.equal(row.title_key, "real title");
+        assert.equal(store.previewCounts().titlePending, 0);
+        store.close();
+
+        // A provider that never answers is closed out after the cap.
+        const deadBackend = new FakeBackend([
+            { messages: [message("1102", { embeds: [{ ...embed, url: "https://youtu.be/deadtitle", videoUrl: "https://youtu.be/deadtitle" }] })] }
+        ]);
+        const deadEngine = engineFor(root, deadBackend, {
+            previewMaxAttempts: 1,
+            download: {
+                fetchImpl: async () => {
+                    throw new Error("fetch failed");
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const dead = await deadEngine.start({ channelId: "136", guildId: null, mode: "links", backend: deadBackend });
+        await deadEngine.processQueue();
+        await deadEngine.previewBatch(dead.id, 5);
+
+        const deadStore = new ScrapeStore(join(root, dead.id));
+        const deadRow = deadStore.queryMedia({ kind: "video" })[0];
+        assert.match(String(deadRow.title_error), /^no title after 1 attempt/);
+        assert.equal(deadStore.previewCounts().titlePending, 0);
+        deadStore.close();
+    });
+
    await test("title search matches word prefixes and exact phrases", () => {
        const dir = join(root, "title-search");
        mkdirSync(dir, { recursive: true });