Backfill YouTube titles through the queue

media-scrape/test/run.ts

@@ -1348,6 +1348,141 @@ void (async () => {
         store.close();
     });
 
+    await test("pendingTitles selects YouTube rows without a title", () => {
+        const dir = join(root, "pending-titles");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("1100", {
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=withtitle",
+                        title: "Has a title",
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/watch?v=withtitle",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/watch?v=notitle",
+                        title: null,
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/watch?v=notitle",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://youtu.be/short",
+                        title: null,
+                        provider: "youtu.be",
+                        videoUrl: "https://youtu.be/short",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    },
+                    {
+                        type: "video",
+                        url: "https://clips.twitch.tv/notyoutube",
+                        title: null,
+                        provider: "Twitch",
+                        videoUrl: "https://clips.twitch.tv/notyoutube",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            })
+        ]);
+
+        const label = (url: string) => url.split("=")[1] ?? url.split("/").pop();
+        const pending = store.pendingTitles(10).map(row => label(row.url));
+        assert.deepEqual(pending.sort(), ["notitle", "short"].sort());
+
+        const target = store.pendingTitles(10).find(row => row.url.includes("notitle"));
+        const dead = store.pendingTitles(10).find(row => row.url.includes("youtu.be"));
+        assert.ok(target);
+        assert.ok(dead);
+
+        store.markTitle(target.id, "Resolved Title");
+        assert.equal(store.getMedia(target.id)?.title, "Resolved Title");
+        assert.equal(store.pendingTitles(10).some(row => row.id === target.id), false);
+
+        store.markTitleError(dead.id, "no title after 1 attempt(s)");
+        assert.equal(store.pendingTitles(10).some(row => row.id === dead.id), false);
+        assert.equal(store.previewCounts().titlePending, 0);
+        store.close();
+    });
+
+    await test("previewBatch resolves titles and gives up after the cap", async () => {
+        const embed = {
+            type: "video",
+            url: "https://www.youtube.com/watch?v=titlepass",
+            title: null,
+            provider: "YouTube",
+            videoUrl: "https://www.youtube.com/watch?v=titlepass",
+            imageUrl: null,
+            thumbnailUrl: null
+        };
+
+        const backend = new FakeBackend([{ messages: [message("1101", { embeds: [embed] })] }]);
+        const engine = engineFor(root, backend, {
+            previewMaxAttempts: 3,
+            download: {
+                fetchImpl: async (input: any) => {
+                    if (String(input).includes("oembed")) {
+                        return new Response(JSON.stringify({ title: "Real Title" }), {
+                            status: 200,
+                            headers: { "content-type": "application/json" }
+                        });
+                    }
+                    return new Response("no", { status: 404 });
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "135", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        await engine.previewBatch(started.id, 5);
+
+        const store = new ScrapeStore(join(root, started.id));
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.title, "Real Title");
+        assert.equal(row.title_key, "real title");
+        assert.equal(store.previewCounts().titlePending, 0);
+        store.close();
+
+        // A provider that never answers is closed out after the cap.
+        const deadBackend = new FakeBackend([
+            { messages: [message("1102", { embeds: [{ ...embed, url: "https://youtu.be/deadtitle", videoUrl: "https://youtu.be/deadtitle" }] })] }
+        ]);
+        const deadEngine = engineFor(root, deadBackend, {
+            previewMaxAttempts: 1,
+            download: {
+                fetchImpl: async () => {
+                    throw new Error("fetch failed");
+                },
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const dead = await deadEngine.start({ channelId: "136", guildId: null, mode: "links", backend: deadBackend });
+        await deadEngine.processQueue();
+        await deadEngine.previewBatch(dead.id, 5);
+
+        const deadStore = new ScrapeStore(join(root, dead.id));
+        const deadRow = deadStore.queryMedia({ kind: "video" })[0];
+        assert.match(String(deadRow.title_error), /^no title after 1 attempt/);
+        assert.equal(deadStore.previewCounts().titlePending, 0);
+        deadStore.close();
+    });
+
     await test("title search matches word prefixes and exact phrases", () => {
         const dir = join(root, "title-search");
         mkdirSync(dir, { recursive: true });