Store embed titles and search them

media-scrape/src/extract.ts

@@ -32,6 +32,8 @@ export interface ExtractedMedia {
     /** False for every embed. Downloads are a later step, never for third-party embeds. */
     downloaded: boolean;
     previewUrl: string | null;
+    /** Embed title, when Discord provides one (YouTube and friends). Null elsewhere. */
+    title: string | null;
 }
 
 function extension(filename: string | null | undefined): string {
@@ -81,7 +83,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
         size: null,
         width: embed.width ?? null,
         height: embed.height ?? null,
-        durationSecs: null
+        durationSecs: null,
+        title: embed.title ?? null
     };
 
     if (embed.videoUrl) {
@@ -135,7 +138,8 @@ function extractSticker(sticker: CoreSticker): ExtractedMedia | null {
         height: null,
         durationSecs: null,
         downloaded: false,
-        previewUrl: null
+        previewUrl: null,
+        title: null
     };
 }
 
@@ -162,7 +166,8 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] {
                     ? attachment.durationSecs
                     : null,
             downloaded: false,
-            previewUrl: null
+            previewUrl: null,
+            title: null
         });
     }
 

media-scrape/src/store.ts

@@ -136,6 +136,9 @@ export interface MediaFilterOptions {
     limit?: number;
     offset?: number;
     sort?: MediaSort;
+    /** Title search: multi-word prefix tokens by default, a phrase when `titleExact`. */
+    titleQuery?: string;
+    titleExact?: boolean;
 }
 
 export interface MediaRow {
@@ -166,9 +169,22 @@ export interface MediaRow {
     download_error: string | null;
     preview_error: string | null;
     preview_attempts: number;
+    /** Embed title from Discord (YouTube and friends). */
+    title: string | null;
+    /** Lowercased, punctuation-collapsed title for token-prefix search. */
+    title_key: string | null;
     created_at: string;
 }
 
+/** Normalize a title for prefix-token search: lowercase, punctuation to spaces, collapsed. */
+export function normalizeTitleKey(title: string): string {
+    return title
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, " ")
+        .trim()
+        .replace(/\s+/g, " ");
+}
+
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS messages (
     id TEXT PRIMARY KEY,
@@ -206,6 +222,8 @@ CREATE TABLE IF NOT EXISTS media (
     download_error TEXT,
     preview_error TEXT,
     preview_attempts INTEGER NOT NULL DEFAULT 0,
+    title TEXT,
+    title_key TEXT,
     created_at TEXT NOT NULL
 );
 
@@ -251,6 +269,10 @@ export class ScrapeStore {
         if (!names.has("preview_attempts")) {
             this.db.exec("ALTER TABLE media ADD COLUMN preview_attempts INTEGER NOT NULL DEFAULT 0");
         }
+        if (!names.has("title")) {
+            this.db.exec("ALTER TABLE media ADD COLUMN title TEXT");
+            this.db.exec("ALTER TABLE media ADD COLUMN title_key TEXT");
+        }
     }
 
     close(): void {
@@ -264,7 +286,7 @@ export class ScrapeStore {
             "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
         );
         const insertMedia = this.db.prepare(
-            "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+            "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
         );
         const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
         const linkSource = this.db.prepare(
@@ -301,6 +323,8 @@ export class ScrapeStore {
                         item.durationSecs,
                         item.downloaded ? 1 : 0,
                         item.previewUrl,
+                        item.title,
+                        item.title ? normalizeTitleKey(item.title) : null,
                         createdAt
                     );
                     if (result.changes > 0) added[item.kind]++;
@@ -640,6 +664,18 @@ export class ScrapeStore {
             conditions.push("duration_secs > ?");
             values.push(options.lengthGreaterThan);
         }
+        if (options.titleQuery?.trim()) {
+            const query = normalizeTitleKey(options.titleQuery);
+            if (options.titleExact) {
+                conditions.push("title_key LIKE ?");
+                values.push(`%${query}%`);
+            } else {
+                for (const token of query.split(" ").filter(Boolean)) {
+                    conditions.push("(' ' || COALESCE(title_key, '')) LIKE ?");
+                    values.push(`% ${token}%`);
+                }
+            }
+        }
         if (options.after) {
             conditions.push("posted_at >= ?");
             values.push(options.after);

media-scrape/test/run.ts

@@ -29,6 +29,7 @@ import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
 import {
     dirSizeBytes,
     listScrapes,
+    normalizeTitleKey,
     readScrapeMeta,
     ScrapeStore,
     writeScrapeMeta
@@ -1347,6 +1348,61 @@ void (async () => {
         store.close();
     });
 
+    await test("title search matches word prefixes and exact phrases", () => {
+        const dir = join(root, "title-search");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+        const embed = (url: string, title: string) => ({
+            type: "video",
+            url,
+            title,
+            provider: "YouTube",
+            videoUrl: url,
+            imageUrl: null,
+            thumbnailUrl: null
+        });
+
+        store.insertPage([
+            message("1000", {
+                attachments: [attachment({ filename: "no-title.png", contentType: "image/png" })],
+                embeds: [
+                    embed("https://www.youtube.com/watch?v=a", "Man who jumps over car"),
+                    embed("https://www.youtube.com/watch?v=b", "Human cannonball"),
+                    embed("https://www.youtube.com/watch?v=c", "Woman's Day: 3 Things!")
+                ]
+            })
+        ]);
+
+        const ids = (options: Record<string, unknown>) =>
+            store
+                .queryMedia({ limit: 50, ...options })
+                .map(row => row.url.split("v=")[1])
+                .sort();
+
+        // Prefix tokens in any order, but only at word starts.
+        assert.deepEqual(ids({ titleQuery: "Ma Ca Ju" }), ["a"]);
+        assert.deepEqual(ids({ titleQuery: "man" }), ["a"]);
+        assert.deepEqual(ids({ titleQuery: "man car" }), ["a"]);
+        assert.deepEqual(ids({ titleQuery: "cannon" }), ["b"]);
+        assert.deepEqual(ids({ titleQuery: "hum" }), ["b"]);
+        assert.deepEqual(ids({ titleQuery: "an" }), []);
+
+        // Exact mode is one phrase.
+        assert.deepEqual(ids({ titleQuery: "man who jumps", titleExact: true }), ["a"]);
+        assert.deepEqual(ids({ titleQuery: "man car", titleExact: true }), []);
+
+        // Punctuation folds away.
+        assert.deepEqual(ids({ titleQuery: "woman's day" }), ["c"]);
+        assert.deepEqual(ids({ titleQuery: "3 things" }), ["c"]);
+
+        store.close();
+    });
+
+    await test("normalizeTitleKey collapses punctuation and case", () => {
+        assert.equal(normalizeTitleKey("Woman's Day: 3 Things!"), "woman s day 3 things");
+        assert.equal(normalizeTitleKey("  A---B  "), "a b");
+    });
+
     await test("provider filters are the curated archive list", () => {
         assert.ok(PROVIDER_FILTERS.length >= 12, `provider list has ${PROVIDER_FILTERS.length} entries`);
         assert.equal(new Set(PROVIDER_FILTERS).size, PROVIDER_FILTERS.length);