Normalize YouTube Shorts links to watch URLs

CURRENT.md

@@ -40,6 +40,14 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C51 YouTube Shorts normalization. Shorts links broke the embedded player and View full
+    because the viewer needs `watch?v=`. Extraction now rewrites `youtube.com/shorts/<id>` (and
+    `m.`) to the watch form, the viewer also accepts `/shorts/<id>` defensively, and a one-time
+    `PRAGMA user_version` migration rewrites existing rows. The live archive had 45 shorts rows;
+    one collided with an already indexed watch URL and folded into it (preview fields merged,
+    message links moved) — the migration is idempotent and unit-tested, including the collision.
+    45 → 0 shorts rows, and the user's example `uAf18d7mrnU` is now
+    `watch?v=uAf18d7mrnU` with `embed/uAf18d7mrnU` as the play URL.
   - [x] C50 Title backfill in the queue. YouTube-like rows (provider or URL matching
     youtube/youtu.be) without a title are now a second work type in the backfill queue: resolved
     through the provider oEmbed, stored as `title`/`title_key`, with `title_attempts` and

docs/media-scraper.md

@@ -96,7 +96,10 @@ One SQLite file per scrape (`index.db`, `node:sqlite`).
 slopbot backend could ask for it (one request; null otherwise). Indexes: `kind`, `posted_at`, and
 `media_sources(message_id)`. The database is an index; the media
 folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time
-(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Embed rows also store
+(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). `youtube.com/shorts/<id>`
+links are stored as `watch?v=<id>` (extraction and a one-time `user_version` migration), because the
+embedded player and View full need the watch form; a shorts row that collides with an already
+indexed watch URL folds into it (preview/file fields merge, message links move over). Embed rows also store
 Discord's `title` (YouTube and friends) plus `title_key`, the lowercased, punctuation-collapsed
 form used by the title search: multi-word prefix tokens by default ("Ma Ca Ju" finds *Man who jumps
 over car*), one phrase with the exact-match toggle, and rows without a title drop out while a

media-scrape/src/extract.ts

@@ -36,6 +36,15 @@ export interface ExtractedMedia {
     title: string | null;
 }
 
+/**
+ * Shorts links break the embed player: it needs `watch?v=`. Rewrite them for storage so the
+ * viewer, "View full", and refreshes all use the canonical form. Other URLs pass through.
+ */
+export function normalizeYouTubeUrl(url: string): string {
+    const match = /^https?:\/\/(?:www\.|m\.)?youtube\.com\/shorts\/([A-Za-z0-9_-]{6,})/.exec(url);
+    return match ? `https://www.youtube.com/watch?v=${match[1]}` : url;
+}
+
 function extension(filename: string | null | undefined): string {
     const match = /\.([a-z0-9]+)$/i.exec(filename ?? "");
     return match ? match[1].toLowerCase() : "";
@@ -75,9 +84,11 @@ function hostname(url: string | null): string | null {
 
 function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
     const provider = embed.provider ?? hostname(embed.url);
+    const embedUrl = embed.url ? normalizeYouTubeUrl(embed.url) : null;
+    const videoUrl = embed.videoUrl ? normalizeYouTubeUrl(embed.videoUrl) : null;
     const common = {
         source: "embed" as const,
-        playUrl: embed.videoUrl ?? null,
+        playUrl: videoUrl,
         provider,
         downloaded: false,
         size: null,
@@ -91,7 +102,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
         return {
             ...common,
             kind: "video",
-            url: embed.url ?? embed.videoUrl,
+            url: embedUrl ?? videoUrl ?? "",
             contentType: null,
             previewUrl: embed.thumbnailUrl ?? embed.imageUrl
         };
@@ -102,7 +113,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
         return {
             ...common,
             kind: "gif",
-            url: embed.url ?? embed.imageUrl ?? "",
+            url: embedUrl ?? embed.imageUrl ?? "",
             contentType: null,
             previewUrl: embed.thumbnailUrl ?? embed.imageUrl
         };
@@ -112,7 +123,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
         return {
             ...common,
             kind: "image",
-            url: embed.url ?? embed.imageUrl,
+            url: embedUrl ?? embed.imageUrl,
             contentType: null,
             previewUrl: embed.thumbnailUrl ?? embed.imageUrl
         };

media-scrape/src/store.ts

@@ -7,7 +7,7 @@ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "n
 import { join } from "node:path";
 import { DatabaseSync } from "node:sqlite";
 import { fileURLToPath } from "node:url";
-import { extractMedia } from "./extract.js";
+import { extractMedia, normalizeYouTubeUrl } from "./extract.js";
 import { aspectCondition, resolutionCondition } from "./filter-presets.js";
 import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
 
@@ -281,6 +281,86 @@ export class ScrapeStore {
             this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0");
             this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
         }
+
+        this.migrateYouTubeShorts();
+    }
+
+    /**
+     * One-time rewrite of youtube.com/shorts links to watch URLs. The embed player needs `v=`,
+     * so old rows with a shorts URL would never play. `PRAGMA user_version` makes it idempotent
+     * per scrape database.
+     */
+    private migrateYouTubeShorts(): void {
+        const version = this.db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined;
+        if ((version?.user_version ?? 0) >= 1) return;
+
+        const rows = this.db
+            .prepare(
+                "SELECT * FROM media WHERE url LIKE '%youtube.com/shorts/%' OR play_url LIKE '%youtube.com/shorts/%'"
+            )
+            .all() as unknown as MediaRow[];
+
+        if (rows.length) {
+            const update = this.db.prepare("UPDATE media SET url = ?, play_url = ? WHERE id = ?");
+            const findTarget = this.db.prepare("SELECT id FROM media WHERE url = ? AND id != ?");
+            const merge = this.db.prepare(
+                `UPDATE media SET
+                    preview_hash = COALESCE(preview_hash, ?),
+                    preview_bytes = COALESCE(preview_bytes, ?),
+                    preview_ext = COALESCE(preview_ext, ?),
+                    file_path = COALESCE(file_path, ?),
+                    file_hash = COALESCE(file_hash, ?),
+                    file_bytes = COALESCE(file_bytes, ?),
+                    downloaded = MAX(downloaded, ?),
+                    title = COALESCE(title, ?),
+                    title_key = COALESCE(title_key, ?),
+                    play_url = COALESCE(play_url, ?)
+                 WHERE id = ?`
+            );
+            const relink = this.db.prepare(
+                "UPDATE OR IGNORE media_sources SET media_id = ? WHERE media_id = ?"
+            );
+            const dropLinks = this.db.prepare("DELETE FROM media_sources WHERE media_id = ?");
+            const drop = this.db.prepare("DELETE FROM media WHERE id = ?");
+
+            this.db.exec("BEGIN");
+            try {
+                for (const row of rows) {
+                    const url = normalizeYouTubeUrl(row.url);
+                    const playUrl = row.play_url ? normalizeYouTubeUrl(row.play_url) : null;
+                    const target = findTarget.get(url, row.id) as { id: number } | undefined;
+
+                    // The same video was already indexed under its watch URL: fold this row in.
+                    if (!target) {
+                        update.run(url, playUrl, row.id);
+                        continue;
+                    }
+
+                    merge.run(
+                        row.preview_hash,
+                        row.preview_bytes,
+                        row.preview_ext,
+                        row.file_path,
+                        row.file_hash,
+                        row.file_bytes,
+                        row.downloaded,
+                        row.title,
+                        row.title_key,
+                        playUrl,
+                        target.id
+                    );
+                    relink.run(target.id, row.id);
+                    dropLinks.run(row.id);
+                    drop.run(row.id);
+                }
+                this.db.exec("COMMIT");
+            } catch (error) {
+                this.db.exec("ROLLBACK");
+                throw error;
+            }
+        }
+
+        this.db.exec("PRAGMA user_version = 1");
     }
 
     close(): void {

media-scrape/test/run.ts

@@ -5,6 +5,7 @@
 
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
+import { DatabaseSync } from "node:sqlite";
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -20,7 +21,7 @@ import {
 import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { BackfillQueue } from "../src/backfill.js";
-import { extractMedia } from "../src/extract.js";
+import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { betterCandidate, sortCandidates } from "../src/dedupe.js";
 import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
@@ -1348,6 +1349,91 @@ void (async () => {
         store.close();
     });
 
+    await test("youtube shorts links normalize to watch URLs", () => {
+        assert.equal(
+            normalizeYouTubeUrl("https://www.youtube.com/shorts/uAf18d7mrnU"),
+            "https://www.youtube.com/watch?v=uAf18d7mrnU"
+        );
+        assert.equal(
+            normalizeYouTubeUrl("https://m.youtube.com/shorts/abc123XYZ?feature=share"),
+            "https://www.youtube.com/watch?v=abc123XYZ"
+        );
+        assert.equal(
+            normalizeYouTubeUrl("https://www.youtube.com/watch?v=keep"),
+            "https://www.youtube.com/watch?v=keep"
+        );
+        assert.equal(normalizeYouTubeUrl("https://youtu.be/short"), "https://youtu.be/short");
+    });
+
+    await test("shorts store normalized and old rows migrate", () => {
+        const dir = join(root, "shorts");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("1200", {
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/shorts/uAf18d7mrnU",
+                        title: "A short",
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/shorts/uAf18d7mrnU",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            })
+        ]);
+
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
+        assert.equal(row.play_url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
+        store.close();
+
+        // An old database (user_version 0) gets rewritten on open.
+        const legacyDir = join(root, "shorts-legacy");
+        mkdirSync(legacyDir, { recursive: true });
+        new ScrapeStore(legacyDir).close();
+
+        const raw = new DatabaseSync(join(legacyDir, "index.db"));
+        const insert = raw.prepare(
+            "INSERT INTO media (kind, source, url, play_url, provider, downloaded, preview_hash, created_at) VALUES ('video', 'embed', ?, ?, 'YouTube', 0, ?, ?)"
+        );
+        insert.run(
+            "https://www.youtube.com/shorts/legacyId123",
+            "https://www.youtube.com/shorts/legacyId123",
+            null,
+            "2026-09-17T00:00:00.000Z"
+        );
+        // The same video already indexed under its watch URL: the shorts row folds in.
+        insert.run(
+            "https://www.youtube.com/watch?v=legacyId456",
+            null,
+            null,
+            "2026-09-17T00:00:00.000Z"
+        );
+        insert.run(
+            "https://www.youtube.com/shorts/legacyId456",
+            "https://www.youtube.com/shorts/legacyId456",
+            "previewhash",
+            "2026-09-17T00:00:00.000Z"
+        );
+        raw.exec("PRAGMA user_version = 0");
+        raw.close();
+
+        const reopened = new ScrapeStore(legacyDir);
+        const rows = reopened.queryMedia({ kind: "video", limit: 50 });
+        assert.equal(rows.length, 2);
+        const legacyRow = rows.find(row => row.url.includes("legacyId123"));
+        assert.ok(legacyRow);
+        assert.equal(legacyRow.play_url, "https://www.youtube.com/watch?v=legacyId123");
+        const merged = rows.find(row => row.url.includes("legacyId456"));
+        assert.ok(merged);
+        assert.equal(merged.preview_hash, "previewhash");
+        reopened.close();
+    });
+
     await test("pendingTitles selects YouTube rows without a title", () => {
         const dir = join(root, "pending-titles");
         mkdirSync(dir, { recursive: true });

poc/public/media.js

@@ -2238,7 +2238,8 @@
             const host = url.hostname.replace(/^www\./, "");
 
             if (host === "youtube.com" || host === "m.youtube.com") {
-                const id = url.searchParams.get("v");
+                const id =
+                    url.searchParams.get("v") ?? /^\/shorts\/([A-Za-z0-9_-]+)/.exec(url.pathname)?.[1];
                 if (id) return `https://www.youtube.com/embed/${id}`;
             }
             if (host === "youtu.be") return `https://www.youtube.com/embed${url.pathname}`;