Normalize YouTube Shorts links to watch URLs

media-scrape/test/run.ts

@@ -5,6 +5,7 @@
 
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
+import { DatabaseSync } from "node:sqlite";
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -20,7 +21,7 @@ import {
 import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { BackfillQueue } from "../src/backfill.js";
-import { extractMedia } from "../src/extract.js";
+import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { betterCandidate, sortCandidates } from "../src/dedupe.js";
 import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
@@ -1348,6 +1349,91 @@ void (async () => {
         store.close();
     });
 
+    await test("youtube shorts links normalize to watch URLs", () => {
+        assert.equal(
+            normalizeYouTubeUrl("https://www.youtube.com/shorts/uAf18d7mrnU"),
+            "https://www.youtube.com/watch?v=uAf18d7mrnU"
+        );
+        assert.equal(
+            normalizeYouTubeUrl("https://m.youtube.com/shorts/abc123XYZ?feature=share"),
+            "https://www.youtube.com/watch?v=abc123XYZ"
+        );
+        assert.equal(
+            normalizeYouTubeUrl("https://www.youtube.com/watch?v=keep"),
+            "https://www.youtube.com/watch?v=keep"
+        );
+        assert.equal(normalizeYouTubeUrl("https://youtu.be/short"), "https://youtu.be/short");
+    });
+
+    await test("shorts store normalized and old rows migrate", () => {
+        const dir = join(root, "shorts");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+
+        store.insertPage([
+            message("1200", {
+                embeds: [
+                    {
+                        type: "video",
+                        url: "https://www.youtube.com/shorts/uAf18d7mrnU",
+                        title: "A short",
+                        provider: "YouTube",
+                        videoUrl: "https://www.youtube.com/shorts/uAf18d7mrnU",
+                        imageUrl: null,
+                        thumbnailUrl: null
+                    }
+                ]
+            })
+        ]);
+
+        const row = store.queryMedia({ kind: "video" })[0];
+        assert.equal(row.url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
+        assert.equal(row.play_url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
+        store.close();
+
+        // An old database (user_version 0) gets rewritten on open.
+        const legacyDir = join(root, "shorts-legacy");
+        mkdirSync(legacyDir, { recursive: true });
+        new ScrapeStore(legacyDir).close();
+
+        const raw = new DatabaseSync(join(legacyDir, "index.db"));
+        const insert = raw.prepare(
+            "INSERT INTO media (kind, source, url, play_url, provider, downloaded, preview_hash, created_at) VALUES ('video', 'embed', ?, ?, 'YouTube', 0, ?, ?)"
+        );
+        insert.run(
+            "https://www.youtube.com/shorts/legacyId123",
+            "https://www.youtube.com/shorts/legacyId123",
+            null,
+            "2026-09-17T00:00:00.000Z"
+        );
+        // The same video already indexed under its watch URL: the shorts row folds in.
+        insert.run(
+            "https://www.youtube.com/watch?v=legacyId456",
+            null,
+            null,
+            "2026-09-17T00:00:00.000Z"
+        );
+        insert.run(
+            "https://www.youtube.com/shorts/legacyId456",
+            "https://www.youtube.com/shorts/legacyId456",
+            "previewhash",
+            "2026-09-17T00:00:00.000Z"
+        );
+        raw.exec("PRAGMA user_version = 0");
+        raw.close();
+
+        const reopened = new ScrapeStore(legacyDir);
+        const rows = reopened.queryMedia({ kind: "video", limit: 50 });
+        assert.equal(rows.length, 2);
+        const legacyRow = rows.find(row => row.url.includes("legacyId123"));
+        assert.ok(legacyRow);
+        assert.equal(legacyRow.play_url, "https://www.youtube.com/watch?v=legacyId123");
+        const merged = rows.find(row => row.url.includes("legacyId456"));
+        assert.ok(merged);
+        assert.equal(merged.preview_hash, "previewhash");
+        reopened.close();
+    });
+
     await test("pendingTitles selects YouTube rows without a title", () => {
         const dir = join(root, "pending-titles");
         mkdirSync(dir, { recursive: true });