diff --git a/CURRENT.md b/CURRENT.md index bbe16a8..39bf036 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,14 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C51 YouTube Shorts normalization. Shorts links broke the embedded player and View full + because the viewer needs `watch?v=`. Extraction now rewrites `youtube.com/shorts/` (and + `m.`) to the watch form, the viewer also accepts `/shorts/` defensively, and a one-time + `PRAGMA user_version` migration rewrites existing rows. The live archive had 45 shorts rows; + one collided with an already indexed watch URL and folded into it (preview fields merged, + message links moved) — the migration is idempotent and unit-tested, including the collision. + 45 → 0 shorts rows, and the user's example `uAf18d7mrnU` is now + `watch?v=uAf18d7mrnU` with `embed/uAf18d7mrnU` as the play URL. - [x] C50 Title backfill in the queue. YouTube-like rows (provider or URL matching youtube/youtu.be) without a title are now a second work type in the backfill queue: resolved through the provider oEmbed, stored as `title`/`title_key`, with `title_attempts` and diff --git a/docs/media-scraper.md b/docs/media-scraper.md index d1f042a..8dbc6c1 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -96,7 +96,10 @@ One SQLite file per scrape (`index.db`, `node:sqlite`). slopbot backend could ask for it (one request; null otherwise). Indexes: `kind`, `posted_at`, and `media_sources(message_id)`. The database is an index; the media folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time -(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Embed rows also store +(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). `youtube.com/shorts/` +links are stored as `watch?v=` (extraction and a one-time `user_version` migration), because the +embedded player and View full need the watch form; a shorts row that collides with an already +indexed watch URL folds into it (preview/file fields merge, message links move over). Embed rows also store Discord's `title` (YouTube and friends) plus `title_key`, the lowercased, punctuation-collapsed form used by the title search: multi-word prefix tokens by default ("Ma Ca Ju" finds *Man who jumps over car*), one phrase with the exact-match toggle, and rows without a title drop out while a diff --git a/media-scrape/src/extract.ts b/media-scrape/src/extract.ts index 682fc22..c11ed16 100644 --- a/media-scrape/src/extract.ts +++ b/media-scrape/src/extract.ts @@ -36,6 +36,15 @@ export interface ExtractedMedia { title: string | null; } +/** + * Shorts links break the embed player: it needs `watch?v=`. Rewrite them for storage so the + * viewer, "View full", and refreshes all use the canonical form. Other URLs pass through. + */ +export function normalizeYouTubeUrl(url: string): string { + const match = /^https?:\/\/(?:www\.|m\.)?youtube\.com\/shorts\/([A-Za-z0-9_-]{6,})/.exec(url); + return match ? `https://www.youtube.com/watch?v=${match[1]}` : url; +} + function extension(filename: string | null | undefined): string { const match = /\.([a-z0-9]+)$/i.exec(filename ?? ""); return match ? match[1].toLowerCase() : ""; @@ -75,9 +84,11 @@ function hostname(url: string | null): string | null { function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { const provider = embed.provider ?? hostname(embed.url); + const embedUrl = embed.url ? normalizeYouTubeUrl(embed.url) : null; + const videoUrl = embed.videoUrl ? normalizeYouTubeUrl(embed.videoUrl) : null; const common = { source: "embed" as const, - playUrl: embed.videoUrl ?? null, + playUrl: videoUrl, provider, downloaded: false, size: null, @@ -91,7 +102,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { return { ...common, kind: "video", - url: embed.url ?? embed.videoUrl, + url: embedUrl ?? videoUrl ?? "", contentType: null, previewUrl: embed.thumbnailUrl ?? embed.imageUrl }; @@ -102,7 +113,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { return { ...common, kind: "gif", - url: embed.url ?? embed.imageUrl ?? "", + url: embedUrl ?? embed.imageUrl ?? "", contentType: null, previewUrl: embed.thumbnailUrl ?? embed.imageUrl }; @@ -112,7 +123,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { return { ...common, kind: "image", - url: embed.url ?? embed.imageUrl, + url: embedUrl ?? embed.imageUrl, contentType: null, previewUrl: embed.thumbnailUrl ?? embed.imageUrl }; diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index d678e2a..2dd02f3 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -7,7 +7,7 @@ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "n import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { fileURLToPath } from "node:url"; -import { extractMedia } from "./extract.js"; +import { extractMedia, normalizeYouTubeUrl } from "./extract.js"; import { aspectCondition, resolutionCondition } from "./filter-presets.js"; import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js"; @@ -281,6 +281,86 @@ export class ScrapeStore { this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0"); this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT"); } + + this.migrateYouTubeShorts(); + } + + /** + * One-time rewrite of youtube.com/shorts links to watch URLs. The embed player needs `v=`, + * so old rows with a shorts URL would never play. `PRAGMA user_version` makes it idempotent + * per scrape database. + */ + private migrateYouTubeShorts(): void { + const version = this.db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined; + if ((version?.user_version ?? 0) >= 1) return; + + const rows = this.db + .prepare( + "SELECT * FROM media WHERE url LIKE '%youtube.com/shorts/%' OR play_url LIKE '%youtube.com/shorts/%'" + ) + .all() as unknown as MediaRow[]; + + if (rows.length) { + const update = this.db.prepare("UPDATE media SET url = ?, play_url = ? WHERE id = ?"); + const findTarget = this.db.prepare("SELECT id FROM media WHERE url = ? AND id != ?"); + const merge = this.db.prepare( + `UPDATE media SET + preview_hash = COALESCE(preview_hash, ?), + preview_bytes = COALESCE(preview_bytes, ?), + preview_ext = COALESCE(preview_ext, ?), + file_path = COALESCE(file_path, ?), + file_hash = COALESCE(file_hash, ?), + file_bytes = COALESCE(file_bytes, ?), + downloaded = MAX(downloaded, ?), + title = COALESCE(title, ?), + title_key = COALESCE(title_key, ?), + play_url = COALESCE(play_url, ?) + WHERE id = ?` + ); + const relink = this.db.prepare( + "UPDATE OR IGNORE media_sources SET media_id = ? WHERE media_id = ?" + ); + const dropLinks = this.db.prepare("DELETE FROM media_sources WHERE media_id = ?"); + const drop = this.db.prepare("DELETE FROM media WHERE id = ?"); + + this.db.exec("BEGIN"); + try { + for (const row of rows) { + const url = normalizeYouTubeUrl(row.url); + const playUrl = row.play_url ? normalizeYouTubeUrl(row.play_url) : null; + const target = findTarget.get(url, row.id) as { id: number } | undefined; + + // The same video was already indexed under its watch URL: fold this row in. + if (!target) { + update.run(url, playUrl, row.id); + continue; + } + + merge.run( + row.preview_hash, + row.preview_bytes, + row.preview_ext, + row.file_path, + row.file_hash, + row.file_bytes, + row.downloaded, + row.title, + row.title_key, + playUrl, + target.id + ); + relink.run(target.id, row.id); + dropLinks.run(row.id); + drop.run(row.id); + } + this.db.exec("COMMIT"); + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + this.db.exec("PRAGMA user_version = 1"); } close(): void { diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 8679a57..fc75c95 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -5,6 +5,7 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; +import { DatabaseSync } from "node:sqlite"; import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -20,7 +21,7 @@ import { import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { BackfillQueue } from "../src/backfill.js"; -import { extractMedia } from "../src/extract.js"; +import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; import { betterCandidate, sortCandidates } from "../src/dedupe.js"; import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js"; @@ -1348,6 +1349,91 @@ void (async () => { store.close(); }); + await test("youtube shorts links normalize to watch URLs", () => { + assert.equal( + normalizeYouTubeUrl("https://www.youtube.com/shorts/uAf18d7mrnU"), + "https://www.youtube.com/watch?v=uAf18d7mrnU" + ); + assert.equal( + normalizeYouTubeUrl("https://m.youtube.com/shorts/abc123XYZ?feature=share"), + "https://www.youtube.com/watch?v=abc123XYZ" + ); + assert.equal( + normalizeYouTubeUrl("https://www.youtube.com/watch?v=keep"), + "https://www.youtube.com/watch?v=keep" + ); + assert.equal(normalizeYouTubeUrl("https://youtu.be/short"), "https://youtu.be/short"); + }); + + await test("shorts store normalized and old rows migrate", () => { + const dir = join(root, "shorts"); + mkdirSync(dir, { recursive: true }); + const store = new ScrapeStore(dir); + + store.insertPage([ + message("1200", { + embeds: [ + { + type: "video", + url: "https://www.youtube.com/shorts/uAf18d7mrnU", + title: "A short", + provider: "YouTube", + videoUrl: "https://www.youtube.com/shorts/uAf18d7mrnU", + imageUrl: null, + thumbnailUrl: null + } + ] + }) + ]); + + const row = store.queryMedia({ kind: "video" })[0]; + assert.equal(row.url, "https://www.youtube.com/watch?v=uAf18d7mrnU"); + assert.equal(row.play_url, "https://www.youtube.com/watch?v=uAf18d7mrnU"); + store.close(); + + // An old database (user_version 0) gets rewritten on open. + const legacyDir = join(root, "shorts-legacy"); + mkdirSync(legacyDir, { recursive: true }); + new ScrapeStore(legacyDir).close(); + + const raw = new DatabaseSync(join(legacyDir, "index.db")); + const insert = raw.prepare( + "INSERT INTO media (kind, source, url, play_url, provider, downloaded, preview_hash, created_at) VALUES ('video', 'embed', ?, ?, 'YouTube', 0, ?, ?)" + ); + insert.run( + "https://www.youtube.com/shorts/legacyId123", + "https://www.youtube.com/shorts/legacyId123", + null, + "2026-09-17T00:00:00.000Z" + ); + // The same video already indexed under its watch URL: the shorts row folds in. + insert.run( + "https://www.youtube.com/watch?v=legacyId456", + null, + null, + "2026-09-17T00:00:00.000Z" + ); + insert.run( + "https://www.youtube.com/shorts/legacyId456", + "https://www.youtube.com/shorts/legacyId456", + "previewhash", + "2026-09-17T00:00:00.000Z" + ); + raw.exec("PRAGMA user_version = 0"); + raw.close(); + + const reopened = new ScrapeStore(legacyDir); + const rows = reopened.queryMedia({ kind: "video", limit: 50 }); + assert.equal(rows.length, 2); + const legacyRow = rows.find(row => row.url.includes("legacyId123")); + assert.ok(legacyRow); + assert.equal(legacyRow.play_url, "https://www.youtube.com/watch?v=legacyId123"); + const merged = rows.find(row => row.url.includes("legacyId456")); + assert.ok(merged); + assert.equal(merged.preview_hash, "previewhash"); + reopened.close(); + }); + await test("pendingTitles selects YouTube rows without a title", () => { const dir = join(root, "pending-titles"); mkdirSync(dir, { recursive: true }); diff --git a/poc/public/media.js b/poc/public/media.js index bce03d2..a4cf228 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -2238,7 +2238,8 @@ const host = url.hostname.replace(/^www\./, ""); if (host === "youtube.com" || host === "m.youtube.com") { - const id = url.searchParams.get("v"); + const id = + url.searchParams.get("v") ?? /^\/shorts\/([A-Za-z0-9_-]+)/.exec(url.pathname)?.[1]; if (id) return `https://www.youtube.com/embed/${id}`; } if (host === "youtu.be") return `https://www.youtube.com/embed${url.pathname}`;