zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize YouTube Shorts links to watch URLs

Browse Source
This commit is contained in:
Fisto
2026-09-17 13:34:10 +02:00
parent 52049bf186
commit 8c72def863
6 changed files with 197 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CURRENT.md
+8
View File
@@ -40,6 +40,14 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C51 YouTube Shorts normalization. Shorts links broke the embedded player and View full
because the viewer needs `watch?v=`. Extraction now rewrites `youtube.com/shorts/<id>` (and
`m.`) to the watch form, the viewer also accepts `/shorts/<id>` defensively, and a one-time
`PRAGMA user_version` migration rewrites existing rows. The live archive had 45 shorts rows;
one collided with an already indexed watch URL and folded into it (preview fields merged,
message links moved) — the migration is idempotent and unit-tested, including the collision.
45 → 0 shorts rows, and the user's example `uAf18d7mrnU` is now
`watch?v=uAf18d7mrnU` with `embed/uAf18d7mrnU` as the play URL.
- [x] C50 Title backfill in the queue. YouTube-like rows (provider or URL matching
youtube/youtu.be) without a title are now a second work type in the backfill queue: resolved
through the provider oEmbed, stored as `title`/`title_key`, with `title_attempts` and
docs/media-scraper.md
+4 -1
View File
@@ -96,7 +96,10 @@ One SQLite file per scrape (`index.db`, `node:sqlite`).
slopbot backend could ask for it (one request; null otherwise). Indexes: `kind`, `posted_at`, and
`media_sources(message_id)`. The database is an index; the media
folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time
(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Embed rows also store
(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). `youtube.com/shorts/<id>`
links are stored as `watch?v=<id>` (extraction and a one-time `user_version` migration), because the
embedded player and View full need the watch form; a shorts row that collides with an already
indexed watch URL folds into it (preview/file fields merge, message links move over). Embed rows also store
Discord's `title` (YouTube and friends) plus `title_key`, the lowercased, punctuation-collapsed
form used by the title search: multi-word prefix tokens by default ("Ma Ca Ju" finds *Man who jumps
over car*), one phrase with the exact-match toggle, and rows without a title drop out while a
media-scrape/src/extract.ts
+15 -4
View File
@@ -36,6 +36,15 @@ export interface ExtractedMedia {
title: string | null;
}
/**
* Shorts links break the embed player: it needs `watch?v=`. Rewrite them for storage so the
* viewer, "View full", and refreshes all use the canonical form. Other URLs pass through.
*/
export function normalizeYouTubeUrl(url: string): string {
const match = /^https?:\/\/(?:www\.|m\.)?youtube\.com\/shorts\/([A-Za-z0-9_-]{6,})/.exec(url);
return match ? `https://www.youtube.com/watch?v=${match[1]}` : url;
}
function extension(filename: string | null | undefined): string {
const match = /\.([a-z0-9]+)$/i.exec(filename ?? "");
return match ? match[1].toLowerCase() : "";
@@ -75,9 +84,11 @@ function hostname(url: string | null): string | null {
function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
const provider = embed.provider ?? hostname(embed.url);
const embedUrl = embed.url ? normalizeYouTubeUrl(embed.url) : null;
const videoUrl = embed.videoUrl ? normalizeYouTubeUrl(embed.videoUrl) : null;
const common = {
source: "embed" as const,
playUrl: embed.videoUrl ?? null,
playUrl: videoUrl,
provider,
downloaded: false,
size: null,
@@ -91,7 +102,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
return {
...common,
kind: "video",
url: embed.url ?? embed.videoUrl,
url: embedUrl ?? videoUrl ?? "",
contentType: null,
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
};
@@ -102,7 +113,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
return {
...common,
kind: "gif",
url: embed.url ?? embed.imageUrl ?? "",
url: embedUrl ?? embed.imageUrl ?? "",
contentType: null,
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
};
@@ -112,7 +123,7 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
return {
...common,
kind: "image",
url: embed.url ?? embed.imageUrl,
url: embedUrl ?? embed.imageUrl,
contentType: null,
previewUrl: embed.thumbnailUrl ?? embed.imageUrl
};
media-scrape/src/store.ts
+81 -1
View File
@@ -7,7 +7,7 @@ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "n
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { fileURLToPath } from "node:url";
import { extractMedia } from "./extract.js";
import { extractMedia, normalizeYouTubeUrl } from "./extract.js";
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
@@ -281,6 +281,86 @@ export class ScrapeStore {
this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0");
this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
}
this.migrateYouTubeShorts();
}
/**
* One-time rewrite of youtube.com/shorts links to watch URLs. The embed player needs `v=`,
* so old rows with a shorts URL would never play. `PRAGMA user_version` makes it idempotent
* per scrape database.
*/
private migrateYouTubeShorts(): void {
const version = this.db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined;
if ((version?.user_version ?? 0) >= 1) return;
const rows = this.db
.prepare(
"SELECT * FROM media WHERE url LIKE '%youtube.com/shorts/%' OR play_url LIKE '%youtube.com/shorts/%'"
)
.all() as unknown as MediaRow[];
if (rows.length) {
const update = this.db.prepare("UPDATE media SET url = ?, play_url = ? WHERE id = ?");
const findTarget = this.db.prepare("SELECT id FROM media WHERE url = ? AND id != ?");
const merge = this.db.prepare(
`UPDATE media SET
preview_hash = COALESCE(preview_hash, ?),
preview_bytes = COALESCE(preview_bytes, ?),
preview_ext = COALESCE(preview_ext, ?),
file_path = COALESCE(file_path, ?),
file_hash = COALESCE(file_hash, ?),
file_bytes = COALESCE(file_bytes, ?),
downloaded = MAX(downloaded, ?),
title = COALESCE(title, ?),
title_key = COALESCE(title_key, ?),
play_url = COALESCE(play_url, ?)
WHERE id = ?`
);
const relink = this.db.prepare(
"UPDATE OR IGNORE media_sources SET media_id = ? WHERE media_id = ?"
);
const dropLinks = this.db.prepare("DELETE FROM media_sources WHERE media_id = ?");
const drop = this.db.prepare("DELETE FROM media WHERE id = ?");
this.db.exec("BEGIN");
try {
for (const row of rows) {
const url = normalizeYouTubeUrl(row.url);
const playUrl = row.play_url ? normalizeYouTubeUrl(row.play_url) : null;
const target = findTarget.get(url, row.id) as { id: number } | undefined;
// The same video was already indexed under its watch URL: fold this row in.
if (!target) {
update.run(url, playUrl, row.id);
continue;
}
merge.run(
row.preview_hash,
row.preview_bytes,
row.preview_ext,
row.file_path,
row.file_hash,
row.file_bytes,
row.downloaded,
row.title,
row.title_key,
playUrl,
target.id
);
relink.run(target.id, row.id);
dropLinks.run(row.id);
drop.run(row.id);
}
this.db.exec("COMMIT");
} catch (error) {
this.db.exec("ROLLBACK");
throw error;
}
}
this.db.exec("PRAGMA user_version = 1");
}
close(): void {
media-scrape/test/run.ts
+87 -1
View File
@@ -5,6 +5,7 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { DatabaseSync } from "node:sqlite";
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -20,7 +21,7 @@ import {
import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
import { BackfillQueue } from "../src/backfill.js";
import { extractMedia } from "../src/extract.js";
import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js";
import { betterCandidate, sortCandidates } from "../src/dedupe.js";
import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
@@ -1348,6 +1349,91 @@ void (async () => {
store.close();
});
await test("youtube shorts links normalize to watch URLs", () => {
assert.equal(
normalizeYouTubeUrl("https://www.youtube.com/shorts/uAf18d7mrnU"),
"https://www.youtube.com/watch?v=uAf18d7mrnU"
);
assert.equal(
normalizeYouTubeUrl("https://m.youtube.com/shorts/abc123XYZ?feature=share"),
"https://www.youtube.com/watch?v=abc123XYZ"
);
assert.equal(
normalizeYouTubeUrl("https://www.youtube.com/watch?v=keep"),
"https://www.youtube.com/watch?v=keep"
);
assert.equal(normalizeYouTubeUrl("https://youtu.be/short"), "https://youtu.be/short");
});
await test("shorts store normalized and old rows migrate", () => {
const dir = join(root, "shorts");
mkdirSync(dir, { recursive: true });
const store = new ScrapeStore(dir);
store.insertPage([
message("1200", {
embeds: [
{
type: "video",
url: "https://www.youtube.com/shorts/uAf18d7mrnU",
title: "A short",
provider: "YouTube",
videoUrl: "https://www.youtube.com/shorts/uAf18d7mrnU",
imageUrl: null,
thumbnailUrl: null
}
]
})
]);
const row = store.queryMedia({ kind: "video" })[0];
assert.equal(row.url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
assert.equal(row.play_url, "https://www.youtube.com/watch?v=uAf18d7mrnU");
store.close();
// An old database (user_version 0) gets rewritten on open.
const legacyDir = join(root, "shorts-legacy");
mkdirSync(legacyDir, { recursive: true });
new ScrapeStore(legacyDir).close();
const raw = new DatabaseSync(join(legacyDir, "index.db"));
const insert = raw.prepare(
"INSERT INTO media (kind, source, url, play_url, provider, downloaded, preview_hash, created_at) VALUES ('video', 'embed', ?, ?, 'YouTube', 0, ?, ?)"
);
insert.run(
"https://www.youtube.com/shorts/legacyId123",
"https://www.youtube.com/shorts/legacyId123",
null,
"2026-09-17T00:00:00.000Z"
);
// The same video already indexed under its watch URL: the shorts row folds in.
insert.run(
"https://www.youtube.com/watch?v=legacyId456",
null,
null,
"2026-09-17T00:00:00.000Z"
);
insert.run(
"https://www.youtube.com/shorts/legacyId456",
"https://www.youtube.com/shorts/legacyId456",
"previewhash",
"2026-09-17T00:00:00.000Z"
);
raw.exec("PRAGMA user_version = 0");
raw.close();
const reopened = new ScrapeStore(legacyDir);
const rows = reopened.queryMedia({ kind: "video", limit: 50 });
assert.equal(rows.length, 2);
const legacyRow = rows.find(row => row.url.includes("legacyId123"));
assert.ok(legacyRow);
assert.equal(legacyRow.play_url, "https://www.youtube.com/watch?v=legacyId123");
const merged = rows.find(row => row.url.includes("legacyId456"));
assert.ok(merged);
assert.equal(merged.preview_hash, "previewhash");
reopened.close();
});
await test("pendingTitles selects YouTube rows without a title", () => {
const dir = join(root, "pending-titles");
mkdirSync(dir, { recursive: true });
poc/public/media.js
+2 -1
View File
@@ -2238,7 +2238,8 @@
const host = url.hostname.replace(/^www\./, "");
if (host === "youtube.com" || host === "m.youtube.com") {
const id = url.searchParams.get("v");
const id =
url.searchParams.get("v") ?? /^\/shorts\/([A-Za-z0-9_-]+)/.exec(url.pathname)?.[1];
if (id) return `https://www.youtube.com/embed/${id}`;
}
if (host === "youtu.be") return `https://www.youtube.com/embed${url.pathname}`;