zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture YouTube titles from Discord's embed cache

Browse Source
This commit is contained in:
Fisto
2026-09-20 19:50:34 +02:00
parent f22ab5ea5f
commit cc3d62336c
9 changed files with 122 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CURRENT.md
+9
View File
@@ -40,6 +40,15 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C58 Titles from Discord's embed cache. Gone YouTube videos have no oEmbed answer, but
Discord keeps the embed title (and, with the rebuilt core bundle, the author/channel). The
title lane now asks the source message's embed first and only then the provider oEmbed; the
core maps `authorName` from `embed.author.name`, extraction stores it as `title_author`, and
the viewer shows a prominent title/channel heading plus "Video unavailable (removed or
private)" when a YouTube row has neither. The 783 title errors were reset and the lane
backfilled them: the reported example (message 703788394337009754) now reads "Gun Test: Firing
Empty Shell's Handheld XM556 Microgun". The channel name populates after the Vesktop client
restarts on the new bundle (the client is externally started).
- [x] C57 Page mode. The media grid has an Infinite/Page flippy toggle in the bar. Page mode
renders one `pageSize` slice via `offset = (page-1)*pageSize`, shows Prev / editable page
number / total / Next, persists the page in localStorage, and scrolls to the next page one at
media-scrape/src/engine.ts
+17 -1
View File
@@ -100,6 +100,14 @@ export interface ScrapeEngineOptions {
) => Promise<string | null>;
/** Delay before the single suspect retry. Default 50 s. */
suspectRetryMs?: number;
/**
* Title and author from the source message's Discord embed. Preferred over the provider
* oEmbed: Discord keeps the embed data even after a video is deleted.
*/
embedMeta?: (
scrapeId: string,
media: { id: number; url: string; playUrl: string | null }
) => Promise<{ title: string; author: string | null } | null>;
/** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
probe?: (
path: string
@@ -605,7 +613,15 @@ export class ScrapeEngine {
break;
}
const resolved = await resolveEmbedTitle(row.url, options);
// Discord's cached embed data survives a deleted video; ask it first.
const fromDiscord = await this.options.embedMeta?.(meta.id, {
id: row.id,
url: row.url,
playUrl: row.play_url
});
const resolved =
fromDiscord ??
(await resolveEmbedTitle(row.url, options));
if (resolved) {
store.markTitle(row.id, resolved.title, resolved.author);
result.titled++;
media-scrape/src/extract.ts
+8 -3
View File
@@ -34,6 +34,8 @@ export interface ExtractedMedia {
previewUrl: string | null;
/** Embed title, when Discord provides one (YouTube and friends). Null elsewhere. */
title: string | null;
/** Embed author (the YouTube channel Discord shows). Null elsewhere. */
titleAuthor: string | null;
}
/**
@@ -95,7 +97,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
width: embed.width ?? null,
height: embed.height ?? null,
durationSecs: null,
title: embed.title ?? null
title: embed.title ?? null,
titleAuthor: embed.authorName ?? null
};
if (embed.videoUrl) {
@@ -150,7 +153,8 @@ function extractSticker(sticker: CoreSticker): ExtractedMedia | null {
durationSecs: null,
downloaded: false,
previewUrl: null,
title: null
title: null,
titleAuthor: null
};
}
@@ -178,7 +182,8 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] {
: null,
downloaded: false,
previewUrl: null,
title: null
title: null,
titleAuthor: null
});
}
media-scrape/src/store.ts
+2 -1
View File
@@ -404,7 +404,7 @@ export class ScrapeStore {
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
);
const insertMedia = this.db.prepare(
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, title_author, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
const linkSource = this.db.prepare(
@@ -443,6 +443,7 @@ export class ScrapeStore {
item.previewUrl,
item.title,
item.title ? normalizeTitleKey(item.title) : null,
item.titleAuthor,
createdAt
);
if (result.changes > 0) added[item.kind]++;
media-scrape/src/types.ts
+2
View File
@@ -35,6 +35,8 @@ export interface CoreEmbed {
type: string | null;
url: string | null;
title: string | null;
/** Discord's embed author line (the YouTube channel name). */
authorName?: string | null;
provider: string | null;
videoUrl: string | null;
imageUrl: string | null;
poc/public/media.css
+6
View File
@@ -885,3 +885,9 @@
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-500);
}
.vk-media-viewer-title {
font: 600 13px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-text);
margin-bottom: 6px;
}
poc/public/media.js
+12
View File
@@ -2669,6 +2669,18 @@
}
meta.replaceChildren();
// The clip title matters most for finding footage; show it big when we have one.
const heading =
item.title || (item.titleError && (item.provider ?? "").toLowerCase().includes("youtube"))
? item.title
? `${item.title}${item.titleAuthor ? ` — ${item.titleAuthor}` : ""}`
: "Video unavailable (removed or private)"
: "";
if (heading) {
meta.append(el("div", "vk-media-viewer-title", heading));
}
const values = metaValues(item);
for (const [key, label] of META_FIELDS) {
if (!values[key]) continue;
poc/src/media.ts
+65
View File
@@ -136,6 +136,8 @@ export interface MediaItemResult {
remotePreviewUrl: string | null;
/** Why no preview exists, when a fetch failed (for example "http 404"). */
previewError: string | null;
/** Why no title was resolved (provider oEmbed and Discord's embed both failed). */
titleError: string | null;
/** Local animated GIF for hover playback, when the original was cached. */
animUrl: string | null;
localUrl: string | null;
@@ -217,6 +219,7 @@ export class MediaService {
maxRetries: tunables.scrape.maxRetries,
previewMaxAttempts: tunables.backfill.maxAttempts,
suspectRetryMs: 50_000,
embedMeta: async (scrapeId, media) => this.embedMetaFor(scrapeId, media),
refreshUrl: async (scrapeId, media) => {
const meta = this.engine.get(scrapeId);
if (!meta) return null;
@@ -683,6 +686,67 @@ export class MediaService {
this.engine.stop(id);
}
/**
* Title and channel from the source message's Discord embed. Discord keeps the embed data
* after a video is deleted, so this is the only title source for gone YouTube videos.
*/
private async embedMetaFor(
scrapeId: string,
media: { id: number; url: string; playUrl: string | null }
): Promise<{ title: string; author: string | null } | null> {
const meta = this.engine.get(scrapeId);
if (!meta) return null;
try {
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
let messageId: string | null = null;
try {
messageId = store.sourceMessageId(media.id);
} finally {
store.close();
}
if (!messageId) return null;
const message = await this.fetchSourceMessage(meta, messageId);
if (!message) return null;
const path = (url: string) => url.split("?")[0];
const embeds: any[] = message.embeds ?? [];
const embed = embeds.find(
candidate =>
(candidate?.url && path(candidate.url) === path(media.url)) ||
(media.playUrl && candidate?.videoUrl && path(candidate.videoUrl) === path(media.playUrl)) ||
(candidate?.video?.url && media.playUrl && path(candidate.video.url) === path(media.playUrl))
);
const title = embed?.title ?? null;
if (!title) return null;
const author = embed?.authorName ?? embed?.author?.name ?? null;
return { title, author };
} catch (error) {
this.deps.log?.(`embed meta failed for ${scrapeId}:${media.id}: ${String(error)}`);
return null;
}
}
/** The source message through slopbot REST (guild scrapes) or the puppet bridge. */
private async fetchSourceMessage(meta: ScrapeMeta, messageId: string): Promise<any | null> {
if (this.bot && meta.guildId && meta.backend === "slopbot") {
const channel = await this.bot.channels.fetch(meta.channelId);
if (channel?.isTextBased()) return await channel.messages.fetch(messageId);
return null;
}
const core = this.deps.getCore();
if (!core || !this.deps.isCoreConnected()) return null;
const result = (await core.fetchMessages(meta.channelId, 1, {
before: (BigInt(messageId) + 1n).toString()
})) as { messages?: Array<Record<string, any>> };
return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null;
}
/** Continue a paused scrape from its cursor with the same backend family. */
async resumeScrape(scrapeId: string): Promise<ScrapeMeta | null> {
const meta = this.engine.get(scrapeId);
@@ -1224,6 +1288,7 @@ export class MediaService {
playUrl: row.play_url,
remotePreviewUrl: row.preview_url,
previewError: row.preview_error,
titleError: row.title_error,
animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null,
localUrl:
row.file_path && !meta.downloadDir
src/renderer/puppet/actions.ts
+1
View File
@@ -229,6 +229,7 @@ function mapEmbeds(message: any) {
type: embed?.type ?? null,
url: embed?.url ?? null,
title: embed?.title ?? null,
authorName: embed?.author?.name ?? null,
provider: embed?.provider?.name ?? null,
videoUrl: embedUrl(embed?.video),
imageUrl: embedUrl(embed?.image),