diff --git a/CURRENT.md b/CURRENT.md index ae68397..8cb388f 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,15 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C58 Titles from Discord's embed cache. Gone YouTube videos have no oEmbed answer, but + Discord keeps the embed title (and, with the rebuilt core bundle, the author/channel). The + title lane now asks the source message's embed first and only then the provider oEmbed; the + core maps `authorName` from `embed.author.name`, extraction stores it as `title_author`, and + the viewer shows a prominent title/channel heading plus "Video unavailable (removed or + private)" when a YouTube row has neither. The 783 title errors were reset and the lane + backfilled them: the reported example (message 703788394337009754) now reads "Gun Test: Firing + Empty Shell's Handheld XM556 Microgun". The channel name populates after the Vesktop client + restarts on the new bundle (the client is externally started). - [x] C57 Page mode. The media grid has an Infinite/Page flippy toggle in the bar. Page mode renders one `pageSize` slice via `offset = (page-1)*pageSize`, shows Prev / editable page number / total / Next, persists the page in localStorage, and scrolls to the next page one at diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 81a78e8..4703338 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -100,6 +100,14 @@ export interface ScrapeEngineOptions { ) => Promise; /** Delay before the single suspect retry. Default 50 s. */ suspectRetryMs?: number; + /** + * Title and author from the source message's Discord embed. Preferred over the provider + * oEmbed: Discord keeps the embed data even after a video is deleted. + */ + embedMeta?: ( + scrapeId: string, + media: { id: number; url: string; playUrl: string | null } + ) => Promise<{ title: string; author: string | null } | null>; /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */ probe?: ( path: string @@ -605,7 +613,15 @@ export class ScrapeEngine { break; } - const resolved = await resolveEmbedTitle(row.url, options); + // Discord's cached embed data survives a deleted video; ask it first. + const fromDiscord = await this.options.embedMeta?.(meta.id, { + id: row.id, + url: row.url, + playUrl: row.play_url + }); + const resolved = + fromDiscord ?? + (await resolveEmbedTitle(row.url, options)); if (resolved) { store.markTitle(row.id, resolved.title, resolved.author); result.titled++; diff --git a/media-scrape/src/extract.ts b/media-scrape/src/extract.ts index c11ed16..f9a5099 100644 --- a/media-scrape/src/extract.ts +++ b/media-scrape/src/extract.ts @@ -34,6 +34,8 @@ export interface ExtractedMedia { previewUrl: string | null; /** Embed title, when Discord provides one (YouTube and friends). Null elsewhere. */ title: string | null; + /** Embed author (the YouTube channel Discord shows). Null elsewhere. */ + titleAuthor: string | null; } /** @@ -95,7 +97,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { width: embed.width ?? null, height: embed.height ?? null, durationSecs: null, - title: embed.title ?? null + title: embed.title ?? null, + titleAuthor: embed.authorName ?? null }; if (embed.videoUrl) { @@ -150,7 +153,8 @@ function extractSticker(sticker: CoreSticker): ExtractedMedia | null { durationSecs: null, downloaded: false, previewUrl: null, - title: null + title: null, + titleAuthor: null }; } @@ -178,7 +182,8 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] { : null, downloaded: false, previewUrl: null, - title: null + title: null, + titleAuthor: null }); } diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index f1901cb..546715e 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -404,7 +404,7 @@ export class ScrapeStore { "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)" ); const insertMedia = this.db.prepare( - "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, play_url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, title, title_key, title_author, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ); const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?"); const linkSource = this.db.prepare( @@ -443,6 +443,7 @@ export class ScrapeStore { item.previewUrl, item.title, item.title ? normalizeTitleKey(item.title) : null, + item.titleAuthor, createdAt ); if (result.changes > 0) added[item.kind]++; diff --git a/media-scrape/src/types.ts b/media-scrape/src/types.ts index 34c37ac..c98f5c2 100644 --- a/media-scrape/src/types.ts +++ b/media-scrape/src/types.ts @@ -35,6 +35,8 @@ export interface CoreEmbed { type: string | null; url: string | null; title: string | null; + /** Discord's embed author line (the YouTube channel name). */ + authorName?: string | null; provider: string | null; videoUrl: string | null; imageUrl: string | null; diff --git a/poc/public/media.css b/poc/public/media.css index 0a71e82..f9b7ac5 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -885,3 +885,9 @@ font: 400 11px/1.5 ui-monospace, Menlo, monospace; color: var(--color-neutral-500); } + +.vk-media-viewer-title { + font: 600 13px/1.5 ui-monospace, Menlo, monospace; + color: var(--color-text); + margin-bottom: 6px; +} diff --git a/poc/public/media.js b/poc/public/media.js index 65d385f..75b1dee 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -2669,6 +2669,18 @@ } meta.replaceChildren(); + + // The clip title matters most for finding footage; show it big when we have one. + const heading = + item.title || (item.titleError && (item.provider ?? "").toLowerCase().includes("youtube")) + ? item.title + ? `${item.title}${item.titleAuthor ? ` — ${item.titleAuthor}` : ""}` + : "Video unavailable (removed or private)" + : ""; + if (heading) { + meta.append(el("div", "vk-media-viewer-title", heading)); + } + const values = metaValues(item); for (const [key, label] of META_FIELDS) { if (!values[key]) continue; diff --git a/poc/src/media.ts b/poc/src/media.ts index e32e255..87141bf 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -136,6 +136,8 @@ export interface MediaItemResult { remotePreviewUrl: string | null; /** Why no preview exists, when a fetch failed (for example "http 404"). */ previewError: string | null; + /** Why no title was resolved (provider oEmbed and Discord's embed both failed). */ + titleError: string | null; /** Local animated GIF for hover playback, when the original was cached. */ animUrl: string | null; localUrl: string | null; @@ -217,6 +219,7 @@ export class MediaService { maxRetries: tunables.scrape.maxRetries, previewMaxAttempts: tunables.backfill.maxAttempts, suspectRetryMs: 50_000, + embedMeta: async (scrapeId, media) => this.embedMetaFor(scrapeId, media), refreshUrl: async (scrapeId, media) => { const meta = this.engine.get(scrapeId); if (!meta) return null; @@ -683,6 +686,67 @@ export class MediaService { this.engine.stop(id); } + /** + * Title and channel from the source message's Discord embed. Discord keeps the embed data + * after a video is deleted, so this is the only title source for gone YouTube videos. + */ + private async embedMetaFor( + scrapeId: string, + media: { id: number; url: string; playUrl: string | null } + ): Promise<{ title: string; author: string | null } | null> { + const meta = this.engine.get(scrapeId); + if (!meta) return null; + + try { + const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); + let messageId: string | null = null; + try { + messageId = store.sourceMessageId(media.id); + } finally { + store.close(); + } + if (!messageId) return null; + + const message = await this.fetchSourceMessage(meta, messageId); + if (!message) return null; + + const path = (url: string) => url.split("?")[0]; + const embeds: any[] = message.embeds ?? []; + const embed = embeds.find( + candidate => + (candidate?.url && path(candidate.url) === path(media.url)) || + (media.playUrl && candidate?.videoUrl && path(candidate.videoUrl) === path(media.playUrl)) || + (candidate?.video?.url && media.playUrl && path(candidate.video.url) === path(media.playUrl)) + ); + + const title = embed?.title ?? null; + if (!title) return null; + const author = embed?.authorName ?? embed?.author?.name ?? null; + return { title, author }; + } catch (error) { + this.deps.log?.(`embed meta failed for ${scrapeId}:${media.id}: ${String(error)}`); + return null; + } + } + + /** The source message through slopbot REST (guild scrapes) or the puppet bridge. */ + private async fetchSourceMessage(meta: ScrapeMeta, messageId: string): Promise { + if (this.bot && meta.guildId && meta.backend === "slopbot") { + const channel = await this.bot.channels.fetch(meta.channelId); + if (channel?.isTextBased()) return await channel.messages.fetch(messageId); + return null; + } + + const core = this.deps.getCore(); + if (!core || !this.deps.isCoreConnected()) return null; + + const result = (await core.fetchMessages(meta.channelId, 1, { + before: (BigInt(messageId) + 1n).toString() + })) as { messages?: Array> }; + + return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null; + } + /** Continue a paused scrape from its cursor with the same backend family. */ async resumeScrape(scrapeId: string): Promise { const meta = this.engine.get(scrapeId); @@ -1224,6 +1288,7 @@ export class MediaService { playUrl: row.play_url, remotePreviewUrl: row.preview_url, previewError: row.preview_error, + titleError: row.title_error, animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null, localUrl: row.file_path && !meta.downloadDir diff --git a/src/renderer/puppet/actions.ts b/src/renderer/puppet/actions.ts index 7929a4d..f0b24c9 100644 --- a/src/renderer/puppet/actions.ts +++ b/src/renderer/puppet/actions.ts @@ -229,6 +229,7 @@ function mapEmbeds(message: any) { type: embed?.type ?? null, url: embed?.url ?? null, title: embed?.title ?? null, + authorName: embed?.author?.name ?? null, provider: embed?.provider?.name ?? null, videoUrl: embedUrl(embed?.video), imageUrl: embedUrl(embed?.image),