diff --git a/CURRENT.md b/CURRENT.md index b465f58..ead6d94 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,11 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL; + the full third-party media is never fetched. Live run `...-20260916091458` on the test + channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also + fixed the state transition: the job stays `running` through download and preview passes so + clients do not see `done` early. - [x] C12 Media tab filters and full-window layout: kind chips, source filter (attachment/sticker/embed), source-channel filter, and the view now fills the window below the header and call strip. API/MCP gained `source` and `channelId` on `query_media`. Smoke-tested diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 7969ae3..061e661 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -101,14 +101,18 @@ Downloads run after a finished walk when the scrape used `mode: "download"`: (`source != 'embed'`). - Progress counters: `downloaded`, `downloadErrors`. -Previews run after every walk, in both modes: +Previews run after every walk: -- Only Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) are - transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`). - Third-party embeds are skipped. +- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) + are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`). +- In `download` mode only, embed rows also get their stored `preview_url` (YouTube, X, TikTok + thumbnails, and so on) cached. The embed's full media URL is never fetched; embed rows are + excluded from `pendingDownloads`. - One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape. - Stored by hash in `thumbs/.webp`; `preview_hash` and `preview_bytes` land on the row, and the counter is `previewed`. Previews count toward the storage limit. +- The job stays `running` through the download and preview passes, so a client never sees `done` + on a half-archived scrape. ## Job states diff --git a/docs/media.md b/docs/media.md index d6057f8..73e427c 100644 --- a/docs/media.md +++ b/docs/media.md @@ -80,14 +80,12 @@ media-scrape/db/ - `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows are never download targets. The Media tab prefers the local file when one exists, so the archive survives CDN URL expiry. -- Discord-hosted image and GIF previews are cached small (WebP, 320px) in `thumbs/` after every - walk, in both modes, so the grid does not depend on expiring CDN links. Third-party embed - thumbnails are never fetched. -- Third-party embeds are **never downloaded**: YouTube, Instagram, X, TikTok, and every other - embed stays a link with metadata. -- Discord-hosted image and sticker previews are cached small (`thumbs/`, WebP, via the CDN - resizer) in both modes, so the Media grid does not depend on expiring CDN links. Previews count - toward the storage limit. +- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every + other embed stays a link. In `download` mode their thumbnail is cached (below); the video itself + is never fetched. +- Previews are cached small (WebP, 320px) in `thumbs/` after every walk. In both modes this covers + Discord-hosted images and GIFs; in `download` mode it also covers embed thumbnails, so the grid + does not depend on expiring CDN links. Previews count toward the storage limit. - Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry. diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index 18e2477..ca52cc7 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -19,6 +19,8 @@ export interface DownloadOptions { maxRetries?: number; /** Preview size cap. Default 5 MiB. */ previewMaxBytes?: number; + /** Cache third-party embed thumbnails. Download mode only. */ + embedPreviews?: boolean; sleep?: (ms: number) => Promise; random?: () => number; log?: (line: string) => void; @@ -125,20 +127,30 @@ export async function downloadOne( const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]); /** - * A small WebP preview URL for a Discord-hosted image or GIF. - * Third-party embeds return null: they are never fetched. + * The preview target for a media row. + * + * Attachments and stickers: their Discord-hosted URL, resized to a small WebP. + * Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized). + * Returns null when the row has no preview source. */ -export function previewUrlFor(media: { url: string; kind: string }): string | null { - if (media.kind !== "image" && media.kind !== "gif") return null; +export function previewUrlFor(media: { + url: string; + kind: string; + previewUrl?: string | null; +}): string | null { + const candidate = + media.previewUrl ?? + (media.kind === "image" || media.kind === "gif" ? media.url : null); + if (!candidate) return null; try { - const url = new URL(media.url); - if (!DISCORD_HOSTS.has(url.hostname)) return null; - - url.hostname = "media.discordapp.net"; - url.searchParams.set("width", "320"); - url.searchParams.set("height", "320"); - url.searchParams.set("format", "webp"); + const url = new URL(candidate); + if (DISCORD_HOSTS.has(url.hostname)) { + url.hostname = "media.discordapp.net"; + url.searchParams.set("width", "320"); + url.searchParams.set("height", "320"); + url.searchParams.set("format", "webp"); + } return url.toString(); } catch { return null; @@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu /** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */ export async function downloadPreview( - media: Pick, + media: Pick, dir: string, options: DownloadOptions = {} ): Promise { - const target = previewUrlFor(media); + // Third-party thumbnails only in download mode; the full embed is never fetched. + if (media.source === "embed" && options.embedPreviews !== true) { + return { id: media.id, state: "skipped", error: "embed previews off" }; + } + + const target = previewUrlFor({ + url: media.url, + kind: media.kind, + previewUrl: media.preview_url + }); if (!target) return { id: media.id, state: "skipped", error: "no preview url" }; const fetchImpl = options.fetchImpl ?? fetch; diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 8736c23..700ef02 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -220,12 +220,24 @@ export class ScrapeEngine { await this.sleep(this.gapMs()); } - if (meta.state === "done" && meta.mode === "download") { - await this.runDownloads(store, dir, meta); - } + if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) { + // Keep the job running until post-processing finishes, so a client never + // treats a half-archived scrape as complete. + meta.state = "running"; + this.persist(dir, meta); - if (meta.state === "done" && this.options.previews !== false) { - await this.runPreviews(store, dir, meta); + if (meta.mode === "download") { + await this.runDownloads(store, dir, meta); + } + + if (this.options.previews !== false && !this.stopRequested.has(meta.id)) { + await this.runPreviews(store, dir, meta); + } + + if (meta.state === "running") { + meta.state = "done"; + } + this.persist(dir, meta); } } catch (error) { meta.state = "failed"; @@ -271,10 +283,11 @@ export class ScrapeEngine { } } - /** Cache small previews for Discord-hosted images. Runs in both modes. */ + /** Cache small previews. In download mode this includes embed thumbnails. */ private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { - const options = this.options.download ?? {}; - const pending = store.pendingPreviews(); + const embedPreviews = meta.mode === "download"; + const options = { ...(this.options.download ?? {}), embedPreviews }; + const pending = store.pendingPreviews(5000, embedPreviews); for (const row of pending) { if (this.stopRequested.has(meta.id)) { diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 4b902ff..70208e1 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -294,11 +294,19 @@ export class ScrapeStore { this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id); } - /** Image and GIF rows that still need a cached preview. */ - pendingPreviews(limit = 5000): MediaRow[] { + /** + * Rows that still need a cached preview. Attachment/sticker images and GIFs always; + * embed thumbnails only when `includeEmbeds` is set (download mode). + */ + pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] { + const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))"; + const condition = includeEmbeds + ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))` + : local; + return this.db .prepare( - "SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?" + `SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?` ) .all(limit) as unknown as MediaRow[]; } diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index b39c55c..6d265ef 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -351,7 +351,7 @@ void (async () => { provider: "YouTube", videoUrl: "https://www.youtube.com/embed/x", imageUrl: null, - thumbnailUrl: null + thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" } ] }) @@ -359,6 +359,7 @@ void (async () => { } ]); + const fetched: string[] = []; const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, @@ -367,10 +368,15 @@ void (async () => { sleep: async () => {}, random: () => 0, log: () => {}, - previews: false, + previews: true, download: { - fetchImpl: async () => - new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }), + fetchImpl: async (input: any) => { + fetched.push(String(input)); + return new Response("png-bytes", { + status: 200, + headers: { "content-type": "image/png" } + }); + }, minGapMs: 0, maxGapMs: 0, sleep: async () => {} @@ -383,6 +389,9 @@ void (async () => { const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.downloaded, 1); + assert.equal(meta?.previewed, 2); + assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false); + assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true); const store = new ScrapeStore(join(root, started.id)); const row = store.queryMedia({ kind: "image" })[0]; @@ -426,13 +435,20 @@ void (async () => { store.close(); }); - await test("preview urls are Discord-only resizes", () => { + await test("preview urls resize Discord and pass embed thumbnails through", () => { assert.equal( previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }), "https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp" ); assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null); - assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null); + assert.equal( + previewUrlFor({ + url: "https://www.youtube.com/watch?v=x", + kind: "video", + previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" + }), + "https://i.ytimg.com/vi/x/hqdefault.jpg" + ); }); await test("preview downloads are written by hash", async () => { @@ -440,7 +456,13 @@ void (async () => { mkdirSync(dir, { recursive: true }); const outcome = await downloadPreview( - { id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }, + { + id: 9, + url: "https://cdn.discordapp.com/attachments/1/2/pic.png", + kind: "image", + source: "attachment", + preview_url: null + }, dir, { fetchImpl: async () => @@ -453,17 +475,57 @@ void (async () => { assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true); }); - await test("engine caches previews in links mode", async () => { + await test("embed thumbnails download only when enabled", async () => { + const dir = join(root, "preview-embed"); + mkdirSync(dir, { recursive: true }); + + const media = { + id: 11, + url: "https://www.youtube.com/watch?v=x", + kind: "video" as const, + source: "embed", + preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg" + }; + const calls: string[] = []; + const fetchImpl = async (input: any) => { + calls.push(String(input)); + return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } }); + }; + + const off = await downloadPreview(media, dir, { fetchImpl }); + assert.equal(off.state, "skipped"); + assert.equal(calls.length, 0); + + const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true }); + assert.equal(on.state, "downloaded"); + assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg"); + assert.equal(calls.some(url => url.includes("youtube.com/watch")), false); + }); + + await test("engine caches previews in links mode and skips embeds", async () => { const backend = new FakeBackend([ { messages: [ message("600", { - attachments: [attachment({ filename: "prev.png", contentType: "image/png" })] + attachments: [attachment({ filename: "prev.png", contentType: "image/png" })], + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=x", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/embed/x", + imageUrl: null, + thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" + } + ] }) ] } ]); + const fetched: string[] = []; + const engine = new ScrapeEngine({ rootDir: root, pageSize: 2, @@ -474,8 +536,10 @@ void (async () => { log: () => {}, previews: true, download: { - fetchImpl: async () => - new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }), + fetchImpl: async (input: any) => { + fetched.push(String(input)); + return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }); + }, minGapMs: 0, maxGapMs: 0, sleep: async () => {} @@ -488,6 +552,7 @@ void (async () => { const meta = engine.get(started.id); assert.equal(meta?.state, "done"); assert.equal(meta?.previewed, 1); + assert.equal(fetched.some(url => url.includes("ytimg.com")), false); const store = new ScrapeStore(join(root, started.id)); const row = store.queryMedia({ kind: "image" })[0];