diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 6921cc3..3e61ca7 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -138,7 +138,11 @@ Previews run after every walk: `pendingDownloads`. - Embeds without a thumbnail in Discord's data (fxtwitter, vxReddit, ...) are resolved first: provider oEmbed/fxtwitter API, then the page's `og:image`/`twitter:image` (512 KiB read cap, - 10 s timeout, one attempt). + 10 s timeout, one attempt). The page read never consumes media bytes: a non-HTML content type + stops it. An embed that unfurls a pasted Discord CDN attachment link resolves through the Discord + proxy instead, so it gets the same still (and video frame) as a plain attachment. Every + thumbnail-less embed enters preview passes; one with no resolvable image is closed as + `no preview source`. - Animated GIF previews are stored as a static first frame (`ffmpeg`, 480px WebP); the original animated GIF URL stays in `preview_url` for hover playback. Tenor embeds without a thumbnail are resolved through Tenor's oEmbed. An animated GIF over the size cap is framed by ffmpeg reading diff --git a/media-scrape/src/backfill.ts b/media-scrape/src/backfill.ts index 80ffd70..0d521ea 100644 --- a/media-scrape/src/backfill.ts +++ b/media-scrape/src/backfill.ts @@ -392,7 +392,8 @@ export class BackfillQueue { job.processed += progressed + result.ratelimited; const elapsed = Math.max(1, this.now() - (job.startedAt ?? this.now())); - job.perMinute = job.processed / (elapsed / 60_000); + // Early in a run the measured rate is noise; report null until it settles. + job.perMinute = elapsed >= 30_000 ? job.processed / (elapsed / 60_000) : null; const after = this.engine.rateLimitStatus(); if (after.active) this.limitedUntil = after.until ?? this.now() + after.remainingMs; diff --git a/media-scrape/src/embed-preview.ts b/media-scrape/src/embed-preview.ts index d9b406e..8f21e11 100644 --- a/media-scrape/src/embed-preview.ts +++ b/media-scrape/src/embed-preview.ts @@ -34,6 +34,11 @@ async function getText(url: string, options: DownloadOptions): Promise { assert.equal(calls.some(url => url.includes("youtube.com/watch")), false); }); + await test("pendingPreviews includes embeds without a stored thumbnail", () => { + const dir = join(root, "pending-embeds"); + mkdirSync(dir, { recursive: true }); + const store = new ScrapeStore(dir); + store.insertPage([ + message("670", { + embeds: [ + { + type: "video", + url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4", + title: null, + provider: "cdn.discordapp.com", + videoUrl: "https://cdn.discordapp.com/attachments/1/2/clip.mp4", + imageUrl: null, + thumbnailUrl: null + } + ] + }) + ]); + + assert.equal(store.pendingPreviews(10, false).length, 0); + assert.equal(store.pendingPreviews(10, true).length, 1); + store.close(); + }); + + await test("CDN attachment embeds resolve through the Discord proxy", async () => { + const resolved = await resolveEmbedPreview({ + url: "https://cdn.discordapp.com/attachments/1/2/clip.mp4", + source: "embed", + preview_url: null + }); + + assert.ok(resolved); + const parsed = new URL(String(resolved)); + assert.equal(parsed.hostname, "media.discordapp.net"); + assert.equal(parsed.searchParams.get("width"), "320"); + assert.equal(parsed.searchParams.get("format"), "webp"); + }); + + await test("the og:image fallback never reads media bytes", async () => { + const seen: string[] = []; + const resolved = await resolveEmbedPreview( + { + url: "https://example.com/clip.mp4", + source: "embed", + preview_url: null + }, + { + fetchImpl: async (input: any) => { + seen.push(String(input)); + return new Response("binary", { + status: 200, + headers: { "content-type": "video/mp4" } + }); + } + } + ); + + assert.equal(resolved, null); + assert.equal(seen.length, 1); + }); + await test("engine caches previews in links mode, including embed thumbnails", async () => { const backend = new FakeBackend([ {