From 8f30ec14addd10d3ea0a63d9ce1de1dc09971203 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 23:48:17 +0200 Subject: [PATCH] Cache audio waveforms and rescue oversized GIF previews --- CURRENT.md | 6 ++++++ docs/media-scraper.md | 7 ++++++- docs/media.md | 5 +++-- media-scrape/src/download.ts | 37 ++++++++++++++++++++++++++++++++++++ media-scrape/src/engine.ts | 35 ++++++++++++++++++++++++++++++++++ media-scrape/src/store.ts | 2 +- media-scrape/test/run.ts | 36 ++++++++++++++++++++++++++++++++++- 7 files changed, 123 insertions(+), 5 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index d4cac01..a0caab8 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,12 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C30 Preview completeness. Audio attachments and voice messages get an ffmpeg waveform + strip; oversized animated GIFs fall back to an ffmpeg-read first frame instead of `too large`; + transient network failures stay pending for the next backfill instead of becoming permanent. + Live: a scrape with 58 audio items backfilled 13 waveforms in 40 s, serving `image/webp`. + Remaining non-cached: Lottie stickers/custom emoji (not indexed as media) and embeds whose + pages refuse bots (marked PRIVATE/GONE by their 403/404). - [x] C29 Attachment video previews. Discord's image proxy renders a frame for video files (`?width=320&height=320&format=webp`), so `previewUrlFor` now resizes attachment mp4/webm in links mode too and `pendingPreviews` includes non-embed videos. Live: paused-scrape videos diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 6bbb7e7..2b88633 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -130,7 +130,12 @@ Previews run after every walk: 10 s timeout, one attempt). - Animated GIF previews are stored as a static first frame (`ffmpeg`, 480px WebP); the original animated GIF URL stays in `preview_url` for hover playback. Tenor embeds without a thumbnail are - resolved through Tenor's oEmbed. + resolved through Tenor's oEmbed. An animated GIF over the size cap is framed by ffmpeg reading + the stream directly, so it is not lost to `too large`. +- Audio attachments and voice messages get a waveform strip + (`ffmpeg -filter_complex showwavespic`) as their preview, rendered by ffmpeg from the URL. +- Transient network failures are left pending for the next backfill pass; only skips (4xx, too + large, no source) are marked permanent. - One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape. - Stored by hash in `thumbs/.`, where the extension follows the response type (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the diff --git a/docs/media.md b/docs/media.md index ca02ca4..5f5a41e 100644 --- a/docs/media.md +++ b/docs/media.md @@ -91,8 +91,9 @@ media-scrape/db/ - Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every other embed stays a link. Its thumbnail is cached (below); the video itself is never fetched. - Previews are cached in `thumbs/` after every walk, in both modes: Discord-hosted images, GIFs, - and video files (the proxy renders a frame, resized to 320px WebP) plus third-party embed - thumbnails (YouTube, X, TikTok, and so on). The + and video files (the proxy renders a frame, resized to 320px WebP), third-party embed thumbnails + (YouTube, X, TikTok, and so on), and audio, which gets a waveform strip so audio tiles are not + blank. The stored bytes keep their real type (`preview_ext`), so JPEG provider thumbnails are served as `image/jpeg`, not mislabeled WebP. Embeds without a thumbnail in Discord's data are resolved through the provider's oEmbed/fxtwitter API and finally the page's `og:image`. Previews count diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index c2e21a4..448f48e 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -319,6 +319,43 @@ export async function videoPoster( } } +/** + * A poster frame fetched by ffmpeg itself, without downloading the whole file through us. + * Handles oversized animated GIFs (first frame only) and audio (waveform strip). + */ +export async function remoteFrame( + url: string, + outDir: string, + options: { waveform?: boolean; ffmpeg?: string } = {} +): Promise<{ hash: string; bytes: number } | null> { + const ffmpeg = options.ffmpeg ?? process.env.VESKSLOP_FFMPEG ?? "ffmpeg"; + const temp = join(outDir, `.remote-${Date.now()}-${Math.random().toString(16).slice(2)}.webp`); + const filters = options.waveform + ? ["-filter_complex", "showwavespic=s=480x120:colors=white"] + : ["-frames:v", "1", "-vf", "scale=480:-1"]; + + try { + await execFileAsync(ffmpeg, ["-v", "error", "-y", "-i", url, ...filters, temp], { + timeout: 60_000 + }); + + const buffer = readFileSync(temp); + if (!buffer.length) return null; + + const hash = createHash("sha256").update(buffer).digest("hex"); + renameSync(temp, join(outDir, `${hash}.webp`)); + return { hash, bytes: buffer.length }; + } catch { + return null; + } finally { + try { + rmSync(temp, { force: true }); + } catch { + // already renamed + } + } +} + /** Static first-frame WebP from an animated GIF. Null when ffmpeg is missing or fails. */ export async function gifPoster( inputPath: string, diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index f8dc839..723a48b 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -12,6 +12,7 @@ import { gifPoster, previewGapMs, probeDimensions, + remoteFrame, videoPoster, type DownloadOptions } from "./download.js"; @@ -387,6 +388,24 @@ export class ScrapeEngine { return; } + // Audio has no image proxy: render a waveform strip with ffmpeg. + if (row.kind === "audio" && row.source !== "embed") { + const waveform = await remoteFrame(row.url, join(dir, "thumbs"), { waveform: true }); + if (waveform) { + store.markPreview(row.id, { + hash: waveform.hash, + bytes: waveform.bytes, + ext: "webp" + }); + meta.previewed = (meta.previewed ?? 0) + 1; + } else { + store.markPreviewError(row.id, "waveform failed"); + } + this.persist(dir, meta); + await this.sleep(previewGapMs(options)); + continue; + } + // Some providers (fxtwitter, vxReddit, ...) ship no thumbnail in Discord's data; // resolve one from the provider or the page before caching it. let previewRow = row; @@ -429,6 +448,22 @@ export class ScrapeEngine { mkdirSync(dirname(target), { recursive: true }); copyFileSync(join(dir, "thumbs", `${hash}.${ext}`), target); } + } else if ( + outcome.state === "skipped" && + outcome.error === "too large" && + /\.gif(\?|$)/i.test(previewRow.preview_url ?? previewRow.url) + ) { + // Oversized animated GIF: let ffmpeg read the stream and keep one frame. + const frame = await remoteFrame(previewRow.preview_url ?? previewRow.url, join(dir, "thumbs")); + if (frame) { + store.markPreview(row.id, { hash: frame.hash, bytes: frame.bytes, ext: "webp" }); + meta.previewed = (meta.previewed ?? 0) + 1; + } else { + store.markPreviewError(row.id, "too large"); + } + } else if (outcome.state === "failed") { + // Transient network failure: leave the row pending for the next pass. + this.log(`${meta.id}: preview ${row.id} failed: ${outcome.error ?? "unknown"}`); } else { store.markPreviewError(row.id, outcome.error ?? outcome.state); } diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 119e462..210c67a 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -343,7 +343,7 @@ export class ScrapeStore { */ pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] { const local = - "((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video'))"; + "((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))"; const condition = includeEmbeds ? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))` : local; diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 4a6f429..2a3d99a 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -9,7 +9,14 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { downloadOne, downloadPreview, gifPoster, previewUrlFor, videoPoster } from "../src/download.js"; +import { + downloadOne, + downloadPreview, + gifPoster, + previewUrlFor, + remoteFrame, + videoPoster +} from "../src/download.js"; import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; @@ -954,6 +961,33 @@ void (async () => { ); }); + await test("remoteFrame renders an audio waveform", async () => { + const dir = join(root, "waveform-test"); + mkdirSync(dir, { recursive: true }); + const wavPath = join(dir, "tone.wav"); + + try { + execFileSync("ffmpeg", [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=1", + wavPath + ]); + } catch { + console.log("ok - remoteFrame skipped (no ffmpeg)"); + return; + } + + const frame = await remoteFrame(wavPath, dir, { waveform: true }); + assert.ok(frame?.hash); + assert.ok((frame?.bytes ?? 0) > 0); + assert.equal(existsSync(join(dir, `${frame?.hash}.webp`)), true); + }); + await test("snowflake helpers round-trip dates", () => { const id = snowflakeFromIso("2026-09-16T00:00:00.000Z"); assert.ok(id);