diff --git a/CURRENT.md b/CURRENT.md index fb22612..eaca4cb 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -37,6 +37,9 @@ mobile draft at 390px. embeds excluded in SQL; local files served at `/api/media///file` and preferred by the Media tab. Live run 2026-09-16: scrape `...-20260916083401` fetched the 784 KB attachment; storage counted actual bytes. + - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB + cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: + `previewed: 2`; thumbs served over `/api/media///thumb` (200). - [~] C9 live verification. Slopbot path verified live on 2026-09-16: login Slopbot#5775, guild `177931418704281600`. Test channel links scrape: 17 messages, 4 images + 13 videos, 0 errors; download scrape fetched the 784 KB attachment. Large channel `gernal_chut`: 11 pages, 1088 @@ -44,8 +47,8 @@ mobile draft at 390px. verified live on a group DM (`zıbb & fred`) and a 1:1 DM: backend `puppet`, done, 0 errors. The puppet field check is done too: the group DM returned attachments plus mapped embeds (YouTube, Instagram), and the large slopbot channel produced YouTube, Gfycat, Imgur, Steam, - and Twitch embeds alongside attachments. Still owed: preview caching (C11) and a stress walk - on the second guild (`559346566754729995`, channel `559346566754729997`). + and Twitch embeds alongside attachments. Still owed: a stress walk on the second guild + (`559346566754729995`, channel `559346566754729997`). - Flextalk voice profile, end to end. The profile window is a movable panel that recenters on each open; it carries the character sliders with engine defaults and +/-2 snapping, five named presets, radio-kit effects (high-pass 40-400 Hz, echo 0-100, word mode with a gap in ms), and diff --git a/docs/daemon.md b/docs/daemon.md index fc1939a..6e7677b 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -174,6 +174,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_AUDIO_CLIENT_MATCH` | `vesktop,electron,chromium,discord` | Stream name patterns moved into the capture sink. | | `VESKSLOP_MEDIA_DB_DIR` | `/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. | | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. | +| `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | | `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index efcfdfe..7969ae3 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -101,8 +101,14 @@ Downloads run after a finished walk when the scrape used `mode: "download"`: (`source != 'embed'`). - Progress counters: `downloaded`, `downloadErrors`. -Previews are **planned (C11)**: the Discord CDN resizer (`?width=320&height=320&format=webp`) into -`thumbs/`, stored by hash, failure never fails the scrape. +Previews run after every walk, in both modes: + +- Only Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) are + transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`). + Third-party embeds are skipped. +- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape. +- Stored by hash in `thumbs/.webp`; `preview_hash` and `preview_bytes` land on the row, and + the counter is `previewed`. Previews count toward the storage limit. ## Job states diff --git a/docs/media.md b/docs/media.md index 7724b6b..d6863da 100644 --- a/docs/media.md +++ b/docs/media.md @@ -8,8 +8,8 @@ the byte downloader are implemented (C3-C8, C10). The Media and Bot tabs are C6. - slopbot on a larger channel: 11 pages, 1088 messages (96 images, 1 GIF, 203 videos), 0 errors. - puppet on a group DM and a 1:1 DM: done, 0 errors, backend `puppet`. -Remaining: preview caching (C11) and a 429 observation on a very large walk. Checkpoints live in -[CURRENT.md](../CURRENT.md). +Preview caching is implemented and live-verified (2 WebP thumbs served over the thumb route). +Remaining: a 429 observation on a very large walk. Checkpoints live in [CURRENT.md](../CURRENT.md). The media archive saves the media history of a channel. A trusted user runs `scrape media` in the channel. The daemon walks the history through a scrape backend and indexes every image, GIF, @@ -80,8 +80,9 @@ media-scrape/db/ - `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows are never download targets. The Media tab prefers the local file when one exists, so the archive survives CDN URL expiry. -- Cached image previews are still planned (C11). Until then, the Media tab hotlinks Discord CDN URLs - in `links` mode. +- Discord-hosted image and GIF previews are cached small (WebP, 320px) in `thumbs/` after every + walk, in both modes, so the grid does not depend on expiring CDN links. Third-party embed + thumbnails are never fetched. - Third-party embeds are **never downloaded**: YouTube, Instagram, X, TikTok, and every other embed stays a link with metadata. - Discord-hosted image and sticker previews are cached small (`thumbs/`, WebP, via the CDN diff --git a/docs/webui.md b/docs/webui.md index 2751528..a39e02f 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -251,6 +251,7 @@ replaceable. | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | | GET | `/api/media` | `kind?`, `limit?`, `offset?` | Media items across every scrape, newest first; `localUrl` when a file was downloaded | | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | +| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview | | GET | `/api/media/scrapes` | none | Every scrape with state and counters | | GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over | diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index 3e9a8d2..18e2477 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -17,6 +17,8 @@ export interface DownloadOptions { maxGapMs?: number; /** Retries after the first attempt, for network and 5xx errors only. */ maxRetries?: number; + /** Preview size cap. Default 5 MiB. */ + previewMaxBytes?: number; sleep?: (ms: number) => Promise; random?: () => number; log?: (line: string) => void; @@ -120,6 +122,64 @@ export async function downloadOne( return { id: media.id, state: "failed", error: lastError }; } +const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]); + +/** + * A small WebP preview URL for a Discord-hosted image or GIF. + * Third-party embeds return null: they are never fetched. + */ +export function previewUrlFor(media: { url: string; kind: string }): string | null { + if (media.kind !== "image" && media.kind !== "gif") return null; + + try { + const url = new URL(media.url); + if (!DISCORD_HOSTS.has(url.hostname)) return null; + + url.hostname = "media.discordapp.net"; + url.searchParams.set("width", "320"); + url.searchParams.set("height", "320"); + url.searchParams.set("format", "webp"); + return url.toString(); + } catch { + return null; + } +} + +/** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */ +export async function downloadPreview( + media: Pick, + dir: string, + options: DownloadOptions = {} +): Promise { + const target = previewUrlFor(media); + if (!target) return { id: media.id, state: "skipped", error: "no preview url" }; + + const fetchImpl = options.fetchImpl ?? fetch; + const maxBytes = options.previewMaxBytes ?? 5 * 1024 * 1024; + + try { + const response = await fetchImpl(target); + if (!response.ok) return { id: media.id, state: "skipped", error: `http ${response.status}` }; + + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > maxBytes) { + return { id: media.id, state: "skipped", error: "too large" }; + } + + const hash = createHash("sha256").update(buffer).digest("hex"); + const file = `${hash}.webp`; + writeFileSync(join(dir, file), buffer); + + return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength }; + } catch (error) { + return { + id: media.id, + state: "failed", + error: error instanceof Error ? error.message : String(error) + }; + } +} + /** The gap before the next file, in milliseconds. */ export function downloadGapMs(options: DownloadOptions = {}): number { const min = options.minGapMs ?? 1000; diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 642da76..8736c23 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -4,7 +4,7 @@ */ import { join } from "node:path"; -import { downloadGapMs, downloadOne, type DownloadOptions } from "./download.js"; +import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js"; import { createScrapeDirs, listScrapes, @@ -48,6 +48,8 @@ export interface ScrapeEngineOptions { log?: (line: string) => void; /** Download settings for `mode: "download"`. */ download?: DownloadOptions; + /** Cache Discord-hosted image previews after every walk. Default true. */ + previews?: boolean; } interface QueuedJob { @@ -221,6 +223,10 @@ export class ScrapeEngine { if (meta.state === "done" && meta.mode === "download") { await this.runDownloads(store, dir, meta); } + + if (meta.state === "done" && this.options.previews !== false) { + await this.runPreviews(store, dir, meta); + } } catch (error) { meta.state = "failed"; meta.lastError = error instanceof Error ? error.message : String(error); @@ -265,6 +271,31 @@ export class ScrapeEngine { } } + /** Cache small previews for Discord-hosted images. Runs in both modes. */ + private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { + const options = this.options.download ?? {}; + const pending = store.pendingPreviews(); + + for (const row of pending) { + if (this.stopRequested.has(meta.id)) { + meta.state = "paused"; + return; + } + + const outcome = await downloadPreview(row, join(dir, "thumbs"), options); + + if (outcome.state === "downloaded" && outcome.hash) { + store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 }); + meta.previewed = (meta.previewed ?? 0) + 1; + } else { + store.markPreviewError(row.id, outcome.error ?? outcome.state); + } + + this.persist(dir, meta); + await this.sleep(downloadGapMs(options)); + } + } + private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise { const pageSize = this.options.pageSize ?? 100; const maxRetries = this.options.maxRetries ?? 8; diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index 70121c0..8763636 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -5,7 +5,14 @@ export { extractMedia, type ExtractedMedia } from "./extract.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; -export { downloadGapMs, downloadOne, type DownloadOptions, type DownloadOutcome } from "./download.js"; +export { + downloadGapMs, + downloadOne, + downloadPreview, + previewUrlFor, + type DownloadOptions, + type DownloadOutcome +} from "./download.js"; export { ScrapeEngine, type FetchPage, diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 5208a04..4fcab84 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -114,6 +114,7 @@ export interface MediaRow { preview_bytes: number | null; file_path: string | null; download_error: string | null; + preview_error: string | null; created_at: string; } @@ -145,6 +146,7 @@ CREATE TABLE IF NOT EXISTS media ( preview_bytes INTEGER, file_path TEXT, download_error TEXT, + preview_error TEXT, created_at TEXT NOT NULL ); @@ -178,6 +180,7 @@ export class ScrapeStore { if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT"); if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT"); if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); + if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT"); } close(): void { @@ -291,6 +294,25 @@ export class ScrapeStore { this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id); } + /** Image and GIF rows that still need a cached preview. */ + pendingPreviews(limit = 5000): MediaRow[] { + return this.db + .prepare( + "SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?" + ) + .all(limit) as unknown as MediaRow[]; + } + + markPreview(id: number, preview: { hash: string; bytes: number }): void { + this.db + .prepare("UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_error = NULL WHERE id = ?") + .run(preview.hash, preview.bytes, id); + } + + markPreviewError(id: number, error: string): void { + this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id); + } + getMedia(id: number): MediaRow | null { const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as | MediaRow diff --git a/media-scrape/src/types.ts b/media-scrape/src/types.ts index b660a5c..45bb9bc 100644 --- a/media-scrape/src/types.ts +++ b/media-scrape/src/types.ts @@ -87,6 +87,8 @@ export interface ScrapeMeta { downloaded?: number; /** Files that failed after retries; skipped URLs are not counted here. */ downloadErrors?: number; + /** Cached image previews. */ + previewed?: number; } export function emptyMediaCounts(): Record { diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 05c0b9d..2347678 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -8,7 +8,7 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { downloadOne } from "../src/download.js"; +import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; @@ -85,6 +85,7 @@ function engineFor(root: string, backend: ScrapeBackend, extra: Record {}, random: () => 0, log: () => {}, + previews: false, ...extra }); } @@ -366,6 +367,7 @@ void (async () => { sleep: async () => {}, random: () => 0, log: () => {}, + previews: false, download: { fetchImpl: async () => new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }), @@ -394,6 +396,77 @@ void (async () => { assert.equal(existsSync(join(root, started.id, row.file_path)), true); }); + await test("preview urls are Discord-only resizes", () => { + assert.equal( + previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }), + "https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp" + ); + assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null); + assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null); + }); + + await test("preview downloads are written by hash", async () => { + const dir = join(root, "preview-test"); + mkdirSync(dir, { recursive: true }); + + const outcome = await downloadPreview( + { id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }, + dir, + { + fetchImpl: async () => + new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }) + } + ); + + assert.equal(outcome.state, "downloaded"); + assert.equal(outcome.path, `${outcome.hash}.webp`); + assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true); + }); + + await test("engine caches previews in links mode", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("600", { + attachments: [attachment({ filename: "prev.png", contentType: "image/png" })] + }) + ] + } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 2, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: true, + download: { + fetchImpl: async () => + new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }), + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "128", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + const meta = engine.get(started.id); + assert.equal(meta?.state, "done"); + assert.equal(meta?.previewed, 1); + + const store = new ScrapeStore(join(root, started.id)); + const row = store.queryMedia({ kind: "image" })[0]; + assert.ok(row.preview_hash); + assert.ok((row.preview_bytes ?? 0) > 0); + store.close(); + assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true); + }); + await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, diff --git a/poc/public/media.js b/poc/public/media.js index ad1fa6b..22e3fa1 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -198,7 +198,7 @@ if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) { const img = document.createElement("img"); img.loading = "lazy"; - img.src = source; + img.src = item.thumbUrl || source; img.alt = item.kind; card.append(img); } else { diff --git a/poc/src/media.ts b/poc/src/media.ts index a9e0f41..7c751aa 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -94,6 +94,7 @@ export interface MediaItemResult { kind: MediaKind; url: string; localUrl: string | null; + thumbUrl: string | null; provider: string | null; downloaded: boolean; createdAt: string; @@ -141,6 +142,7 @@ export class MediaService { log: line => this.deps.log?.(line), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), + previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024), minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000), maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000) } @@ -323,6 +325,17 @@ export class MediaService { }; } + /** Absolute path for a cached preview, or null. */ + getMediaThumb(scrapeId: string, itemId: number): { path: string; contentType: string } | null { + const item = this.getMediaItem(scrapeId, itemId); + if (!item?.preview_hash) return null; + + return { + path: join(this.rootDir, scrapeId, "thumbs", `${item.preview_hash}.webp`), + contentType: "image/webp" + }; + } + /** One media row inside a scrape, with its scrape and channel for context. */ getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { const meta = this.engine.get(scrapeId); @@ -363,6 +376,7 @@ export class MediaService { kind: row.kind, url: row.url, localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null, + thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, provider: row.provider, downloaded: row.downloaded === 1, createdAt: row.created_at diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 6df046e..d41beae 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -754,6 +754,15 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/media/:scrapeId/:itemId/thumb", + wrap(async (req, res) => { + const file = deps.media.getMediaThumb(String(req.params.scrapeId), Number(req.params.itemId)); + if (!file) throw new HttpError(404, "thumb not found"); + res.type(file.contentType).sendFile(file.path); + }) + ); + app.get( "/api/media/:scrapeId/:itemId/file", wrap(async (req, res) => {