diff --git a/CURRENT.md b/CURRENT.md index 44cd158..a674238 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -33,13 +33,15 @@ mobile draft at 390px. - [x] C7a storage limits, retention, warning/confirm flow, storage and clean API - [x] C7b Settings group: limit field with info hover, usage bar, clean modal - [x] C8 MCP service hooks (`scrape_media`, scrape reads, storage, clean; reads in `run_commands`) - - [ ] C10 downloader: fetch attachment bytes in `mode: "download"`, hash-dedupe, size cap, - serial queue with 1-3 s gaps, retries. The command accepts the mode today but only indexes. + - [x] C10 downloader: attachment and sticker bytes, hash-addressed, serial, size-capped, retries; + embeds excluded in SQL; local files served at `/api/media///file` and preferred + by the Media tab. Live run 2026-09-16: scrape `...-20260916083401` fetched the 784 KB + attachment; storage counted actual bytes. - [~] C9 live verification. Slopbot path verified live on 2026-09-16: login Slopbot#5775, guild - `177931418704281600`, test channel `1549681689222053988`. Scrape `1549681689222053988- - 20260916075150`: state done, 1 page, 17 messages, 4 images + 13 videos, 0 errors, ~20 MB of - indexed source sizes. Still owed: the puppet `message.fetch` field check, a `download` mode - run, a real 429 observation, and pacing tuning. + `177931418704281600`, test channel `1549681689222053988`. Links scrape `...-20260916075150`: + done, 1 page, 17 messages, 4 images + 13 videos, 0 errors. Download scrape `...-20260916083401`: + done, 1 file. Still owed: preview caching (C11), the puppet `message.fetch` field check, a real + 429 observation, and pacing tuning. - Flextalk voice profile, end to end. The profile window is a movable panel that recenters on each open; it carries the character sliders with engine defaults and +/-2 snapping, five named presets, radio-kit effects (high-pass 40-400 Hz, echo 0-100, word mode with a gap in ms), and diff --git a/docs/daemon.md b/docs/daemon.md index 8fffe3c..1d053cb 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -172,6 +172,9 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_AUDIO_MANAGE_SERVER` | `1` | Start `pulseaudio` when `pactl` cannot reach a server. | | `VESKSLOP_AUDIO_SINK` / `_SOURCE` / `_CAPTURE_SINK` | `veskslop_tts` / `veskslop_mic` / `veskslop_capture` | Virtual device names. | | `VESKSLOP_AUDIO_CLIENT_MATCH` | `vesktop,electron,chromium,discord` | Stream name patterns moved into the capture sink. | +| `VESKSLOP_MEDIA_DB_DIR` | `/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. | +| `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. | +| `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | The daemon warns at startup when `dist/` is older than the core sources. Rebuild with `pnpm build` and restart the client, or new actions fail with `UNKNOWN_ACTION`. diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 7943cb2..efcfdfe 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -90,16 +90,19 @@ Indexes: `(kind, timestamp)` via `media_sources`, `channel` metadata in `scrape. ## Downloads and previews -**Planned (C10).** The schema has `file_bytes`, `preview_bytes`, `file_hash`, and `preview_hash`, -and the command accepts `mode: "download"`, but nothing fetches bytes yet. +Downloads run after a finished walk when the scrape used `mode: "download"`: -The plan: +- A serial queue, one file at a time, 1-3 s apart, with a size cap + (`VESKSLOP_MEDIA_MAX_FILE_BYTES`, default 50 MiB) and 2 retries on network or 5xx errors. +- `4xx` from the skip set (400, 401, 403, 404, 410) marks the URL expired; the row is not retried. +- The file name is the SHA-256 of the bytes under `media/`. `file_path`, `file_hash`, and + `file_bytes` are written to the row, so the run resumes and dedupes by content. +- Only attachment and sticker rows are targets. Embed rows are excluded in SQL + (`source != 'embed'`). +- Progress counters: `downloaded`, `downloadErrors`. -- A serial download queue, one file at a time, 1-3 s apart, with a size cap - (`media.maxFileBytes`, default 50 MiB) and 2 retries on network or 5xx errors. -- `4xx` marks the URL expired. The item is re-fetched with its message later. -- Previews use the Discord CDN resizer (`?width=320&height=320&format=webp`) and are stored by hash - in `thumbs/`. A preview failure never fails the scrape. +Previews are **planned (C11)**: the Discord CDN resizer (`?width=320&height=320&format=webp`) into +`thumbs/`, stored by hash, failure never fails the scrape. ## Job states diff --git a/docs/media.md b/docs/media.md index 9139c60..acde8d5 100644 --- a/docs/media.md +++ b/docs/media.md @@ -1,10 +1,12 @@ # Media archive Status: design approved 2026-09-16. The engine, the slopbot backend, the `scrape media` command, -the `/api/media/*` routes, storage limits with retention, the Settings group, and the MCP tools are -implemented (C3-C8). The Media and Bot tabs are C6. The slopbot path is live-verified: the test -channel produced 17 messages and 4 images + 13 videos with no errors. The puppet path and a -`download` run are still owed (C9). Checkpoints live in [CURRENT.md](../CURRENT.md). +the `/api/media/*` routes, storage limits with retention, the Settings group, the MCP tools, and +the byte downloader are implemented (C3-C8, C10). The Media and Bot tabs are C6. Live-verified on +the test channel: links mode produced 17 messages (4 images, 13 videos) with no errors, and +download mode fetched the attachment (784 KB) and served it locally. Remaining: preview caching +(C11), the puppet `message.fetch` field check, and a 429 observation. Checkpoints live in +[CURRENT.md](../CURRENT.md). The media archive saves the media history of a channel. A trusted user runs `scrape media` in the channel. The daemon walks the history through a scrape backend and indexes every image, GIF, @@ -72,9 +74,11 @@ media-scrape/db/ - `links` is the default. The archive stores the Discord CDN URL and the metadata. Nothing large is downloaded. -- `download` fetches attachment bytes into `media/`. Deduplicated by hash. **Not implemented yet** - (checkpoint C10): the command accepts the mode and the schema has the columns, but the download - queue is still owed. +- `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows + are never download targets. The Media tab prefers the local file when one exists, so the archive + survives CDN URL expiry. +- Cached image previews are still planned (C11). Until then, the Media tab hotlinks Discord CDN URLs + in `links` mode. - Third-party embeds are **never downloaded**: YouTube, Instagram, X, TikTok, and every other embed stays a link with metadata. - Discord-hosted image and sticker previews are cached small (`thumbs/`, WebP, via the CDN diff --git a/docs/webui.md b/docs/webui.md index 3f18854..a2fe2a7 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -249,7 +249,8 @@ replaceable. | POST | `/api/audio/stop` | none | Stop speech and clear the queue | | POST | `/api/audio/capture` | `start` | Start or stop capture | | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | -| GET | `/api/media` | `kind?`, `limit?`, `offset?` | Media items across every scrape, newest first | +| GET | `/api/media` | `kind?`, `limit?`, `offset?` | Media items across every scrape, newest first; `localUrl` when a file was downloaded | +| GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | | GET | `/api/media/scrapes` | none | Every scrape with state and counters | | GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over | diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts new file mode 100644 index 0000000..3e9a8d2 --- /dev/null +++ b/media-scrape/src/download.ts @@ -0,0 +1,129 @@ +/* + * Attachment downloader for the media archive. Serial, hash-addressed, conservative. + * Rules: docs/media-scraper.md. Third-party embeds are never targets here. + */ + +import { createHash } from "node:crypto"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { MediaRow } from "./store.js"; + +export interface DownloadOptions { + fetchImpl?: typeof fetch; + /** Skip files larger than this. Default 50 MiB. */ + maxFileBytes?: number; + /** Gap between files. Default 1000-3000 ms. */ + minGapMs?: number; + maxGapMs?: number; + /** Retries after the first attempt, for network and 5xx errors only. */ + maxRetries?: number; + sleep?: (ms: number) => Promise; + random?: () => number; + log?: (line: string) => void; +} + +export interface DownloadOutcome { + id: number; + state: "downloaded" | "skipped" | "failed"; + path?: string; + hash?: string; + bytes?: number; + error?: string; +} + +/** 4xx that will not improve on retry: the signed URL expired or the file is gone. */ +const SKIP_STATUS = new Set([400, 401, 403, 404, 410]); + +const TYPE_EXTENSIONS: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/webp": "webp", + "image/avif": "avif", + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + "video/x-matroska": "mkv", + "audio/mpeg": "mp3", + "audio/ogg": "ogg", + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mp4": "m4a", + "audio/flac": "flac" +}; + +function extensionFor(contentType: string | null, url: string): string { + const type = (contentType ?? "").split(";")[0].trim().toLowerCase(); + if (TYPE_EXTENSIONS[type]) return TYPE_EXTENSIONS[type]; + + try { + const match = /\.([a-z0-9]{2,5})$/i.exec(new URL(url).pathname); + if (match) return match[1].toLowerCase(); + } catch { + // fall through + } + return "bin"; +} + +const defaultSleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Download one media row into `dir`. The file name is the SHA-256 of the bytes. */ +export async function downloadOne( + media: Pick, + dir: string, + options: DownloadOptions = {} +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const maxBytes = options.maxFileBytes ?? 50 * 1024 * 1024; + const maxRetries = options.maxRetries ?? 2; + const sleep = options.sleep ?? defaultSleep; + + let lastError = "download failed"; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const response = await fetchImpl(media.url); + + if (!response.ok) { + if (SKIP_STATUS.has(response.status)) { + return { id: media.id, state: "skipped", error: `http ${response.status}` }; + } + lastError = `http ${response.status}`; + } else { + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > maxBytes) { + return { id: media.id, state: "skipped", error: `too large (${declared} bytes)` }; + } + + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > maxBytes) { + return { id: media.id, state: "skipped", error: "too large" }; + } + + const hash = createHash("sha256").update(buffer).digest("hex"); + const extension = extensionFor(response.headers.get("content-type"), media.url); + const file = `${hash}.${extension}`; + writeFileSync(join(dir, file), buffer); + + return { id: media.id, state: "downloaded", path: join("media", file), hash, bytes: buffer.byteLength }; + } + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + + if (attempt < maxRetries) { + options.log?.(`download ${media.id} failed (${lastError}), retrying`); + await sleep(2 ** attempt * 1000 + 1000); + } + } + + return { id: media.id, state: "failed", error: lastError }; +} + +/** The gap before the next file, in milliseconds. */ +export function downloadGapMs(options: DownloadOptions = {}): number { + const min = options.minGapMs ?? 1000; + const max = Math.max(min, options.maxGapMs ?? 3000); + const random = options.random ?? Math.random; + return min + Math.floor(random() * (max - min + 1)); +} diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index cfd18ca..642da76 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -4,6 +4,7 @@ */ import { join } from "node:path"; +import { downloadGapMs, downloadOne, type DownloadOptions } from "./download.js"; import { createScrapeDirs, listScrapes, @@ -45,6 +46,8 @@ export interface ScrapeEngineOptions { random?: () => number; now?: () => number; log?: (line: string) => void; + /** Download settings for `mode: "download"`. */ + download?: DownloadOptions; } interface QueuedJob { @@ -214,6 +217,10 @@ export class ScrapeEngine { await this.sleep(this.gapMs()); } + + if (meta.state === "done" && meta.mode === "download") { + await this.runDownloads(store, dir, meta); + } } catch (error) { meta.state = "failed"; meta.lastError = error instanceof Error ? error.message : String(error); @@ -225,6 +232,39 @@ export class ScrapeEngine { } } + /** Fetch attachment bytes for a finished walk. Serial, hash-addressed, resumable. */ + private async runDownloads(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { + const options = this.options.download ?? {}; + const pending = store.pendingDownloads(); + + for (const row of pending) { + if (this.stopRequested.has(meta.id)) { + meta.state = "paused"; + return; + } + + const outcome = await downloadOne(row, join(dir, "media"), { + ...options, + log: line => this.log(`${meta.id}: ${line}`) + }); + + if (outcome.state === "downloaded" && outcome.path && outcome.hash) { + store.markDownloaded(row.id, { + path: outcome.path, + hash: outcome.hash, + bytes: outcome.bytes ?? 0 + }); + meta.downloaded = (meta.downloaded ?? 0) + 1; + } else { + store.markDownloadError(row.id, outcome.error ?? outcome.state); + if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1; + } + + this.persist(dir, meta); + await this.sleep(downloadGapMs(options)); + } + } + private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise { const pageSize = this.options.pageSize ?? 100; const maxRetries = this.options.maxRetries ?? 8; diff --git a/media-scrape/src/extract.ts b/media-scrape/src/extract.ts index c324a59..673f6b0 100644 --- a/media-scrape/src/extract.ts +++ b/media-scrape/src/extract.ts @@ -13,10 +13,14 @@ const AUDIO_EXT = new Set(["mp3", "m4a", "ogg", "wav", "flac"]); /** Sticker formatType to file extension. 3 is Lottie JSON; not archived. */ const STICKER_EXT: Record = { 1: "png", 2: "apng", 4: "gif" }; +/** Where the item came from. Embeds are never downloaded. */ +export type MediaSource = "attachment" | "sticker" | "embed"; + /** A media item extracted from a message. `url` is the original source link. */ export interface ExtractedMedia { kind: MediaKind; url: string; + source: MediaSource; provider: string | null; contentType: string | null; size: number | null; @@ -68,6 +72,7 @@ function hostname(url: string | null): string | null { function extractEmbed(embed: CoreEmbed): ExtractedMedia | null { const provider = embed.provider ?? hostname(embed.url); const common = { + source: "embed" as const, provider, downloaded: false, size: null, @@ -117,6 +122,7 @@ function extractSticker(sticker: CoreSticker): ExtractedMedia | null { return { kind: ext === "gif" ? "gif" : "image", + source: "sticker", url: `https://cdn.discordapp.com/stickers/${sticker.id}.${ext}`, provider: "discord", contentType: null, @@ -138,6 +144,7 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] { if (!kind) continue; items.push({ kind, + source: "attachment", url: attachment.url, provider: "discord", contentType: attachment.contentType, diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index 76cf877..70121c0 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -5,6 +5,7 @@ export { extractMedia, type ExtractedMedia } from "./extract.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; +export { downloadGapMs, downloadOne, type DownloadOptions, type DownloadOutcome } from "./download.js"; export { ScrapeEngine, type FetchPage, diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 8c53d52..5208a04 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -98,6 +98,7 @@ export function uniqueScrapeId(root: string, channelId: string, now = new Date() export interface MediaRow { id: number; kind: MediaKind; + source: string | null; url: string; provider: string | null; content_type: string | null; @@ -109,6 +110,10 @@ export interface MediaRow { preview_url: string | null; file_hash: string | null; preview_hash: string | null; + file_bytes: number | null; + preview_bytes: number | null; + file_path: string | null; + download_error: string | null; created_at: string; } @@ -124,6 +129,7 @@ CREATE TABLE IF NOT EXISTS messages ( CREATE TABLE IF NOT EXISTS media ( id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, + source TEXT, url TEXT NOT NULL UNIQUE, provider TEXT, content_type TEXT, @@ -137,6 +143,8 @@ CREATE TABLE IF NOT EXISTS media ( preview_hash TEXT, file_bytes INTEGER, preview_bytes INTEGER, + file_path TEXT, + download_error TEXT, created_at TEXT NOT NULL ); @@ -167,6 +175,9 @@ export class ScrapeStore { if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER"); if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER"); + if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT"); + if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT"); + if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); } close(): void { @@ -180,7 +191,7 @@ export class ScrapeStore { "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)" ); const insertMedia = this.db.prepare( - "INSERT OR IGNORE INTO media (kind, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + "INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" ); const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?"); const linkSource = this.db.prepare( @@ -202,6 +213,7 @@ export class ScrapeStore { for (const item of extractMedia(message)) { const result = insertMedia.run( item.kind, + item.source, item.url, item.provider, item.contentType, @@ -255,6 +267,30 @@ export class ScrapeStore { return result; } + /** Media rows that still need bytes: never downloaded and not skipped. */ + pendingDownloads(limit = 5000): MediaRow[] { + return this.db + .prepare( + "SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?" + ) + .all(limit) as unknown as MediaRow[]; + } + + markDownloaded( + id: number, + file: { path: string; hash: string; bytes: number } + ): void { + this.db + .prepare( + "UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?" + ) + .run(file.path, file.hash, file.bytes, id); + } + + markDownloadError(id: number, error: string): void { + this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id); + } + getMedia(id: number): MediaRow | null { const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as | MediaRow diff --git a/media-scrape/src/types.ts b/media-scrape/src/types.ts index 4dee596..b660a5c 100644 --- a/media-scrape/src/types.ts +++ b/media-scrape/src/types.ts @@ -83,6 +83,10 @@ export interface ScrapeMeta { media: Record; errors: number; lastError: string | null; + /** Files fetched in download mode. */ + downloaded?: number; + /** Files that failed after retries; skipped URLs are not counted here. */ + downloadErrors?: number; } export function emptyMediaCounts(): Record { diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 063549a..05c0b9d 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -4,9 +4,11 @@ */ import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { downloadOne } from "../src/download.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; @@ -276,6 +278,122 @@ void (async () => { assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" }); }); + await test("download stores bytes by hash and skips expired URLs", async () => { + const dir = join(root, "download-test"); + mkdirSync(dir, { recursive: true }); + + const ok = await downloadOne( + { id: 1, url: "https://cdn.discordapp.com/attachments/1/2/note.txt", kind: "image" }, + dir, + { + fetchImpl: async () => + new Response("hello", { status: 200, headers: { "content-type": "text/plain" } }) + } + ); + assert.equal(ok.state, "downloaded"); + assert.equal(ok.hash, createHash("sha256").update("hello").digest("hex")); + assert.equal(existsSync(join(dir, `${ok.hash}.txt`)), true); + + const expired = await downloadOne( + { id: 2, url: "https://cdn.discordapp.com/attachments/1/2/gone.png", kind: "image" }, + dir, + { fetchImpl: async () => new Response("no", { status: 404 }) } + ); + assert.equal(expired.state, "skipped"); + + const large = await downloadOne( + { id: 3, url: "https://cdn.discordapp.com/attachments/1/2/big.mp4", kind: "video" }, + dir, + { + fetchImpl: async () => + new Response("x", { status: 200, headers: { "content-length": "999999999" } }) + } + ); + assert.equal(large.state, "skipped"); + }); + + await test("download retries a 5xx and then succeeds", async () => { + const dir = join(root, "download-retry"); + mkdirSync(dir, { recursive: true }); + + let calls = 0; + const outcome = await downloadOne( + { id: 4, url: "https://cdn.discordapp.com/attachments/1/2/flaky.png", kind: "image" }, + dir, + { + maxRetries: 2, + sleep: async () => {}, + fetchImpl: async () => { + calls++; + return calls === 1 + ? new Response("boom", { status: 500 }) + : new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }); + } + } + ); + + assert.equal(outcome.state, "downloaded"); + assert.equal(calls, 2); + }); + + await test("engine downloads attachments in download mode", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("500", { + attachments: [attachment({ filename: "pic.png", contentType: "image/png" })], + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=x", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/embed/x", + imageUrl: null, + thumbnailUrl: null + } + ] + }) + ] + } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 2, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + download: { + fetchImpl: async () => + new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }), + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "127", guildId: null, mode: "download", backend }); + await engine.processQueue(); + + const meta = engine.get(started.id); + assert.equal(meta?.state, "done"); + assert.equal(meta?.downloaded, 1); + + const store = new ScrapeStore(join(root, started.id)); + const row = store.queryMedia({ kind: "image" })[0]; + assert.equal(row.downloaded, 1); + assert.ok(row.file_path); + assert.ok((row.file_bytes ?? 0) > 0); + const embed = store.queryMedia({ kind: "video" })[0]; + assert.equal(embed.downloaded, 0); + assert.equal(embed.file_path, null); + store.close(); + assert.equal(existsSync(join(root, started.id, row.file_path)), true); + }); + await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, diff --git a/poc/public/media.js b/poc/public/media.js index 0975d02..2799211 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -170,15 +170,16 @@ function renderGrid() { grid.replaceChildren(); for (const item of state.items) { + const source = item.localUrl || item.url; const card = el("a", "vk-media-card"); - card.href = item.url; + card.href = source; card.target = "_blank"; card.rel = "noreferrer"; - if ((item.kind === "image" || item.kind === "gif") && /^https?:/.test(item.url)) { + if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) { const img = document.createElement("img"); img.loading = "lazy"; - img.src = item.url; + img.src = source; img.alt = item.kind; card.append(img); } else { diff --git a/poc/src/media.ts b/poc/src/media.ts index 0e1a479..a9e0f41 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -6,6 +6,7 @@ */ import { rmSync } from "node:fs"; +import { join } from "node:path"; import { dirSizeBytes, isNearLimit, @@ -88,14 +89,34 @@ function formatBytes(bytes: number): string { export interface MediaItemResult { scrapeId: string; + itemId: number; channelId: string; kind: MediaKind; url: string; + localUrl: string | null; provider: string | null; downloaded: boolean; createdAt: string; } +const FILE_CONTENT_TYPES: Record = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + avif: "image/avif", + mp4: "video/mp4", + webm: "video/webm", + mov: "video/quicktime", + mkv: "video/x-matroska", + mp3: "audio/mpeg", + ogg: "audio/ogg", + wav: "audio/wav", + m4a: "audio/mp4", + flac: "audio/flac" +}; + export interface MediaStatus { slopbot: SlopbotStatus; scrapes: ScrapeMeta[]; @@ -110,9 +131,19 @@ export class MediaService { private botStarting: Promise | null = null; constructor(private readonly deps: MediaServiceDeps) { + const number = (value: string | undefined, fallback: number) => { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; + }; + this.engine = new ScrapeEngine({ rootDir: this.rootDir, - log: line => this.deps.log?.(line) + log: line => this.deps.log?.(line), + download: { + maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), + minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000), + maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000) + } }); } @@ -280,6 +311,18 @@ export class MediaService { }; } + /** Absolute path and content type for a downloaded file, or null. */ + getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null { + const item = this.getMediaItem(scrapeId, itemId); + if (!item?.file_path) return null; + + const extension = item.file_path.split(".").pop()?.toLowerCase() ?? ""; + return { + path: join(this.rootDir, scrapeId, item.file_path), + contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream" + }; + } + /** One media row inside a scrape, with its scrape and channel for context. */ getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { const meta = this.engine.get(scrapeId); @@ -315,9 +358,11 @@ export class MediaService { if (results.length >= limit) break; results.push({ scrapeId: meta.id, + itemId: row.id, channelId: meta.channelId, kind: row.kind, url: row.url, + localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null, provider: row.provider, downloaded: row.downloaded === 1, createdAt: row.created_at diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 3e36270..7de1ee2 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -754,6 +754,15 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/media/:scrapeId/:itemId/file", + wrap(async (req, res) => { + const file = deps.media.getMediaFile(String(req.params.scrapeId), Number(req.params.itemId)); + if (!file) throw new HttpError(404, "file not found"); + res.type(file.contentType).sendFile(file.path); + }) + ); + app.get( "/api/media/storage", wrap(async (_req, res) => {