From 42fd52b96be0582bdab4823e67939f9a9b89e502 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 11:53:59 +0200 Subject: [PATCH] Add custom download destinations, confirmations, and the bot error feed --- CURRENT.md | 7 ++ docs/media-mcp.md | 2 +- docs/media-scraper.md | 3 + docs/media.md | 15 ++- docs/webui.md | 4 +- media-scrape/src/download.ts | 6 +- media-scrape/src/engine.ts | 41 +++++-- media-scrape/src/types.ts | 2 + media-scrape/test/run.ts | 73 ++++++++++++ poc/public/media.css | 34 ++++++ poc/public/media.js | 210 +++++++++++++++++++++++++++++++---- poc/src/control-bridge.ts | 19 +++- poc/src/media.ts | 93 ++++++++++++---- poc/src/webui.ts | 24 +++- 14 files changed, 471 insertions(+), 62 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index f642398..2270756 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,13 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C15 Scrape UX: custom download destination (absolute path per scrape; files there, + previews local, embed thumbnails copied to both, custom files not served), confirmation popup + before Links/Download, New scrape/History right-aligned with New scrape furthest right, modal + X and backdrop close, active-only progress strip, and a Bot error feed with a red unseen badge + that clears on open plus a Clear button. Live run `...-20260916095029`: 6 files and 7 embed + thumbnails in `/tmp/opencode/media-dest//`, 11 local previews, 0 local files, `localUrl` + null; error record and clear verified over the API. - [x] C14 Links-mode embed thumbnails, item metadata, filters, and the scrape popup. Metadata: poster (`author_id`/`author_name`), `posted_at`, `width`/`height`, `aspect_ratio`, with an `ffprobe` fallback after downloads. Filters: poster, aspect preset (vertical/square/4:3/16:9/ diff --git a/docs/media-mcp.md b/docs/media-mcp.md index 2ee024b..6426669 100644 --- a/docs/media-mcp.md +++ b/docs/media-mcp.md @@ -35,7 +35,7 @@ preset keys from `filter-presets.ts`; `after`/`before` are ISO times on `posted_ | Tool | Access | Purpose | | --- | --- | --- | -| `scrape_media` | trusted | Start a scrape (`channelId`, `mode`) | +| `scrape_media` | trusted | Start a scrape (`channelId`, `guildId?`, `mode?`, `backend?`, `downloadDir?`) | | `list_media_scrapes` | trusted | Every scrape with state, size, and counts | | `get_media_scrape` | trusted | One scrape, live counters and cursor | | `stop_media_scrape` | trusted | Stop the active scrape | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index e33c66b..0d13861 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -103,6 +103,9 @@ Downloads run after a finished walk when the scrape used `mode: "download"`: `file_bytes` are written to the row, so the run resumes and dedupes by content. - Only attachment and sticker rows are targets. Embed rows are excluded in SQL (`source != 'embed'`). +- With a custom destination, files go to `//media/` and the row stores the + absolute path; without one they stay in `/media/`. Embed previews are copied to + `//thumbnails/` while the local copy stays for the Media tab. - Progress counters: `downloaded`, `downloadErrors`. Previews run after every walk: diff --git a/docs/media.md b/docs/media.md index 6dea8c0..6d361f4 100644 --- a/docs/media.md +++ b/docs/media.md @@ -80,6 +80,10 @@ media-scrape/db/ - `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows are never download targets. The Media tab prefers the local file when one exists, so the archive survives CDN URL expiry. +- A custom download destination sends the bytes to `//media/` instead. Previews + stay local, and embed thumbnails are copied to `//thumbnails/` so the download + has a poster. Files in a custom destination are not served or linked by the Media tab; the user + owns that path. - Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every other embed stays a link. Its thumbnail is cached (below); the video itself is never fetched. - Previews are cached small (WebP, 320px) in `thumbs/` after every walk, in both modes: Discord @@ -134,10 +138,13 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun 21:9, Other), resolution (SD, HD, FHD, QHD, 4K+, Unknown), and a posted-date range (from/to). Grid with lazy loading; previews come from the cache, then the local file, then the CDN; each tile's tooltip shows the poster and posted time. A scrape strip shows live progress with Stop. - **New scrape…** opens a popup with Server and Group chat trees (servers expand to channels; each - row has Links and Download), and **History** lists each scrape with its start time, duration, - pages, messages, media counts, bytes, files, and previews. The view fills the window below the - header and the call strip. + **New scrape…** (rightmost) opens a popup with Server and Group chat trees (servers expand to + channels; each row has Links and Download). Every scrape button asks for confirmation, and + Download can target a custom absolute destination. **History** lists each scrape with its start + time, duration, pages, messages, media counts, bytes, files, and previews. The popups close on + backdrop click, X, or Close. Only active scrapes show in the strip; the view fills the window + below the header and the call strip. The Bot tab lists bot and scrape errors with a red badge + that disappears when the tab is opened and a Clear button. - **Bot** tab: slopbot connection state, the scrape queue, and the storage limit with a usage bar. The bar is segmented and colour coded by media kind (images, GIFs, video, audio, other) plus the database overhead. diff --git a/docs/webui.md b/docs/webui.md index a27d424..c95475d 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -263,7 +263,9 @@ replaceable. | GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts | | GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over | -| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` | +| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?`, `downloadDir?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`; `downloadDir` is an absolute path for downloaded files (previews stay local). Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` | +| GET | `/api/media/errors` | none | Bot and scrape errors, newest first | +| POST | `/api/media/errors/clear` | none | Clear the error list | | POST | `/api/media/scrape/stop` | `id` | Stop a media scrape | | POST | `/api/media/clean` | `ids?` | Delete the given scrapes, or every scrape when `ids` is absent | | GET | `/api/logs` | `limit` | The persistent action log | diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index 4f47f0a..f2ae447 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -44,8 +44,10 @@ export interface DownloadOptions { maxRetries?: number; /** Preview size cap. Default 5 MiB. */ previewMaxBytes?: number; - /** Cache third-party embed thumbnails. Download mode only. */ + /** Cache third-party embed thumbnails. */ embedPreviews?: boolean; + /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */ + destinationDir?: string | null; sleep?: (ms: number) => Promise; random?: () => number; log?: (line: string) => void; @@ -134,7 +136,7 @@ export async function downloadOne( const file = `${hash}.${extension}`; writeFileSync(join(dir, file), buffer); - return { id: media.id, state: "downloaded", path: join("media", file), hash, bytes: buffer.byteLength }; + return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength }; } } catch (error) { lastError = error instanceof Error ? error.message : String(error); diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 428800e..88f8fbc 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -3,7 +3,8 @@ * Rules: docs/media-scraper.md. */ -import { join } from "node:path"; +import { copyFileSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; import { downloadGapMs, downloadOne, @@ -58,6 +59,8 @@ export interface ScrapeEngineOptions { previews?: boolean; /** Read dimensions from a downloaded file. Defaults to ffprobe. */ probe?: (path: string) => Promise<{ width: number; height: number } | null>; + /** Called when a scrape fails. */ + onError?: (scrapeId: string, message: string) => void; } interface QueuedJob { @@ -91,6 +94,7 @@ export class ScrapeEngine { guildId: string | null; mode: ScrapeMode; backend: ScrapeBackend; + downloadDir?: string | null; }): Promise { const now = this.now(); const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now)); @@ -111,7 +115,8 @@ export class ScrapeEngine { messages: 0, media: emptyMediaCounts(), errors: 0, - lastError: null + lastError: null, + downloadDir: input.downloadDir ?? null }; writeScrapeMeta(dir, meta); @@ -251,6 +256,7 @@ export class ScrapeEngine { meta.state = "failed"; meta.lastError = error instanceof Error ? error.message : String(error); this.log(`${meta.id}: failed: ${meta.lastError}`); + this.options.onError?.(meta.id, meta.lastError); } finally { if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) { meta.finishedAt = new Date(this.now()).toISOString(); @@ -263,7 +269,13 @@ export class ScrapeEngine { /** Fetch attachment bytes for a finished walk. Serial, hash-addressed, resumable. */ private async runDownloads(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { - const options = this.options.download ?? {}; + const customDir = meta.downloadDir ?? null; + const options: DownloadOptions = { + ...(this.options.download ?? {}), + destinationDir: customDir + }; + const targetDir = customDir ? join(customDir, meta.id, "media") : join(dir, "media"); + mkdirSync(targetDir, { recursive: true }); const pending = store.pendingDownloads(); for (const row of pending) { @@ -272,21 +284,24 @@ export class ScrapeEngine { return; } - const outcome = await downloadOne(row, join(dir, "media"), { + const outcome = await downloadOne(row, targetDir, { ...options, log: line => this.log(`${meta.id}: ${line}`) }); if (outcome.state === "downloaded" && outcome.path && outcome.hash) { + const filePath = customDir ? join(targetDir, outcome.path) : join("media", outcome.path); store.markDownloaded(row.id, { - path: outcome.path, + path: filePath, hash: outcome.hash, bytes: outcome.bytes ?? 0 }); meta.downloaded = (meta.downloaded ?? 0) + 1; if (row.width === null || row.height === null) { - const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path)); + const dimensions = await (this.options.probe ?? probeDimensions)( + join(targetDir, outcome.path) + ); if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height); } } else { @@ -301,7 +316,11 @@ export class ScrapeEngine { /** Cache small previews. Embed thumbnails are cached in both modes. */ private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { - const options = { ...(this.options.download ?? {}), embedPreviews: true }; + const options = { + ...(this.options.download ?? {}), + embedPreviews: true, + destinationDir: meta.downloadDir ?? null + }; const pending = store.pendingPreviews(5000, true); for (const row of pending) { @@ -315,6 +334,14 @@ export class ScrapeEngine { if (outcome.state === "downloaded" && outcome.hash) { store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 }); meta.previewed = (meta.previewed ?? 0) + 1; + + // Embed thumbnails also land in the custom destination, so the download has a + // viewable poster for embeds while the media tab keeps its local copy. + if (options.destinationDir && row.source === "embed") { + const target = join(options.destinationDir, meta.id, "thumbnails", `${outcome.hash}.webp`); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(join(dir, "thumbs", `${outcome.hash}.webp`), target); + } } else { store.markPreviewError(row.id, outcome.error ?? outcome.state); } diff --git a/media-scrape/src/types.ts b/media-scrape/src/types.ts index 72e86ab..eb2af74 100644 --- a/media-scrape/src/types.ts +++ b/media-scrape/src/types.ts @@ -94,6 +94,8 @@ export interface ScrapeMeta { previewed?: number; /** When the scrape reached done or failed. */ finishedAt?: string | null; + /** Absolute destination for downloaded files. Null means the local `media/` folder. */ + downloadDir?: string | null; } export function emptyMediaCounts(): Record { diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 0da9dca..2609ed4 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -637,6 +637,79 @@ void (async () => { assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true); }); + await test("custom download destination keeps files off-archive and copies embed previews", async () => { + const dest = join(root, "custom-dest"); + mkdirSync(dest, { recursive: true }); + + const backend = new FakeBackend([ + { + messages: [ + message("702", { + attachments: [attachment({ filename: "offsite.png", contentType: "image/png" })], + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=x", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/embed/x", + imageUrl: null, + thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" + } + ] + }) + ] + } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 2, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: true, + download: { + fetchImpl: async () => + new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }), + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ + channelId: "130", + guildId: null, + mode: "download", + backend, + downloadDir: dest + }); + await engine.processQueue(); + + const meta = engine.get(started.id); + assert.equal(meta?.downloadDir, dest); + assert.equal(meta?.downloaded, 1); + + const store = new ScrapeStore(join(root, started.id)); + const file = store.queryMedia({ kind: "image" })[0]; + const filePath = file.file_path; + assert.ok(filePath && filePath.startsWith(dest)); + assert.equal(existsSync(filePath), true); + assert.equal(existsSync(join(root, started.id, "media", filePath.split("/").pop() ?? "")), false); + + const embed = store.queryMedia({ kind: "video" })[0]; + assert.ok(embed.preview_hash); + assert.equal( + existsSync(join(dest, started.id, "thumbnails", `${embed.preview_hash}.webp`)), + true + ); + assert.equal(existsSync(join(root, started.id, "thumbs", `${embed.preview_hash}.webp`)), true); + store.close(); + }); + await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, diff --git a/poc/public/media.css b/poc/public/media.css index f9d843b..0d73870 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -286,3 +286,37 @@ background: var(--color-accent-400, #4f8cff); animation: vk-media-pulse 1.2s ease-in-out infinite; } + +.vk-media-spacer { + flex: 1; +} + +.vk-media-check { + display: flex; + align-items: center; + gap: 6px; + font: 400 11px/1.4 ui-monospace, Menlo, monospace; + color: var(--color-neutral-300); +} + +.vk-media-dest { + width: 100%; +} + +.vk-media-error { + color: #ff8080; +} + +.vk-media-badge { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 14px; + height: 14px; + margin-left: 4px; + padding: 0 4px; + border-radius: 7px; + background: #e5484d; + color: #fff; + font: 600 9px/1 ui-monospace, Menlo, monospace; +} diff --git a/poc/public/media.js b/poc/public/media.js index 56452a3..68f8404 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -22,6 +22,7 @@ dms: [], loadedChannels: false, authors: null, + errors: [], filters: { source: "", channelId: "", @@ -92,6 +93,13 @@ document.querySelector("#scope-bot")?.classList.toggle("active", view === "bot"); document.querySelector("#scope-guild")?.classList.remove("active"); document.querySelector("#scope-dm")?.classList.remove("active"); + + if (view === "bot") { + localStorage.setItem("vk-media-errors-seen", String(errorCount)); + updateErrorBadge(); + void refreshErrors(); + } + render(); } @@ -191,7 +199,7 @@ renderGrid(); } - async function startScrape(channelId, guildId, mode) { + async function startScrape(channelId, guildId, mode, downloadDir) { if (!channelId) { setStatus("no channel selected"); return; @@ -202,7 +210,8 @@ channelId, guildId: guildId ?? null, mode, - backend: guildId ? "auto" : "puppet" + backend: guildId ? "auto" : "puppet", + downloadDir: downloadDir ?? null }); setStatus(`started ${scrape.id} (${mode})`); scrapeModal.hidden = true; @@ -219,8 +228,8 @@ function renderScrapes() { scrapeList.replaceChildren(); const active = state.scrapes.filter(scrape => ["queued", "running", "paused"].includes(scrape.state)); - const rows = active.length ? active : state.scrapes.slice(0, 5); - for (const scrape of rows) { + + for (const scrape of active) { const row = el("div", "vk-media-scrape"); const media = scrape.media ?? {}; @@ -239,21 +248,79 @@ ) ); - if (active.includes(scrape)) { - row.append( - button("Stop", async () => { - await window.VK.api.post("/api/media/scrape/stop", { id: scrape.id }); - void loadScrapes(); - }) - ); - } else { - row.append(el("span", "vk-media-status", durationLabel(scrape))); - } + row.append( + button("Stop", async () => { + await window.VK.api.post("/api/media/scrape/stop", { id: scrape.id }); + void loadScrapes(); + }) + ); scrapeList.append(row); } } + // ---- Bot errors ---- + + let errorCount = 0; + + function seenErrorCount() { + return Number(localStorage.getItem("vk-media-errors-seen") ?? 0); + } + + function updateErrorBadge() { + const badge = document.querySelector("#scope-bot-badge"); + if (!badge) return; + + const unseen = Math.max(0, errorCount - seenErrorCount()); + badge.textContent = String(unseen); + badge.hidden = unseen === 0; + } + + function renderErrors() { + scrapeList.replaceChildren(); + + const head = el("div", "vk-media-scrape"); + head.append(el("span", null, `${(state.errors ?? []).length} error(s)`)); + head.append( + button("Clear errors", async () => { + try { + await window.VK.api.post("/api/media/errors/clear", {}); + } catch { + // the list stays until the next refresh + } + state.errors = []; + errorCount = 0; + localStorage.setItem("vk-media-errors-seen", "0"); + updateErrorBadge(); + renderErrors(); + }) + ); + scrapeList.append(head); + + for (const entry of state.errors ?? []) { + scrapeList.append( + el( + "div", + "vk-media-scrape vk-media-error", + `${new Date(entry.at).toLocaleTimeString()} · ${entry.source} · ${entry.message}` + ) + ); + } + + if (!(state.errors ?? []).length) scrapeList.append(el("div", "vk-media-scrape", "No errors.")); + } + + async function refreshErrors() { + try { + state.errors = await window.VK.api.get("/api/media/errors"); + } catch { + state.errors = []; + } + errorCount = (state.errors ?? []).length; + updateErrorBadge(); + if (state.view === "bot") renderErrors(); + } + function renderGrid() { grid.replaceChildren(); for (const item of state.items) { @@ -319,8 +386,7 @@ toolbar.append(chip); } - toolbar.append(button("New scrape…", openScrapeModal)); - toolbar.append(button("History", openHistoryModal)); + const sourceSelect = document.createElement("select"); sourceSelect.className = "vk-chip vk-chip-off"; @@ -428,6 +494,10 @@ void loadMedia(true); }) ); + + toolbar.append(el("span", "vk-media-spacer")); + toolbar.append(button("History", openHistoryModal)); + toolbar.append(button("New scrape…", openScrapeModal)); } else { toolbar.append(el("span", null, "Slopbot is the dedicated bot core. Setup: docs/slopbot.md.")); } @@ -444,12 +514,14 @@ statusLine.textContent = bot?.connected ? `connected as ${bot.user} in ${bot.guilds.length} guild(s)` : "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)"; + void loadScrapes().then(renderScrapes); + void refreshErrors(); } else { void loadChannels().then(renderToolbar); void loadAuthors().then(renderToolbar); void loadMedia(true); + void loadScrapes(); } - void loadScrapes(); } overlay.querySelector("#media-close").addEventListener("click", hide); @@ -465,6 +537,16 @@ document.querySelector("#scope-guild")?.addEventListener("click", hide); document.querySelector("#scope-dm")?.addEventListener("click", hide); + const errorBadge = el("span", "vk-media-badge"); + errorBadge.id = "scope-bot-badge"; + errorBadge.hidden = true; + scopeBot?.append(errorBadge); + + void refreshErrors(); + setInterval(() => { + if (state.view !== "bot") void refreshErrors(); + }, 15000); + // ---- Scrape popup: server or group-chat trees ---- const scrapeModal = document.createElement("div"); @@ -472,7 +554,11 @@ scrapeModal.hidden = true; scrapeModal.innerHTML = `
- New scrape +
+ New scrape + + +
@@ -491,8 +577,8 @@ function channelRow(channelId, guildId, label) { const row = el("div", "vk-media-channel"); row.append(el("span", "vk-media-channel-name", label)); - row.append(button("Links", () => void startScrape(channelId, guildId, "links"))); - row.append(button("Download", () => void startScrape(channelId, guildId, "download"))); + row.append(button("Links", () => askScrape(channelId, guildId, "links", label))); + row.append(button("Download", () => askScrape(channelId, guildId, "download", label))); return row; } @@ -558,6 +644,78 @@ scrapeModal.querySelector("#scrape-cancel").addEventListener("click", () => { scrapeModal.hidden = true; }); + scrapeModal.querySelector("#scrape-x").addEventListener("click", () => { + scrapeModal.hidden = true; + }); + scrapeModal.addEventListener("click", event => { + if (event.target === scrapeModal) scrapeModal.hidden = true; + }); + + // ---- Confirmation popup for Links / Download ---- + + const confirmModal = document.createElement("div"); + confirmModal.className = "vk-media-modal"; + confirmModal.hidden = true; + confirmModal.innerHTML = ` +
+ Start scrape? +
+ + +
+ + +
+
+ `; + document.body.append(confirmModal); + + let confirmAction = null; + + function askScrape(channelId, guildId, mode, label) { + const isDownload = mode === "download"; + const destRow = confirmModal.querySelector("#confirm-dest-row"); + const destInput = confirmModal.querySelector("#confirm-dest"); + const checkbox = confirmModal.querySelector("#confirm-custom-dest"); + + confirmModal.querySelector("#confirm-title").textContent = + isDownload ? "Download scrape?" : "Links scrape?"; + confirmModal.querySelector("#confirm-note").textContent = isDownload + ? `${label} — downloads files${guildId ? "" : " with the puppet"}.` + : `${label} — indexes links and caches previews${guildId ? "" : " with the puppet"}.`; + + destRow.hidden = !isDownload; + destInput.hidden = true; + destInput.value = ""; + checkbox.checked = false; + checkbox.onchange = () => { + destInput.hidden = !checkbox.checked; + }; + + confirmAction = () => + startScrape(channelId, guildId, mode, isDownload && checkbox.checked ? destInput.value.trim() : null); + confirmModal.hidden = false; + } + + confirmModal.querySelector("#confirm-cancel").addEventListener("click", () => { + confirmModal.hidden = true; + }); + confirmModal.querySelector("#confirm-start").addEventListener("click", () => { + const action = confirmAction; + confirmModal.hidden = true; + void action?.(); + }); + confirmModal.addEventListener("click", event => { + if (event.target === confirmModal) confirmModal.hidden = true; + }); // ---- History popup ---- @@ -566,7 +724,11 @@ historyModal.hidden = true; historyModal.innerHTML = `
- Scrape history +
+ Scrape history + + +
@@ -630,6 +792,12 @@ historyModal.querySelector("#history-close").addEventListener("click", () => { historyModal.hidden = true; }); + historyModal.querySelector("#history-x").addEventListener("click", () => { + historyModal.hidden = true; + }); + historyModal.addEventListener("click", event => { + if (event.target === historyModal) historyModal.hidden = true; + }); // ---- Settings: media storage ---- diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index 27cdbc4..bbf6d46 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -774,11 +774,24 @@ function buildServer(): McpServer { .enum(["auto", "slopbot", "puppet"]) .optional() .describe("auto (default) picks slopbot for its guilds, the client otherwise"), - confirm: z.boolean().optional().describe("Accept the near-limit warning") + confirm: z.boolean().optional().describe("Accept the near-limit warning"), + downloadDir: z + .string() + .optional() + .describe("Absolute destination for downloads; previews stay local") } }, - async ({ channelId, guildId, mode, backend, confirm }) => - jsonText(await media.startScrape({ channelId, guildId: guildId ?? null, mode, backend, confirm })) + async ({ channelId, guildId, mode, backend, confirm, downloadDir }) => + jsonText( + await media.startScrape({ + channelId, + guildId: guildId ?? null, + mode, + backend, + confirm, + downloadDir: downloadDir ?? null + }) + ) ); server.registerTool( diff --git a/poc/src/media.ts b/poc/src/media.ts index 4ddf355..f29a7ef 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -6,7 +6,7 @@ */ import { rmSync } from "node:fs"; -import { join } from "node:path"; +import { isAbsolute, join } from "node:path"; import { dirSizeBytes, isNearLimit, @@ -49,6 +49,14 @@ export interface StartScrapeInput { backend?: "auto" | "slopbot" | "puppet"; /** Set by a re-run of the command after the near-limit warning. */ confirm?: boolean; + /** Absolute destination for downloaded files. Null keeps them under the scrape dir. */ + downloadDir?: string | null; +} + +export interface MediaErrorEntry { + at: string; + source: string; + message: string; } export interface ScrapeBytes { @@ -138,6 +146,9 @@ export class MediaService { private botFetcher: SlopbotFetcher | null = null; private botStarting: Promise | null = null; + /** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */ + private readonly errors: MediaErrorEntry[] = []; + constructor(private readonly deps: MediaServiceDeps) { const number = (value: string | undefined, fallback: number) => { const parsed = Number(value); @@ -147,6 +158,7 @@ export class MediaService { this.engine = new ScrapeEngine({ rootDir: this.rootDir, log: line => this.deps.log?.(line), + onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024), @@ -171,7 +183,9 @@ export class MediaService { this.deps.log?.("slopbot connected"); }) .catch(error => { - this.deps.log?.(`slopbot login failed: ${String(error)}`); + const message = `slopbot login failed: ${String(error)}`; + this.deps.log?.(message); + this.recordError("slopbot", message); }); } @@ -264,6 +278,10 @@ export class MediaService { } async startScrape(input: StartScrapeInput): Promise { + if (input.downloadDir && !isAbsolute(input.downloadDir)) { + throw new Error("downloadDir must be an absolute path"); + } + this.applyRetention(); const status = this.storageStatus(); @@ -275,30 +293,38 @@ export class MediaService { ); } - const mode = input.mode ?? "links"; - let backend: ScrapeBackend; + try { + const mode = input.mode ?? "links"; + let backend: ScrapeBackend; - if (input.backend === "puppet") { - backend = this.puppetBackend(); - } else { - const available = await this.ensureSlopbot(); - const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false); + if (input.backend === "puppet") { + backend = this.puppetBackend(); + } else { + const available = await this.ensureSlopbot(); + const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false); - if (input.backend === "slopbot" && (!available || !inGuild)) { - throw new Error("slopbot is not connected to that guild"); + if (input.backend === "slopbot" && (!available || !inGuild)) { + throw new Error("slopbot is not connected to that guild"); + } + + const fetcher = this.botFetcher; + backend = + available && inGuild && fetcher + ? new SlopbotBackend(fetcher) + : this.puppetBackend(); } - const fetcher = this.botFetcher; - backend = - available && inGuild && fetcher ? new SlopbotBackend(fetcher) : this.puppetBackend(); + return this.engine.start({ + channelId: input.channelId, + guildId: input.guildId ?? null, + mode, + backend, + downloadDir: input.downloadDir ?? null + }); + } catch (error) { + this.recordError("scrape", error instanceof Error ? error.message : String(error)); + throw error; } - - return this.engine.start({ - channelId: input.channelId, - guildId: input.guildId ?? null, - mode, - backend - }); } stopScrape(id: string): void { @@ -322,6 +348,10 @@ export class MediaService { /** Absolute path and content type for a downloaded file, or null. */ getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null { + // A custom destination is the user's filesystem, outside the archive: not served. + const meta = this.engine.get(scrapeId); + if (meta?.downloadDir) return null; + const item = this.getMediaItem(scrapeId, itemId); if (!item?.file_path) return null; @@ -343,6 +373,22 @@ export class MediaService { }; } + private recordError(source: string, message: string): void { + this.errors.unshift({ at: new Date().toISOString(), source, message }); + if (this.errors.length > 50) this.errors.length = 50; + } + + /** Bot and scrape errors, newest first. */ + listErrors(): MediaErrorEntry[] { + return [...this.errors]; + } + + clearErrors(): number { + const count = this.errors.length; + this.errors.length = 0; + return count; + } + /** Distinct posters across every scrape, biggest first. */ listAuthors(): Array<{ id: string | null; name: string | null; count: number }> { const totals = new Map(); @@ -434,7 +480,10 @@ export class MediaService { height: row.height, aspectRatio: row.aspect_ratio, url: row.url, - localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null, + localUrl: + row.file_path && !meta.downloadDir + ? `/api/media/${meta.id}/${row.id}/file` + : null, thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, provider: row.provider, downloaded: row.downloaded === 1, diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 0159dae..7e2ec55 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -832,6 +832,20 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/media/errors", + wrap(async (_req, res) => { + res.json({ ok: true, result: deps.media.listErrors() }); + }) + ); + + app.post( + "/api/media/errors/clear", + wrap(async (_req, res) => { + res.json({ ok: true, result: { cleared: deps.media.clearErrors() } }); + }) + ); + app.post( "/api/media/scrape", wrap(async (req, res) => { @@ -844,8 +858,16 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { const requested = String(req.body?.backend ?? "auto"); const backend = requested === "slopbot" || requested === "puppet" ? requested : "auto"; + const downloadDir = req.body?.downloadDir ? String(req.body.downloadDir) : null; - const scrape = await deps.media.startScrape({ channelId, guildId, mode, confirm, backend }); + const scrape = await deps.media.startScrape({ + channelId, + guildId, + mode, + confirm, + backend, + downloadDir + }); deps.getPuppet()?.appendAction({ type: "media", action: "scrape",