From dea1724eb387e9f237040399280eabe927815638 Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 07:33:35 +0200 Subject: [PATCH] Add daemon restart, preview diagnostics, browse caching, and rate-limit pauses --- CURRENT.md | 9 ++++ docs/daemon.md | 1 + docs/media-scraper.md | 19 +++++++++ docs/webui.md | 3 ++ media-scrape/src/download.ts | 23 ++++++++++- media-scrape/src/engine.ts | 79 ++++++++++++++++++++++++++++++++++-- media-scrape/src/store.ts | 39 +++++++++++++++++- media-scrape/test/run.ts | 73 +++++++++++++++++++++++++++++++++ poc/public/index.html | 3 +- poc/public/media.js | 59 +++++++-------------------- poc/src/control-bridge.ts | 37 +++++++++++++++-- poc/src/media.ts | 51 +++++++++++++++++++++++ poc/src/webui.ts | 27 ++++++++++++ 13 files changed, 367 insertions(+), 56 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 13c04c8..0bd4bff 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,15 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C36 Daemon restart, preview diagnostics, browse-time caching, and rate-limit pauses. + Settings → Server has a **Restart daemon** button (`POST /api/daemon/restart`: re-execs the + daemon, keeps the client running). The preview pass logs every failure with scrape/item/kind/ + provider/reason plus a per-pass summary, and `GET /api/media/preview-failures` lists them. + Browsing no longer fetches Discord at all: the grid uses cached stills only (placeholders for + missing ones), GIF hover plays the cached animated GIF (kept as `anim_hash`, served from + `/anim`), and streaming only starts on explicit play. `429`s from previews/downloads raise a + cooldown (`VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS`, default 60 s, or `Retry-After`), leaving rows + pending and pausing both walker and previews. - [x] C35 Walk/preview cycling. A live scrape now alternates: walk 5 pages, preview up to one batch (25, with the cooldown), walk on. Tiles fill in during a long walk instead of only at the end, and preview traffic is spread through the walk. Configurable with diff --git a/docs/daemon.md b/docs/daemon.md index df7e787..98a2b8f 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -181,6 +181,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_MEDIA_PREVIEW_BATCH` | `25` | Previews per batch before the cooldown. | | `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS` | `60000` | Cooldown between preview batches. `0` disables batching. | | `VESKSLOP_MEDIA_WALK_CYCLE_PAGES` | `5` | Walk pages between preview cycles on a live scrape. `0` defers previews to the end. | +| `VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS` | `60000` | Minimum pause after a `429` before media traffic resumes. | | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index c6d2584..f7afd91 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -157,6 +157,25 @@ Previews run after every walk: (`VESKSLOP_MEDIA_PREVIEW_BATCH`, `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`), because bursts of thumbnail fetches trigger Discord and provider rate limits. A batch counts rows, not successes. +## Rate limits + +Discord and the provider APIs throttle bursts. The engine detects limiting and holds all media +traffic: + +- A `429` from a preview or download fetch pauses the pass; the row stays pending (never marked as a + permanent error) and is retried later. +- A walk page that fails with a `429` or a rate-limit message registers a cooldown, and every page + and preview waits it out before continuing. +- The pause is `VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS` (default 60 s), or the server's `Retry-After` + when it is longer. Detection is logged as `rate limit detected; pausing media traffic until …`. + +## Browsing cache + +Browsing never calls Discord. The grid renders cached previews or local files only; videos without +a cached still show a labeled placeholder, and playback streams only when the user presses play. +GIF hover plays the animated GIF cached in `thumbs/` (the original is kept alongside the static +frame and referenced by `anim_hash`), never the provider, so scrolling cannot spam anyone. + ## Job states `queued`, `running`, `paused`, `stopping`, `done`, `failed`, `pending-delete`, `deleting`, `deleted`. diff --git a/docs/webui.md b/docs/webui.md index 7227a34..b9f5df4 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -271,6 +271,9 @@ replaceable. | POST | `/api/media/count` | `channelId`, `guildId?`, `after?`, `before?` | Discord search total for the channel or range. Returns `{ count }` or `{ count: null, reason }` (slopbot guild channels only) | | GET | `/api/media/errors` | none | Bot and scrape errors, newest first | | POST | `/api/media/errors/clear` | none | Clear the error list | +| GET | `/api/media/preview-failures` | `limit?` | Rows whose preview failed, with the reason | +| GET | `/api/media/:scrapeId/:itemId/anim` | none | The cached animated GIF for hover playback | +| POST | `/api/daemon/restart` | none | Replace the daemon process with a fresh copy (the client keeps running) | | POST | `/api/media/scrape/stop` | `id` | Stop a media scrape | | POST | `/api/media/scrape/resume` | `id` | Continue a paused scrape from its cursor | | POST | `/api/media/clean` | `ids?` | Delete the given scrapes, or every scrape when `ids` is absent | diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index e151134..f663c92 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -80,12 +80,14 @@ export interface DownloadOptions { export interface DownloadOutcome { id: number; - state: "downloaded" | "skipped" | "failed"; + state: "downloaded" | "skipped" | "failed" | "ratelimited"; path?: string; hash?: string; bytes?: number; /** File extension of the stored bytes, when known. */ ext?: string; + /** Seconds the server asked us to wait, when it is known. */ + retryAfterMs?: number; error?: string; } @@ -157,6 +159,16 @@ export async function downloadOne( const response = await fetchImpl(media.url); if (!response.ok) { + if (response.status === 429) { + const after = Number(response.headers.get("retry-after") ?? 0); + return { + id: media.id, + state: "ratelimited", + retryAfterMs: Number.isFinite(after) && after > 0 ? after * 1000 : undefined, + error: "http 429" + }; + } + if (SKIP_STATUS.has(response.status)) { return { id: media.id, state: "skipped", error: `http ${response.status}` }; } @@ -252,6 +264,15 @@ export async function downloadPreview( try { const response = await fetchImpl(target); + if (response.status === 429) { + const after = Number(response.headers.get("retry-after") ?? 0); + return { + id: media.id, + state: "ratelimited", + retryAfterMs: Number.isFinite(after) && after > 0 ? after * 1000 : undefined, + error: "http 429" + }; + } if (!response.ok) return { id: media.id, state: "skipped", error: `http ${response.status}` }; const buffer = Buffer.from(await response.arrayBuffer()); diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 750fac9..1e7b8a0 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -64,6 +64,8 @@ export interface ScrapeEngineOptions { previews?: boolean; /** Walk pages between preview cycles. 0 defers all previews to the end. Default 5. */ walkCyclePages?: number; + /** Pause after a rate limit is detected. Default 60 s. */ + rateLimitPauseMs?: number; /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */ probe?: ( path: string @@ -88,11 +90,17 @@ function oldestId(messages: CoreMessage[]): string | null { return oldest; } +function isRateLimited(error: unknown): boolean { + const anyError = error as { status?: number; message?: string } | undefined; + return anyError?.status === 429 || /\b429\b|rate.?limit/i.test(String(anyError?.message ?? error)); +} + export class ScrapeEngine { private readonly queue: QueuedJob[] = []; private readonly active = new Map(); private readonly stopRequested = new Set(); private readonly pageTimes: number[] = []; + private rateLimitUntil = 0; private processingPromise: Promise | null = null; constructor(private readonly options: ScrapeEngineOptions) {} @@ -264,6 +272,7 @@ export class ScrapeEngine { break; } + await this.respectRateLimit(); const page = await this.fetchWithBackoff(meta, backend); if (page.rateLimitedMs) await this.sleep(page.rateLimitedMs); @@ -366,6 +375,8 @@ export class ScrapeEngine { return; } + await this.respectRateLimit(); + const outcome = await downloadOne(row, targetDir, { ...options, log: line => this.log(`${meta.id}: ${line}`) @@ -394,6 +405,10 @@ export class ScrapeEngine { meta.previewed = (meta.previewed ?? 0) + 1; } } + } else if (outcome.state === "ratelimited") { + // Leave the row pending and hold off: the server asked for a break. + this.registerRateLimit(outcome.retryAfterMs); + this.log(`${meta.id}: download #${row.id} rate limited (${outcome.error ?? "429"})`); } else { store.markDownloadError(row.id, outcome.error ?? outcome.state); if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1; @@ -412,6 +427,11 @@ export class ScrapeEngine { destinationDir: meta.downloadDir ?? null }; const pending = store.pendingPreviews(limit > 0 ? limit : 5000, true); + let cached = 0; + let skipped = 0; + let failed = 0; + + if (pending.length) this.log(`${meta.id}: preview pass: ${pending.length} pending`); const batchSize = Math.max(1, options.previewBatchSize ?? 25); const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000); let batch = 0; @@ -432,6 +452,8 @@ export class ScrapeEngine { return; } + await this.respectRateLimit(); + // Audio has no image proxy: render a waveform strip with ffmpeg. if (row.kind === "audio" && row.source !== "embed") { const waveform = await remoteFrame(row.url, join(dir, "thumbs"), { waveform: true }); @@ -444,6 +466,8 @@ export class ScrapeEngine { meta.previewed = (meta.previewed ?? 0) + 1; } else { store.markPreviewError(row.id, "waveform failed"); + failed++; + this.log(`${meta.id}: preview #${row.id} audio waveform failed (${row.url.slice(0, 60)})`); } this.persist(dir, meta); await this.sleep(previewGapMs(options)); @@ -458,6 +482,8 @@ export class ScrapeEngine { const resolved = await resolveEmbedPreview(row, options); if (!resolved) { store.markPreviewError(row.id, "no preview source"); + skipped++; + this.log(`${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} has no preview source`); this.persist(dir, meta); await this.sleep(previewGapMs(options)); await afterRow(); @@ -473,19 +499,22 @@ export class ScrapeEngine { let bytes = outcome.bytes ?? 0; let ext = outcome.ext ?? "webp"; - // Animated GIFs are kept static in the grid; hovering plays the original. + // Animated GIFs show a static frame in the grid; hovering plays the cached + // original from disk, so browsing never hotlinks the provider. + let animHash: string | undefined; if (ext === "gif") { const poster = await gifPoster(join(dir, "thumbs", `${hash}.gif`), join(dir, "thumbs")); if (poster) { - rmSync(join(dir, "thumbs", `${hash}.gif`), { force: true }); + animHash = hash; hash = poster.hash; bytes = poster.bytes; ext = "webp"; } } - store.markPreview(row.id, { hash, bytes, ext }); + store.markPreview(row.id, { hash, bytes, ext, animHash }); meta.previewed = (meta.previewed ?? 0) + 1; + cached++; // Embed thumbnails also land in the custom destination, so the download has a // viewable poster for embeds while the media tab keeps its local copy. @@ -506,18 +535,57 @@ export class ScrapeEngine { meta.previewed = (meta.previewed ?? 0) + 1; } else { store.markPreviewError(row.id, "too large"); + skipped++; + this.log(`${meta.id}: preview #${row.id} gif too large and no frame (${row.url.slice(0, 60)})`); } + } else if (outcome.state === "ratelimited") { + // Leave the row pending and hold off: the server asked for a break. + this.registerRateLimit(outcome.retryAfterMs); + this.log(`${meta.id}: preview #${row.id} rate limited (${outcome.error ?? "429"})`); } else if (outcome.state === "failed") { // Transient network failure: leave the row pending for the next pass. - this.log(`${meta.id}: preview ${row.id} failed: ${outcome.error ?? "unknown"}`); + failed++; + this.log( + `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} failed (transient): ` + + `${outcome.error ?? "unknown"} (${row.url.slice(0, 60)})` + ); } else { store.markPreviewError(row.id, outcome.error ?? outcome.state); + skipped++; + this.log( + `${meta.id}: preview #${row.id} ${row.kind}/${row.provider ?? "?"} skipped: ` + + `${outcome.error ?? outcome.state} (${row.url.slice(0, 60)})` + ); } this.persist(dir, meta); await this.sleep(previewGapMs(options)); await afterRow(); } + + const remaining = store.pendingPreviews(5000, true).length; + if (pending.length) { + this.log( + `${meta.id}: preview pass done: ${cached} cached, ${skipped} skipped, ${failed} failed, ` + + `${remaining} still pending` + ); + } + } + + /** A rate limit was seen: hold all walk and preview traffic for a while. */ + private registerRateLimit(retryAfterMs?: number): void { + const pause = Math.max(this.options.rateLimitPauseMs ?? 60_000, retryAfterMs ?? 0); + const until = this.now() + pause; + + if (until > this.rateLimitUntil) { + this.rateLimitUntil = until; + this.log(`rate limit detected; pausing media traffic until ${new Date(until).toISOString()}`); + } + } + + private async respectRateLimit(): Promise { + const wait = this.rateLimitUntil - this.now(); + if (wait > 0) await this.sleep(wait); } private async fetchWithBackoff(meta: ScrapeMeta, backend: ScrapeBackend): Promise { @@ -530,6 +598,9 @@ export class ScrapeEngine { return await backend.fetchPage(meta.channelId, meta.cursor, pageSize); } catch (error) { meta.errors++; + if (isRateLimited(error)) { + this.registerRateLimit((error as { retryAfterMs?: number }).retryAfterMs); + } if (attempt >= maxRetries) throw error; const retryAfter = (error as { retryAfterMs?: number }).retryAfterMs; diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 210c67a..3692d31 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -135,6 +135,7 @@ export interface MediaRow { file_hash: string | null; preview_hash: string | null; preview_ext: string | null; + anim_hash: string | null; file_bytes: number | null; preview_bytes: number | null; file_path: string | null; @@ -173,6 +174,7 @@ CREATE TABLE IF NOT EXISTS media ( file_hash TEXT, preview_hash TEXT, preview_ext TEXT, + anim_hash TEXT, file_bytes INTEGER, preview_bytes INTEGER, file_path TEXT, @@ -217,6 +219,7 @@ export class ScrapeStore { if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT"); if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT"); + if (!names.has("anim_hash")) this.db.exec("ALTER TABLE media ADD COLUMN anim_hash TEXT"); if (!names.has("play_url")) this.db.exec("ALTER TABLE media ADD COLUMN play_url TEXT"); if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL"); } @@ -355,7 +358,16 @@ export class ScrapeStore { .all(limit) as unknown as MediaRow[]; } - markPreview(id: number, preview: { hash: string; bytes: number; ext?: string }): void { + markPreview(id: number, preview: { hash: string; bytes: number; ext?: string; animHash?: string }): void { + if (preview.animHash) { + this.db + .prepare( + "UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, anim_hash = ?, preview_error = NULL WHERE id = ?" + ) + .run(preview.hash, preview.bytes, preview.ext ?? "webp", preview.animHash, id); + return; + } + this.db .prepare( "UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?" @@ -385,6 +397,31 @@ export class ScrapeStore { this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id); } + /** Rows whose preview failed, with the reason, newest first. */ + previewFailures( + limit = 50 + ): Array<{ id: number; kind: MediaKind; provider: string | null; url: string; error: string }> { + const rows = this.db + .prepare( + "SELECT id, kind, provider, url, preview_error FROM media WHERE preview_error IS NOT NULL ORDER BY id DESC LIMIT ?" + ) + .all(limit) as unknown as Array<{ + id: number; + kind: MediaKind; + provider: string | null; + url: string; + preview_error: string; + }>; + + return rows.map(row => ({ + id: row.id, + kind: row.kind, + provider: row.provider, + url: row.url, + error: row.preview_error + })); + } + /** The message a media row came from, for URL refreshes. */ sourceMessageId(mediaId: number): string | null { const row = this.db diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 702e06f..56b7a26 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -877,6 +877,79 @@ void (async () => { assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true); }); + await test("markPreview stores the animated gif reference", () => { + const dir = join(root, "anim-store"); + mkdirSync(dir, { recursive: true }); + + const store = new ScrapeStore(dir); + store.insertPage([ + message("1200", { attachments: [attachment({ filename: "loop.gif" })] }) + ]); + + const row = store.queryMedia({ kind: "gif" })[0]; + store.markPreview(row.id, { hash: "static-hash", bytes: 10, ext: "webp", animHash: "anim-hash" }); + + const updated = store.getMedia(row.id); + assert.equal(updated?.preview_hash, "static-hash"); + assert.equal(updated?.anim_hash, "anim-hash"); + store.close(); + }); + + await test("rate limits pause media traffic", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("1300", { + attachments: [ + attachment({ filename: "a.png", contentType: "image/png" }), + attachment({ filename: "b.png", contentType: "image/png" }) + ] + }) + ] + } + ]); + + const sleeps: number[] = []; + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 10, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async (ms: number) => { + sleeps.push(ms); + }, + random: () => 0, + log: () => {}, + previews: true, + rateLimitPauseMs: 30_000, + download: { + fetchImpl: async () => + new Response("slow down", { status: 429, headers: { "retry-after": "10" } }), + minGapMs: 0, + maxGapMs: 0, + previewMinGapMs: 0, + previewMaxGapMs: 0, + previewBatchSize: 100, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "160", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + const store = new ScrapeStore(join(root, started.id)); + for (const row of store.queryMedia({ kind: "image" })) { + assert.equal(row.preview_hash, null); + assert.equal(row.preview_error, null, "rate limited rows stay pending"); + } + store.close(); + + assert.ok( + sleeps.some(ms => ms > 25_000 && ms <= 30_000), + "the engine paused after the rate limit" + ); + }); + await test("previews run in cycles during the walk", async () => { const events: string[] = []; const pages: Array<{ messages: CoreMessage[] }> = [ diff --git a/poc/public/index.html b/poc/public/index.html index c36d80e..8bbc9ef 100644 --- a/poc/public/index.html +++ b/poc/public/index.html @@ -463,7 +463,8 @@ - + +
diff --git a/poc/public/media.js b/poc/public/media.js index 9e14b44..394026c 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -443,13 +443,6 @@ media.replaceChildren(node); } - /** Only real video files can be framed; embed pages (YouTube etc.) never are. */ - function canFrameVideo(item) { - const direct = /\.(mp4|webm|m4v)(\?|$)/i; - if (item.playUrl && direct.test(item.playUrl)) return true; - return item.source !== "embed" && direct.test(item.url ?? ""); - } - function playIcon(item) { const play = el("span", "vk-media-play", "▶"); play.title = "Play"; @@ -472,11 +465,8 @@ const media = el("div", "vk-media-media"); const imageSource = item.localUrl || item.thumbUrl; const fallbackSource = item.source !== "embed" ? item.url : null; - const streamPath = `/api/media/${item.scrapeId}/${item.itemId}/stream`; - const animatedGif = - item.kind === "gif" && item.remotePreviewUrl && /\.gif(\?|$)/i.test(item.remotePreviewUrl) - ? item.remotePreviewUrl - : null; + // Hover plays the cached animated GIF from the archive, never the provider. + const animatedGif = item.kind === "gif" && item.animUrl ? item.animUrl : null; if (imageSource) { const img = document.createElement("img"); @@ -496,23 +486,6 @@ } media.append(img); - } else if (item.kind === "video" && canFrameVideo(item)) { - // No thumbnail: take the first frame through the same-origin stream endpoint, - // which refreshes expired URLs and avoids cross-origin blocking. - const video = document.createElement("video"); - video.preload = "metadata"; - video.muted = true; - video.playsInline = true; - video.src = streamPath; - video.addEventListener( - "loadedmetadata", - () => { - if (video.duration) video.currentTime = Math.min(0.1, video.duration / 2); - }, - { once: true } - ); - video.addEventListener("error", () => placeholder(media, item), { once: true }); - media.append(video); } else if (fallbackSource && /^(https?:|\/)/.test(fallbackSource)) { const img = document.createElement("img"); img.loading = "lazy"; @@ -1231,22 +1204,6 @@ { once: true } ); media.append(video); - } else if (item.kind === "video" && canFrameVideo(item)) { - // No still available: show the first frame through the stream endpoint. - const video = document.createElement("video"); - video.preload = "metadata"; - video.muted = true; - video.playsInline = true; - video.src = streamUrl; - video.addEventListener( - "loadedmetadata", - () => { - if (video.duration) video.currentTime = Math.min(0.1, video.duration / 2); - }, - { once: true } - ); - media.append(video); - media.append(viewerPlayIcon(item)); } else if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) { const img = document.createElement("img"); img.src = source; @@ -1254,6 +1211,7 @@ media.append(img); } else { media.append(el("span", "vk-media-card-kind", item.kind.toUpperCase())); + if (item.kind === "video") media.append(viewerPlayIcon(item)); } meta.replaceChildren(); @@ -1490,6 +1448,17 @@ document.querySelector("#media-clean")?.addEventListener("click", openCleanModal); + document.querySelector("#daemon-restart")?.addEventListener("click", async () => { + if (!window.confirm("Restart the daemon? The page will disconnect for a few seconds.")) return; + + try { + await window.VK.api.post("/api/daemon/restart", {}); + } catch { + // The daemon exits mid-response on purpose. + } + setStatus("daemon restarting; reload the page in a few seconds"); + }); + window.addEventListener("resize", () => { if (state.view) layoutTop(); }); diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index b1f920b..60abeba 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -9,6 +9,7 @@ * VESKSLOP_SOCKET=/path/to/vesktop-puppet.sock MCP_PORT=8788 pnpm control:bridge */ +import { spawn } from "node:child_process"; import { config as loadEnv } from "dotenv"; import { createMcpExpressApp } from "@modelcontextprotocol/sdk/server/express.js"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; @@ -286,6 +287,7 @@ const webui = startWebUi({ transcripts, preferences, media, + restartDaemon, connectAudioDevices: () => autoConnectAudio(requireCore()) }); @@ -1004,11 +1006,38 @@ app.listen(PORT, HOST, (error?: Error) => { if (stale) console.warn(`[daemon] ${stale}`); }); -function shutdown() { +function shutdown(keepCore = false) { stopSpeakingPoll(); core?.close(); - void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0)); + + const tasks: Array> = [audio.dispose()]; + if (!keepCore) tasks.push(coreProcess.dispose(4000)); + + void Promise.allSettled(tasks).finally(() => process.exit(0)); } -process.on("SIGINT", shutdown); -process.on("SIGTERM", shutdown); +/** Replace this daemon with a fresh copy, leaving the client running. */ +let restartingDaemon = false; + +function restartDaemon(): void { + if (restartingDaemon) return; + restartingDaemon = true; + + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + const args = [...process.execArgv, ...process.argv.slice(1)]; + const command = `${quote(process.execPath)} ${args.map(quote).join(" ")}`; + + console.log("[daemon] restart requested: spawning a replacement"); + const child = spawn("sh", ["-c", `sleep 2; exec ${command}`], { + detached: true, + stdio: "ignore", + cwd: process.cwd(), + env: { ...process.env, VESKSLOP_DAEMON_RESTART: "1" } + }); + child.unref(); + + setTimeout(() => shutdown(true), 300); +} + +process.on("SIGINT", () => shutdown()); +process.on("SIGTERM", () => shutdown()); diff --git a/poc/src/media.ts b/poc/src/media.ts index d453cb2..827bc79 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -123,6 +123,8 @@ export interface MediaItemResult { remotePreviewUrl: string | null; /** Why no preview exists, when a fetch failed (for example "http 404"). */ previewError: string | null; + /** Local animated GIF for hover playback, when the original was cached. */ + animUrl: string | null; localUrl: string | null; thumbUrl: string | null; provider: string | null; @@ -175,6 +177,7 @@ export class MediaService { log: line => this.deps.log?.(line), onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`), walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5), + rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024), @@ -559,6 +562,17 @@ export class MediaService { }; } + /** Absolute path for a cached animated GIF, or null. */ + getMediaAnim(scrapeId: string, itemId: number): { path: string; contentType: string } | null { + const item = this.getMediaItem(scrapeId, itemId); + if (!item?.anim_hash) return null; + + return { + path: join(this.rootDir, scrapeId, "thumbs", `${item.anim_hash}.gif`), + contentType: "image/gif" + }; + } + /** Absolute path for a cached preview, or null. */ getMediaThumb(scrapeId: string, itemId: number): { path: string; contentType: string } | null { const item = this.getMediaItem(scrapeId, itemId); @@ -653,6 +667,42 @@ export class MediaService { return totals; } + /** Preview failures across scrapes with their reasons, newest scrape first. */ + previewFailures( + limit = 50 + ): Array<{ + scrapeId: string; + itemId: number; + kind: MediaKind; + provider: string | null; + url: string; + error: string; + }> { + const results: Array<{ + scrapeId: string; + itemId: number; + kind: MediaKind; + provider: string | null; + url: string; + error: string; + }> = []; + + for (const meta of this.engine.list()) { + if (results.length >= limit) break; + + const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); + try { + for (const row of store.previewFailures(limit - results.length)) { + results.push({ scrapeId: meta.id, itemId: row.id, ...row }); + } + } finally { + store.close(); + } + } + + return results; + } + /** Distinct posters across every scrape, biggest first. */ listAuthors(): Array<{ id: string | null; name: string | null; count: number }> { const totals = new Map(); @@ -801,6 +851,7 @@ export class MediaService { playUrl: row.play_url, remotePreviewUrl: row.preview_url, previewError: row.preview_error, + animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null, localUrl: row.file_path && !meta.downloadDir ? `/api/media/${meta.id}/${row.id}/file` diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 16dddec..b9772cd 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -75,6 +75,8 @@ export interface WebUiDeps { transcripts: TranscriptStore; preferences: PreferencesStore; media: MediaService; + /** Replace the daemon process with a fresh copy, keeping the client running. */ + restartDaemon: () => void; connectAudioDevices: () => Promise<{ connected: boolean; message: string }>; } @@ -267,6 +269,22 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.post( + "/api/daemon/restart", + wrap(async (_req, res) => { + res.json({ ok: true, result: { restarting: true } }); + deps.restartDaemon(); + }) + ); + + app.get( + "/api/media/preview-failures", + wrap(async (req, res) => { + const limit = clampLimit(req.query.limit, 50, 500); + res.json({ ok: true, result: deps.media.previewFailures(limit) }); + }) + ); + app.post( "/api/core/restart", wrap(async (req, res) => { @@ -764,6 +782,15 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/media/:scrapeId/:itemId/anim", + wrap(async (req, res) => { + const file = deps.media.getMediaAnim(String(req.params.scrapeId), Number(req.params.itemId)); + if (!file || !existsSync(file.path)) throw new HttpError(404, "anim not found"); + res.type(file.contentType).sendFile(file.path); + }) + ); + app.get( "/api/media/:scrapeId/:itemId/file", wrap(async (req, res) => {