From d886d22d89ec6f760398629f37102ef70622963d Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 08:40:28 +0200 Subject: [PATCH] Reconcile stale running scrapes and allow per-scrape delete --- CURRENT.md | 4 ++++ docs/media-scraper.md | 7 +++++-- docs/webui.md | 3 ++- media-scrape/src/engine.ts | 19 +++++++++++++++++++ media-scrape/test/run.ts | 27 ++++++++++++++++++++++++++- poc/public/media.js | 22 ++++++++++++++++++++++ poc/src/media.ts | 5 +++++ 7 files changed, 83 insertions(+), 4 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 360b78c..592d7ba 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,10 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C40 Stale scrape states. Two `general_chut` scrapes showed `running` because the daemons + that owned them were killed mid-job; the state file kept the last written value. On daemon + start the engine now reconciles `running`/`queued` scrapes to `paused`, and History has a + per-scrape **Delete** button (clean with a single ID) next to Resume. Unit-tested. - [x] C39 File-type filter and the 4chan default. The Advanced drawer gained extension chips (video mp4/webm/mov/mkv, image png/jpg/gif/webp/avif, audio mp3/ogg/wav/m4a/flac) matched on URL or content type, plus a default-on **Hide 4chan links (usually dead)** toggle. Backend, diff --git a/docs/media-scraper.md b/docs/media-scraper.md index f7afd91..4753fff 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -40,8 +40,11 @@ Rules: - One running scrape at a time. Further requests queue. - `stop` marks the job and lets the current page finish. - `resume` continues a paused scrape from its stored cursor, so an interrupted walk does not fetch - the pages it already has again. The History popup has a **Resume** button and the API a - `POST /api/media/scrape/resume` route. + the pages it already has again. The History popup has **Resume** and **Delete** buttons and the + API a `POST /api/media/scrape/resume` route. +- On daemon start, scrapes still marked `running` or `queued` on disk are stale (their job died + with the previous daemon). They are reconciled to `paused` so the UI tells the truth and Resume + can continue them. - Every page is an outbound read. Sends from commands keep their own pacing. ## Pacing and backoff diff --git a/docs/webui.md b/docs/webui.md index 180f8fd..ed1a3d0 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -140,7 +140,8 @@ box live in Chat, not Call, and the connection stats are not chips yet. **New scrape…** opens a popup with **Server** and **Group chat** tabs: servers expand into their text channels, each channel (and each group chat) has **Links** and **Download** buttons, and group chats always use the puppet. **History** lists - each scrape with start time, duration, pages, messages, media counts, bytes, files, and previews. + each scrape with start time, duration, pages, messages, media counts, bytes, files, and previews, + and offers **Resume** for paused ones and **Delete** for anything not running. The scrape strip shows live progress with a pulse and **Stop**. Images and GIFs render from the cached preview when one exists, then the local file, then the CDN URL; videos and audio are link cards with the poster and posted time in the tooltip. diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 1e7b8a0..1606a70 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -208,6 +208,25 @@ export class ScrapeEngine { return { ...meta }; } + /** + * A scrape left `running` or `queued` on disk has no live job (the daemon that owned it is + * gone). Pause it so the UI shows the truth and Resume can pick it up. + */ + reconcileStaleJobs(): string[] { + const stale: string[] = []; + + for (const meta of listScrapes(this.options.rootDir)) { + if (meta.state !== "running" && meta.state !== "queued") continue; + + meta.state = "paused"; + meta.updatedAt = new Date(this.now()).toISOString(); + writeScrapeMeta(join(this.options.rootDir, meta.id), meta); + stale.push(meta.id); + } + + return stale; + } + /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */ async backfillPreviews(scrapeId: string): Promise { const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId)); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 56b7a26..bc2fd13 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -25,7 +25,13 @@ import { betterCandidate } from "../src/dedupe.js"; import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; -import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js"; +import { + dirSizeBytes, + listScrapes, + readScrapeMeta, + ScrapeStore, + writeScrapeMeta +} from "../src/store.js"; import type { CoreAttachment, CoreMessage } from "../src/types.js"; let failures = 0; @@ -877,6 +883,25 @@ void (async () => { assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true); }); + await test("stale running scrapes are paused on reconcile", async () => { + const backend = new FakeBackend([{ messages: [message("1400")] }]); + const engine = engineFor(root, backend); + const started = await engine.start({ channelId: "170", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + // Simulate a daemon that died mid-job: the meta still says running. + const dir = join(root, started.id); + const meta = readScrapeMeta(dir); + assert.ok(meta); + meta.state = "running"; + writeScrapeMeta(dir, meta); + + const fresh = new ScrapeEngine({ rootDir: root, previews: false }); + const stale = fresh.reconcileStaleJobs(); + assert.deepEqual(stale, [started.id]); + assert.equal(readScrapeMeta(dir)?.state, "paused"); + }); + await test("markPreview stores the animated gif reference", () => { const dir = join(root, "anim-store"); mkdirSync(dir, { recursive: true }); diff --git a/poc/public/media.js b/poc/public/media.js index c3a7a7c..c121beb 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -1336,6 +1336,28 @@ `${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}` ) ); + if (scrape.state !== "running" && scrape.state !== "queued") { + row.append( + button("Delete", async () => { + if ( + !window.confirm( + `Delete scrape ${scrape.id}? This removes its database, downloads, and previews.` + ) + ) { + return; + } + + try { + await window.VK.api.post("/api/media/clean", { ids: [scrape.id] }); + } catch (error) { + setStatus(`delete failed: ${error.message}`); + } + void loadScrapes(); + void openHistoryModal(); + }) + ); + } + if (["paused", "queued"].includes(scrape.state)) { row.append( button("Resume", async () => { diff --git a/poc/src/media.ts b/poc/src/media.ts index 6087df6..422d9d0 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -190,6 +190,11 @@ export class MediaService { } }); + const stale = this.engine.reconcileStaleJobs(); + if (stale.length) { + this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`); + } + // Preview passes only run when a walk ends; fill in scrapes that paused early. void this.backfillPreviews(); }