diff --git a/CURRENT.md b/CURRENT.md index e24777a..9012481 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,11 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C34 Resume from cursor. A paused scrape can continue where it stopped instead of + re-walking the top: `POST /api/media/scrape/resume`, MCP `resume_media_scrape`, and a Resume + button in History. Previews never needed the walk — backfill fetches them straight from the + stored URLs; the walk is only for discovery. Live: the paused `gernal_chut` scrape resumed at + cursor `365256427247239181`, fetched its final page, and entered the preview pass. - [x] C33 Puppet history pagination. The core's `message.fetch` read the client store, which keeps only the newest messages plus the channel's first — every page was 10 messages and the walk stopped after one page (a 10-message, 4-media scrape of an ancient group with thousands diff --git a/docs/daemon.md b/docs/daemon.md index 0c52081..198b069 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -83,6 +83,7 @@ The daemon exposes MCP over Streamable HTTP. The default address is `http://127. | `list_media_scrapes` | Media scrapes with state, pages, messages, and counts. | | `get_media_scrape` | One scrape by ID. | | `stop_media_scrape` | Stop a scrape; it pauses after the current page. | +| `resume_media_scrape` | Continue a paused scrape from its cursor. | | `get_media_storage` | Usage against the limit, per-kind sizes, and the scrape list. | | `query_media` | Media items across scrapes (`kind?`, `source?`, `scrapeId?`, `channelId?`, `limit?`, `offset?`). | | `get_media_item` | One media item by `scrapeId` and `itemId`. | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 172c5e2..9f6baf0 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -39,6 +39,9 @@ Rules: - One running scrape at a time. Further requests queue. - `stop` marks the job and lets the current page finish. +- `resume` continues a paused scrape from its stored cursor, so an interrupted walk does not fetch + the pages it already has again. The History popup has a **Resume** button and the API a + `POST /api/media/scrape/resume` route. - Every page is an outbound read. Sends from commands keep their own pacing. ## Pacing and backoff diff --git a/docs/webui.md b/docs/webui.md index 7d500d7..7227a34 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -272,6 +272,7 @@ replaceable. | GET | `/api/media/errors` | none | Bot and scrape errors, newest first | | POST | `/api/media/errors/clear` | none | Clear the error list | | POST | `/api/media/scrape/stop` | `id` | Stop a media scrape | +| POST | `/api/media/scrape/resume` | `id` | Continue a paused scrape from its cursor | | POST | `/api/media/clean` | `ids?` | Delete the given scrapes, or every scrape when `ids` is absent | | GET | `/api/logs` | `limit` | The persistent action log | | GET | `/api/events` | none | Server-sent events | diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 2acc04c..396d4c9 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -182,6 +182,22 @@ export class ScrapeEngine { this.stopRequested.add(id); } + /** Continue a paused scrape from its stored cursor instead of walking the top again. */ + async resume(scrapeId: string, backend: ScrapeBackend): Promise { + const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId)); + if (!meta) return null; + if (meta.state === "running" || meta.state === "queued") return { ...meta }; + + this.stopRequested.delete(scrapeId); + meta.state = "queued"; + this.active.set(scrapeId, meta); + writeScrapeMeta(join(this.options.rootDir, scrapeId), meta); + + this.queue.push({ meta, dir: join(this.options.rootDir, scrapeId), backend }); + void this.processQueue(); + return { ...meta }; + } + /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */ async backfillPreviews(scrapeId: string): Promise { const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId)); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index c78bee1..1ff3b3e 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -877,6 +877,33 @@ void (async () => { assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true); }); + await test("engine resumes a paused scrape from its cursor", async () => { + const first = new FakeBackend([ + { messages: [message("500"), message("499")] }, + { messages: [message("498")] } + ]); + + const engine = engineFor(root, first); + const started = await engine.start({ channelId: "140", guildId: null, mode: "links", backend: first }); + engine.stop(started.id); + await engine.processQueue(); + + let meta = engine.get(started.id); + assert.equal(meta?.state, "paused"); + assert.equal(meta?.cursor, "499"); + assert.equal(meta?.messages, 2); + + const second = new FakeBackend([{ messages: [message("498")] }]); + await engine.resume(started.id, second); + await engine.processQueue(); + + meta = engine.get(started.id); + assert.equal(meta?.state, "done"); + assert.equal(meta?.messages, 3); + assert.equal(meta?.cursor, "498"); + assert.equal(second.calls[0].before, "499"); + }); + await test("backfillPreviews fills rows left by a preview-less walk", async () => { const backend = new FakeBackend([ { diff --git a/poc/public/media.js b/poc/public/media.js index a01118a..9e14b44 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -1104,6 +1104,20 @@ `${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}` ) ); + if (["paused", "queued"].includes(scrape.state)) { + row.append( + button("Resume", async () => { + try { + await window.VK.api.post("/api/media/scrape/resume", { id: scrape.id }); + } catch (error) { + setStatus(`resume failed: ${error.message}`); + } + void loadScrapes(); + void openHistoryModal(); + }) + ); + } + list.append(row); } diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index 9885da2..b1f920b 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -836,6 +836,15 @@ function buildServer(): McpServer { async ({ scrapeId }) => jsonText(media.getScrape(scrapeId)) ); + server.registerTool( + "resume_media_scrape", + { + description: "Continue a paused media scrape from its cursor.", + inputSchema: { scrapeId: z.string().describe("Scrape ID") } + }, + async ({ scrapeId }) => jsonText(await media.resumeScrape(scrapeId)) + ); + server.registerTool( "stop_media_scrape", { diff --git a/poc/src/media.ts b/poc/src/media.ts index 4c6480b..386b466 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -384,6 +384,28 @@ export class MediaService { this.engine.stop(id); } + /** Continue a paused scrape from its cursor with the same backend family. */ + async resumeScrape(scrapeId: string): Promise { + const meta = this.engine.get(scrapeId); + if (!meta) return null; + + let backend: ScrapeBackend; + if (meta.backend === "slopbot" && meta.guildId) { + const available = await this.ensureSlopbot(); + const inGuild = this.bot?.guilds.cache.has(meta.guildId) ?? false; + const fetcher = this.botFetcher; + if (!available || !inGuild || !fetcher) { + throw new Error("slopbot is not connected to that guild"); + } + backend = new SlopbotBackend(fetcher); + } else { + backend = this.puppetBackend(); + } + + this.deps.log?.(`resuming scrape ${scrapeId} from cursor ${meta.cursor ?? "start"}`); + return this.engine.resume(scrapeId, backend); + } + listScrapes(): ScrapeMeta[] { return this.engine.list(); } diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 8591410..16dddec 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -980,6 +980,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.post( + "/api/media/scrape/resume", + wrap(async (req, res) => { + const id = String(req.body?.id ?? ""); + if (!id) throw new HttpError(400, "id is required"); + + const scrape = await deps.media.resumeScrape(id); + deps.getPuppet()?.appendAction({ type: "media", action: "resume", scrapeId: id }); + res.json({ ok: true, result: scrape }); + }) + ); + app.post( "/api/media/scrape/stop", wrap(async (req, res) => {