From 0ba51e06bad191806253e17b355f465c9d91f7e1 Mon Sep 17 00:00:00 2001 From: Fisto Date: Thu, 17 Sep 2026 00:20:00 +0200 Subject: [PATCH] Cycle previews through the walk --- CURRENT.md | 5 ++++ docs/daemon.md | 1 + media-scrape/src/engine.ts | 21 +++++++++++-- media-scrape/test/run.ts | 61 ++++++++++++++++++++++++++++++++++++++ poc/src/media.ts | 1 + 5 files changed, 86 insertions(+), 3 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 9012481..13c04c8 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,11 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C35 Walk/preview cycling. A live scrape now alternates: walk 5 pages, preview up to one + batch (25, with the cooldown), walk on. Tiles fill in during a long walk instead of only at the + end, and preview traffic is spread through the walk. Configurable with + `VESKSLOP_MEDIA_WALK_CYCLE_PAGES` (0 defers previews to the end). The final pass still drains + whatever remains. - [x] C34 Resume from cursor. A paused scrape can continue where it stopped instead of re-walking the top: `POST /api/media/scrape/resume`, MCP `resume_media_scrape`, and a Resume button in History. Previews never needed the walk — backfill fetches them straight from the diff --git a/docs/daemon.md b/docs/daemon.md index 198b069..df7e787 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -180,6 +180,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. | | `VESKSLOP_MEDIA_PREVIEW_BATCH` | `25` | Previews per batch before the cooldown. | | `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS` | `60000` | Cooldown between preview batches. `0` disables batching. | +| `VESKSLOP_MEDIA_WALK_CYCLE_PAGES` | `5` | Walk pages between preview cycles on a live scrape. `0` defers previews to the end. | | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 396d4c9..750fac9 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -60,8 +60,10 @@ export interface ScrapeEngineOptions { log?: (line: string) => void; /** Download settings for `mode: "download"`. */ download?: DownloadOptions; - /** Cache Discord-hosted image previews after every walk. Default true. */ + /** Cache previews after every walk. Default true. */ previews?: boolean; + /** Walk pages between preview cycles. 0 defers all previews to the end. Default 5. */ + walkCyclePages?: number; /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */ probe?: ( path: string @@ -238,6 +240,8 @@ export class ScrapeEngine { const { meta, dir, backend } = job; const store = new ScrapeStore(dir); const pageSize = this.options.pageSize ?? 100; + const perCycle = this.options.download?.previewBatchSize ?? 25; + let pagesSincePreview = 0; try { if (this.stopRequested.has(meta.id)) { @@ -286,6 +290,17 @@ export class ScrapeEngine { this.pageTimes.push(this.now()); this.persist(dir, meta); + // Cycle between walking and previewing so tiles fill in during a long walk + // instead of only at the end. + const cycle = this.options.walkCyclePages ?? 5; + if (cycle > 0 && this.options.previews !== false) { + pagesSincePreview++; + if (pagesSincePreview >= cycle) { + pagesSincePreview = 0; + await this.runPreviews(store, dir, meta, perCycle); + } + } + if (kept.length < page.messages.length || page.messages.length < pageSize) { meta.state = "done"; break; @@ -390,13 +405,13 @@ export class ScrapeEngine { } /** Cache small previews. Embed thumbnails are cached in both modes. */ - private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise { + private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta, limit = 0): Promise { const options = { ...(this.options.download ?? {}), embedPreviews: true, destinationDir: meta.downloadDir ?? null }; - const pending = store.pendingPreviews(5000, true); + const pending = store.pendingPreviews(limit > 0 ? limit : 5000, true); const batchSize = Math.max(1, options.previewBatchSize ?? 25); const batchPauseMs = Math.max(0, options.previewBatchPauseMs ?? 60_000); let batch = 0; diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 1ff3b3e..702e06f 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -877,6 +877,67 @@ void (async () => { assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true); }); + await test("previews run in cycles during the walk", async () => { + const events: string[] = []; + const pages: Array<{ messages: CoreMessage[] }> = [ + { + messages: [ + message("1000", { + attachments: [attachment({ filename: "a.png", contentType: "image/png" })] + }) + ] + }, + { + messages: [ + message("999", { + attachments: [attachment({ filename: "b.png", contentType: "image/png" })] + }) + ] + } + ]; + + const backend: ScrapeBackend = { + name: "puppet", + async fetchPage() { + events.push("walk"); + return pages.shift() ?? { messages: [] }; + } + }; + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 1, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: true, + walkCyclePages: 1, + download: { + fetchImpl: async () => { + events.push("preview"); + return new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }); + }, + minGapMs: 0, + maxGapMs: 0, + previewMinGapMs: 0, + previewMaxGapMs: 0, + previewBatchSize: 100, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "150", guildId: null, mode: "links", backend }); + await engine.processQueue(); + + assert.equal(engine.get(started.id)?.previewed, 2); + const firstPreview = events.indexOf("preview"); + const secondWalk = events.indexOf("walk", 1); + assert.ok(firstPreview !== -1, "a preview happened during the walk"); + assert.ok(secondWalk > firstPreview, "the walk continued after previews"); + }); + await test("engine resumes a paused scrape from its cursor", async () => { const first = new FakeBackend([ { messages: [message("500"), message("499")] }, diff --git a/poc/src/media.ts b/poc/src/media.ts index 386b466..d453cb2 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -174,6 +174,7 @@ export class MediaService { rootDir: this.rootDir, log: line => this.deps.log?.(line), onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`), + walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5), download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),