From 96fe17b4c9c1f4f90c29621a71b9f348ff2681f4 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 14:02:11 +0200 Subject: [PATCH] Backfill previews for paused scrapes and prefer richer dedupe rows --- docs/daemon.md | 1 + docs/media-scraper.md | 6 +- media-scrape/src/dedupe.ts | 19 ++++ media-scrape/src/download.ts | 11 ++ media-scrape/src/engine.ts | 35 +++++- media-scrape/src/index.ts | 1 + media-scrape/test/run.ts | 85 +++++++++++++++ poc/src/media.ts | 200 +++++++++++++++++++---------------- 8 files changed, 259 insertions(+), 99 deletions(-) create mode 100644 media-scrape/src/dedupe.ts diff --git a/docs/daemon.md b/docs/daemon.md index 40dc3c2..f69c607 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -176,6 +176,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_MEDIA_DB_DIR` | `/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. | | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. | | `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. | +| `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. | | `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index cb222a6..2874251 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -133,7 +133,11 @@ Previews run after every walk: (`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the counter is `previewed`. Previews count toward the storage limit. - The job stays `running` through the download and preview passes, so a client never sees `done` - on a half-archived scrape. + on a half-archived scrape. A paused walk still gets these passes for the pages it collected. +- At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their + previews were cached, so a stopped scrape does not leave blank tiles. +- Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than + file downloads. ## Job states diff --git a/media-scrape/src/dedupe.ts b/media-scrape/src/dedupe.ts new file mode 100644 index 0000000..ffbb1f7 --- /dev/null +++ b/media-scrape/src/dedupe.ts @@ -0,0 +1,19 @@ +/* + * Cross-scrape dedupe preference: among rows for the same URL, the richest wins. + * Newer scrapes are seen first; a candidate replaces the current one only when it + * carries something the current one lacks. + */ + +export interface DedupeCandidate { + downloaded: boolean; + hasPreview: boolean; +} + +export function candidateScore(candidate: DedupeCandidate): number { + return (candidate.downloaded ? 2 : 0) + (candidate.hasPreview ? 1 : 0); +} + +/** True when `candidate` should replace `current` for the same URL. */ +export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean { + return candidateScore(candidate) > candidateScore(current); +} diff --git a/media-scrape/src/download.ts b/media-scrape/src/download.ts index 9838eaa..31297d5 100644 --- a/media-scrape/src/download.ts +++ b/media-scrape/src/download.ts @@ -62,6 +62,9 @@ export interface DownloadOptions { maxRetries?: number; /** Preview size cap. Default 5 MiB. */ previewMaxBytes?: number; + /** Preview gap. Default 200-600 ms, much faster than file downloads. */ + previewMinGapMs?: number; + previewMaxGapMs?: number; /** Cache third-party embed thumbnails. */ embedPreviews?: boolean; /** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */ @@ -351,3 +354,11 @@ export function downloadGapMs(options: DownloadOptions = {}): number { const random = options.random ?? Math.random; return min + Math.floor(random() * (max - min + 1)); } + +/** The gap before the next preview, in milliseconds. Previews are small and quick. */ +export function previewGapMs(options: DownloadOptions = {}): number { + const min = options.previewMinGapMs ?? 200; + const max = Math.max(min, options.previewMaxGapMs ?? 600); + const random = options.random ?? Math.random; + return min + Math.floor(random() * (max - min + 1)); +} diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 3ff1370..f8dc839 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -10,6 +10,7 @@ import { downloadOne, downloadPreview, gifPoster, + previewGapMs, probeDimensions, videoPoster, type DownloadOptions @@ -180,6 +181,24 @@ export class ScrapeEngine { this.stopRequested.add(id); } + /** Cache previews for rows a finished or paused scrape never reached. Returns the count. */ + async backfillPreviews(scrapeId: string): Promise { + const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId)); + if (!meta || meta.state === "running" || meta.state === "queued") return 0; + + const dir = join(this.options.rootDir, scrapeId); + const store = new ScrapeStore(dir); + const before = meta.previewed ?? 0; + + try { + await this.runPreviews(store, dir, meta); + } finally { + store.close(); + } + + return (meta.previewed ?? 0) - before; + } + /** Persist a state change made outside a job, such as retention marking. */ setState(id: string, state: ScrapeState): ScrapeMeta | null { const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id)); @@ -258,9 +277,15 @@ export class ScrapeEngine { await this.sleep(this.gapMs()); } - if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) { + const wasPaused = meta.state === "paused"; + + if ( + (meta.state === "done" || wasPaused) && + (meta.mode === "download" || this.options.previews !== false) + ) { // Keep the job running until post-processing finishes, so a client never - // treats a half-archived scrape as complete. + // treats a half-archived scrape as complete. A paused walk still caches + // previews and downloads for the pages it did collect. meta.state = "running"; this.persist(dir, meta); @@ -273,7 +298,7 @@ export class ScrapeEngine { } if (meta.state === "running") { - meta.state = "done"; + meta.state = wasPaused ? "paused" : "done"; } this.persist(dir, meta); } @@ -370,7 +395,7 @@ export class ScrapeEngine { if (!resolved) { store.markPreviewError(row.id, "no preview source"); this.persist(dir, meta); - await this.sleep(downloadGapMs(options)); + await this.sleep(previewGapMs(options)); continue; } previewRow = { ...row, preview_url: resolved }; @@ -409,7 +434,7 @@ export class ScrapeEngine { } this.persist(dir, meta); - await this.sleep(downloadGapMs(options)); + await this.sleep(previewGapMs(options)); } } diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index 25c7bf6..94710f0 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -4,6 +4,7 @@ */ export { extractMedia, type ExtractedMedia } from "./extract.js"; +export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js"; export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js"; diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index c273a2f..928ffe1 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -14,6 +14,7 @@ import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; +import { betterCandidate } from "../src/dedupe.js"; import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; @@ -861,6 +862,90 @@ void (async () => { assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true); }); + await test("backfillPreviews fills rows left by a preview-less walk", async () => { + const backend = new FakeBackend([ + { + messages: [ + message("800", { + attachments: [attachment({ filename: "later.png", contentType: "image/png" })], + embeds: [ + { + type: "video", + url: "https://www.youtube.com/watch?v=x", + title: null, + provider: "YouTube", + videoUrl: "https://www.youtube.com/embed/x", + imageUrl: null, + thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg" + } + ] + }) + ] + } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 2, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: false, + download: { + fetchImpl: async () => + new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }), + minGapMs: 0, + maxGapMs: 0, + sleep: async () => {} + } + }); + + const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend }); + await engine.processQueue(); + assert.equal(engine.get(started.id)?.previewed, undefined); + + const added = await engine.backfillPreviews(started.id); + assert.equal(added, 2); + + const store = new ScrapeStore(join(root, started.id)); + assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash); + assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash); + store.close(); + }); + + await test("dedupe prefers the richer row for a URL", () => { + assert.equal( + betterCandidate( + { downloaded: false, hasPreview: false }, + { downloaded: false, hasPreview: true } + ), + true + ); + assert.equal( + betterCandidate( + { downloaded: false, hasPreview: true }, + { downloaded: false, hasPreview: false } + ), + false + ); + assert.equal( + betterCandidate( + { downloaded: true, hasPreview: false }, + { downloaded: false, hasPreview: true } + ), + false + ); + assert.equal( + betterCandidate( + { downloaded: false, hasPreview: true }, + { downloaded: true, hasPreview: false } + ), + true + ); + }); + await test("snowflake helpers round-trip dates", () => { const id = snowflakeFromIso("2026-09-16T00:00:00.000Z"); assert.ok(id); diff --git a/poc/src/media.ts b/poc/src/media.ts index 1067499..4ad5f22 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -8,6 +8,7 @@ import { rmSync } from "node:fs"; import { isAbsolute, join } from "node:path"; import { + betterCandidate, dirSizeBytes, isNearLimit, planRetention, @@ -174,10 +175,28 @@ export class MediaService { download: { maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024), previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024), + previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200), + previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600), minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000), maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000) } }); + + // Preview passes only run when a walk ends; fill in scrapes that paused early. + void this.backfillPreviews(); + } + + private async backfillPreviews(): Promise { + for (const meta of this.engine.list()) { + if (meta.state === "running" || meta.state === "queued") continue; + + try { + const added = await this.engine.backfillPreviews(meta.id); + if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`); + } catch (error) { + this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`); + } + } } /** Log slopbot in on first use. False when it is disabled or the login failed. */ @@ -546,37 +565,8 @@ export class MediaService { after?: string; before?: string; } = {}): Record { - // Deduplicate by URL across scrapes, newest scrape wins, like queryMedia. - const byUrl = new Map(); - - for (const meta of this.engine.list()) { - if (meta.state === "deleted") continue; - if (options.channelId && meta.channelId !== options.channelId) continue; - - const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); - try { - let storeOffset = 0; - - for (;;) { - const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset }); - if (!rows.length) break; - storeOffset += rows.length; - - for (const row of rows) { - if (byUrl.has(row.url)) continue; - byUrl.set(row.url, { - kind: row.kind, - downloaded: row.downloaded === 1, - cached: row.preview_hash !== null - }); - } - - if (rows.length < 500) break; - } - } finally { - store.close(); - } - } + // Same richest-per-URL dedupe as queryMedia. + const candidates = this.collectCandidates(options); const totals: Record< MediaKind, @@ -588,14 +578,14 @@ export class MediaService { audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 } }; - for (const entry of byUrl.values()) { - const bucket = totals[entry.kind]; + for (const { row } of candidates) { + const bucket = totals[row.kind]; bucket.total++; - if (entry.downloaded) { + if (row.downloaded === 1) { bucket.downloaded++; } else { bucket.notDownloaded++; - if (entry.cached) bucket.cached++; + if (row.preview_hash !== null) bucket.cached++; } } @@ -639,27 +629,23 @@ export class MediaService { } } - /** Read media rows across every scrape, newest scrape first. */ - queryMedia( - options: { - kind?: MediaKind; - source?: string; - scrapeId?: string; - channelId?: string; - authorId?: string; - aspect?: string; - resolution?: string; - after?: string; - before?: string; - limit?: number; - offset?: number; - } = {} - ): MediaItemResult[] { - const limit = Math.max(1, Math.min(options.limit ?? 60, 500)); - const offset = Math.max(0, options.offset ?? 0); - const results: MediaItemResult[] = []; - const seen = new Set(); - let skipped = 0; + /** + * One entry per URL across every scrape. Scrapes are visited newest first, so a newer row + * wins by default; an older row replaces it only when it is richer (downloaded, or with a + * cached preview). Without this, a stopped or partial scrape hides usable previews. + */ + private collectCandidates(options: { + kind?: MediaKind; + source?: string; + scrapeId?: string; + channelId?: string; + authorId?: string; + aspect?: string; + resolution?: string; + after?: string; + before?: string; + }): Array<{ meta: ScrapeMeta; row: MediaRow }> { + const candidates = new Map(); for (const meta of this.engine.list()) { if (meta.state === "deleted") continue; @@ -686,53 +672,81 @@ export class MediaService { storeOffset += rows.length; for (const row of rows) { - // One card per URL, newest scrape wins. - if (seen.has(row.url)) continue; - seen.add(row.url); - - if (skipped < offset) { - skipped++; + const current = candidates.get(row.url); + if (!current) { + candidates.set(row.url, { meta, row }); continue; } - if (results.length >= limit) break; - results.push({ - scrapeId: meta.id, - itemId: row.id, - channelId: meta.channelId, - kind: row.kind, - source: row.source, - authorId: row.author_id, - authorName: row.author_name, - postedAt: row.posted_at, - width: row.width, - height: row.height, - aspectRatio: row.aspect_ratio, - contentType: row.content_type, - durationSecs: row.duration_secs, - url: row.url, - playUrl: row.play_url, - remotePreviewUrl: row.preview_url, - localUrl: - row.file_path && !meta.downloadDir - ? `/api/media/${meta.id}/${row.id}/file` - : null, - thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, - provider: row.provider, - downloaded: row.downloaded === 1, - createdAt: row.created_at - }); + if ( + betterCandidate( + { + downloaded: current.row.downloaded === 1, + hasPreview: current.row.preview_hash !== null + }, + { downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null } + ) + ) { + candidates.set(row.url, { meta, row }); + } } - if (results.length >= limit || rows.length < 500) break; + if (rows.length < 500) break; } } finally { store.close(); } - - if (results.length >= limit) break; } - return results; + return [...candidates.values()]; + } + + /** Read media rows across every scrape, richest per URL, newest scrape first. */ + queryMedia( + options: { + kind?: MediaKind; + source?: string; + scrapeId?: string; + channelId?: string; + authorId?: string; + aspect?: string; + resolution?: string; + after?: string; + before?: string; + limit?: number; + offset?: number; + } = {} + ): MediaItemResult[] { + const limit = Math.max(1, Math.min(options.limit ?? 60, 500)); + const offset = Math.max(0, options.offset ?? 0); + + return this.collectCandidates(options) + .slice(offset, offset + limit) + .map(({ meta, row }) => ({ + scrapeId: meta.id, + itemId: row.id, + channelId: meta.channelId, + kind: row.kind, + source: row.source, + authorId: row.author_id, + authorName: row.author_name, + postedAt: row.posted_at, + width: row.width, + height: row.height, + aspectRatio: row.aspect_ratio, + contentType: row.content_type, + durationSecs: row.duration_secs, + url: row.url, + playUrl: row.play_url, + remotePreviewUrl: row.preview_url, + localUrl: + row.file_path && !meta.downloadDir + ? `/api/media/${meta.id}/${row.id}/file` + : null, + thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, + provider: row.provider, + downloaded: row.downloaded === 1, + createdAt: row.created_at + })); } }