zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backfill previews for paused scrapes and prefer richer dedupe rows

Browse Source
This commit is contained in:
Fisto
2026-09-16 14:02:11 +02:00
parent 2309393e3c
commit 96fe17b4c9
8 changed files with 259 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/daemon.md
+1
View File
@@ -176,6 +176,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
| `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. |
| `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
| `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
| `VESKSLOP_MEDIA_PREVIEW_MIN_MS` / `_MAX_MS` | `200` / `600` | Gap between preview fetches. |
| `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
| `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
| `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |
docs/media-scraper.md
+5 -1
View File
@@ -133,7 +133,11 @@ Previews run after every walk:
(`preview_ext`, default webp); `preview_hash` and `preview_bytes` land on the row, and the
counter is `previewed`. Previews count toward the storage limit.
- The job stays `running` through the download and preview passes, so a client never sees `done`
on a half-archived scrape.
on a half-archived scrape. A paused walk still gets these passes for the pages it collected.
- At daemon start, `backfillPreviews` runs the same pass over scrapes that paused before their
previews were cached, so a stopped scrape does not leave blank tiles.
- Preview gaps default to 200-600 ms (`VESKSLOP_MEDIA_PREVIEW_MIN_MS`/`_MAX_MS`), much faster than
file downloads.
## Job states
media-scrape/src/dedupe.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Cross-scrape dedupe preference: among rows for the same URL, the richest wins.
* Newer scrapes are seen first; a candidate replaces the current one only when it
* carries something the current one lacks.
*/
export interface DedupeCandidate {
downloaded: boolean;
hasPreview: boolean;
}
export function candidateScore(candidate: DedupeCandidate): number {
return (candidate.downloaded ? 2 : 0) + (candidate.hasPreview ? 1 : 0);
}
/** True when `candidate` should replace `current` for the same URL. */
export function betterCandidate(current: DedupeCandidate, candidate: DedupeCandidate): boolean {
return candidateScore(candidate) > candidateScore(current);
}
media-scrape/src/download.ts
+11
View File
@@ -62,6 +62,9 @@ export interface DownloadOptions {
maxRetries?: number;
/** Preview size cap. Default 5 MiB. */
previewMaxBytes?: number;
/** Preview gap. Default 200-600 ms, much faster than file downloads. */
previewMinGapMs?: number;
previewMaxGapMs?: number;
/** Cache third-party embed thumbnails. */
embedPreviews?: boolean;
/** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
@@ -351,3 +354,11 @@ export function downloadGapMs(options: DownloadOptions = {}): number {
const random = options.random ?? Math.random;
return min + Math.floor(random() * (max - min + 1));
}
/** The gap before the next preview, in milliseconds. Previews are small and quick. */
export function previewGapMs(options: DownloadOptions = {}): number {
const min = options.previewMinGapMs ?? 200;
const max = Math.max(min, options.previewMaxGapMs ?? 600);
const random = options.random ?? Math.random;
return min + Math.floor(random() * (max - min + 1));
}
media-scrape/src/engine.ts
+30 -5
View File
@@ -10,6 +10,7 @@ import {
downloadOne,
downloadPreview,
gifPoster,
previewGapMs,
probeDimensions,
videoPoster,
type DownloadOptions
@@ -180,6 +181,24 @@ export class ScrapeEngine {
this.stopRequested.add(id);
}
/** Cache previews for rows a finished or paused scrape never reached. Returns the count. */
async backfillPreviews(scrapeId: string): Promise<number> {
const meta = this.active.get(scrapeId) ?? readScrapeMeta(join(this.options.rootDir, scrapeId));
if (!meta || meta.state === "running" || meta.state === "queued") return 0;
const dir = join(this.options.rootDir, scrapeId);
const store = new ScrapeStore(dir);
const before = meta.previewed ?? 0;
try {
await this.runPreviews(store, dir, meta);
} finally {
store.close();
}
return (meta.previewed ?? 0) - before;
}
/** Persist a state change made outside a job, such as retention marking. */
setState(id: string, state: ScrapeState): ScrapeMeta | null {
const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
@@ -258,9 +277,15 @@ export class ScrapeEngine {
await this.sleep(this.gapMs());
}
if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
const wasPaused = meta.state === "paused";
if (
(meta.state === "done" || wasPaused) &&
(meta.mode === "download" || this.options.previews !== false)
) {
// Keep the job running until post-processing finishes, so a client never
// treats a half-archived scrape as complete.
// treats a half-archived scrape as complete. A paused walk still caches
// previews and downloads for the pages it did collect.
meta.state = "running";
this.persist(dir, meta);
@@ -273,7 +298,7 @@ export class ScrapeEngine {
}
if (meta.state === "running") {
meta.state = "done";
meta.state = wasPaused ? "paused" : "done";
}
this.persist(dir, meta);
}
@@ -370,7 +395,7 @@ export class ScrapeEngine {
if (!resolved) {
store.markPreviewError(row.id, "no preview source");
this.persist(dir, meta);
await this.sleep(downloadGapMs(options));
await this.sleep(previewGapMs(options));
continue;
}
previewRow = { ...row, preview_url: resolved };
@@ -409,7 +434,7 @@ export class ScrapeEngine {
}
this.persist(dir, meta);
await this.sleep(downloadGapMs(options));
await this.sleep(previewGapMs(options));
}
}
media-scrape/src/index.ts
+1
View File
@@ -4,6 +4,7 @@
*/
export { extractMedia, type ExtractedMedia } from "./extract.js";
export { betterCandidate, candidateScore, type DedupeCandidate } from "./dedupe.js";
export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
media-scrape/test/run.ts
+85
View File
@@ -14,6 +14,7 @@ import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
import { extractMedia } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js";
import { betterCandidate } from "../src/dedupe.js";
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
import { isNearLimit, planRetention } from "../src/retention.js";
import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -861,6 +862,90 @@ void (async () => {
assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
});
await test("backfillPreviews fills rows left by a preview-less walk", async () => {
const backend = new FakeBackend([
{
messages: [
message("800", {
attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
embeds: [
{
type: "video",
url: "https://www.youtube.com/watch?v=x",
title: null,
provider: "YouTube",
videoUrl: "https://www.youtube.com/embed/x",
imageUrl: null,
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
}
]
})
]
}
]);
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
pageMinMs: 0,
pageMaxMs: 0,
sleep: async () => {},
random: () => 0,
log: () => {},
previews: false,
download: {
fetchImpl: async () =>
new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
}
});
const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
await engine.processQueue();
assert.equal(engine.get(started.id)?.previewed, undefined);
const added = await engine.backfillPreviews(started.id);
assert.equal(added, 2);
const store = new ScrapeStore(join(root, started.id));
assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
store.close();
});
await test("dedupe prefers the richer row for a URL", () => {
assert.equal(
betterCandidate(
{ downloaded: false, hasPreview: false },
{ downloaded: false, hasPreview: true }
),
true
);
assert.equal(
betterCandidate(
{ downloaded: false, hasPreview: true },
{ downloaded: false, hasPreview: false }
),
false
);
assert.equal(
betterCandidate(
{ downloaded: true, hasPreview: false },
{ downloaded: false, hasPreview: true }
),
false
);
assert.equal(
betterCandidate(
{ downloaded: false, hasPreview: true },
{ downloaded: true, hasPreview: false }
),
true
);
});
await test("snowflake helpers round-trip dates", () => {
const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
assert.ok(id);
poc/src/media.ts
+82 -68
View File
@@ -8,6 +8,7 @@
import { rmSync } from "node:fs";
import { isAbsolute, join } from "node:path";
import {
betterCandidate,
dirSizeBytes,
isNearLimit,
planRetention,
@@ -174,10 +175,28 @@ export class MediaService {
download: {
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
}
});
// Preview passes only run when a walk ends; fill in scrapes that paused early.
void this.backfillPreviews();
}
private async backfillPreviews(): Promise<void> {
for (const meta of this.engine.list()) {
if (meta.state === "running" || meta.state === "queued") continue;
try {
const added = await this.engine.backfillPreviews(meta.id);
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
} catch (error) {
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
}
}
}
/** Log slopbot in on first use. False when it is disabled or the login failed. */
@@ -546,37 +565,8 @@ export class MediaService {
after?: string;
before?: string;
} = {}): Record<MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number }> {
// Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
const byUrl = new Map<string, { kind: MediaKind; downloaded: boolean; cached: boolean }>();
for (const meta of this.engine.list()) {
if (meta.state === "deleted") continue;
if (options.channelId && meta.channelId !== options.channelId) continue;
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
try {
let storeOffset = 0;
for (;;) {
const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
if (!rows.length) break;
storeOffset += rows.length;
for (const row of rows) {
if (byUrl.has(row.url)) continue;
byUrl.set(row.url, {
kind: row.kind,
downloaded: row.downloaded === 1,
cached: row.preview_hash !== null
});
}
if (rows.length < 500) break;
}
} finally {
store.close();
}
}
// Same richest-per-URL dedupe as queryMedia.
const candidates = this.collectCandidates(options);
const totals: Record<
MediaKind,
@@ -588,14 +578,14 @@ export class MediaService {
audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }
};
for (const entry of byUrl.values()) {
const bucket = totals[entry.kind];
for (const { row } of candidates) {
const bucket = totals[row.kind];
bucket.total++;
if (entry.downloaded) {
if (row.downloaded === 1) {
bucket.downloaded++;
} else {
bucket.notDownloaded++;
if (entry.cached) bucket.cached++;
if (row.preview_hash !== null) bucket.cached++;
}
}
@@ -639,9 +629,12 @@ export class MediaService {
}
}
/** Read media rows across every scrape, newest scrape first. */
queryMedia(
options: {
/**
* One entry per URL across every scrape. Scrapes are visited newest first, so a newer row
* wins by default; an older row replaces it only when it is richer (downloaded, or with a
* cached preview). Without this, a stopped or partial scrape hides usable previews.
*/
private collectCandidates(options: {
kind?: MediaKind;
source?: string;
scrapeId?: string;
@@ -651,15 +644,8 @@ export class MediaService {
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaItemResult[] {
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
const offset = Math.max(0, options.offset ?? 0);
const results: MediaItemResult[] = [];
const seen = new Set<string>();
let skipped = 0;
}): Array<{ meta: ScrapeMeta; row: MediaRow }> {
const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
for (const meta of this.engine.list()) {
if (meta.state === "deleted") continue;
@@ -686,17 +672,57 @@ export class MediaService {
storeOffset += rows.length;
for (const row of rows) {
// One card per URL, newest scrape wins.
if (seen.has(row.url)) continue;
seen.add(row.url);
if (skipped < offset) {
skipped++;
const current = candidates.get(row.url);
if (!current) {
candidates.set(row.url, { meta, row });
continue;
}
if (results.length >= limit) break;
results.push({
if (
betterCandidate(
{
downloaded: current.row.downloaded === 1,
hasPreview: current.row.preview_hash !== null
},
{ downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null }
)
) {
candidates.set(row.url, { meta, row });
}
}
if (rows.length < 500) break;
}
} finally {
store.close();
}
}
return [...candidates.values()];
}
/** Read media rows across every scrape, richest per URL, newest scrape first. */
queryMedia(
options: {
kind?: MediaKind;
source?: string;
scrapeId?: string;
channelId?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaItemResult[] {
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
const offset = Math.max(0, options.offset ?? 0);
return this.collectCandidates(options)
.slice(offset, offset + limit)
.map(({ meta, row }) => ({
scrapeId: meta.id,
itemId: row.id,
channelId: meta.channelId,
@@ -721,18 +747,6 @@ export class MediaService {
provider: row.provider,
downloaded: row.downloaded === 1,
createdAt: row.created_at
});
}
if (results.length >= limit || rows.length < 500) break;
}
} finally {
store.close();
}
if (results.length >= limit) break;
}
return results;
}));
}
}