From 3bba0519711b672e34153f91d767d62c493a28c9 Mon Sep 17 00:00:00 2001 From: Fisto Date: Mon, 21 Sep 2026 10:40:53 +0200 Subject: [PATCH] Add friendly scrape names and a completeness check --- AGENTS.md | 7 +- CURRENT.md | 9 ++ docs/testing.md | 14 ++++ media-scrape/src/store.ts | 11 +++ poc/package.json | 1 + poc/public/media.js | 33 ++++++-- poc/scripts/verify-scrapes.ts | 152 ++++++++++++++++++++++++++++++++++ 7 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 poc/scripts/verify-scrapes.ts diff --git a/AGENTS.md b/AGENTS.md index 24abc2f..e7f952d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,9 +117,10 @@ pnpm testTypes # tsc --noEmit pnpm lint cd poc -pnpm check # type check the daemon -pnpm test:comm # unit tests -pnpm test:ui # optional browser smoke test for the Media tab +pnpm check # type check the daemon +pnpm test:comm # unit tests +pnpm test:ui # optional browser smoke test for the Media tab +pnpm verify:scrapes # scrape completeness report (exit 1 when a scrape looks truncated) ``` `pnpm test:ui` is optional. It needs a running daemon with media data and a Chromium: set diff --git a/CURRENT.md b/CURRENT.md index 8cb388f..95888a6 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,15 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C59 Scrape names and the completeness check. Scrape listings (History, Clean media, + Bot-tab job/lane cards) now lead with the first two words of the channel or group-chat name + plus the scrape id. New `poc/scripts/verify-scrapes.ts` (`pnpm verify:scrapes`) checks every + index: done state, no lastError, messages non-zero, messages vs the Discord search total, and + the oldest message against the source's snowflake creation time (60-day grace). Scrapes a + later full walk supersedes are reported as `old`, not failures. Result: 10 complete, 4 + superseded, 1 genuinely incomplete — the early DM test scrape + `805255536223715370-20260916215657` (10 messages, pagination stopped before the fix, never + re-scraped). - [x] C58 Titles from Discord's embed cache. Gone YouTube videos have no oEmbed answer, but Discord keeps the embed title (and, with the rebuilt core bundle, the author/channel). The title lane now asks the source message's embed first and only then the provider oEmbed; the diff --git a/docs/testing.md b/docs/testing.md index b9e8162..4ccc0aa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -61,6 +61,20 @@ The mock also covers tiers, continuation, polls, reactions, and the action log. with `set_puppet_config` before a test. Inject a reaction with `TEST_TRIGGER_REACTION=1 TEST_REACTION_MESSAGE_ID=msg-1 TEST_REACTION_EMOJI="1️⃣" pnpm test:trigger`. +## Scrape completeness + +`pnpm verify:scrapes` (from `poc/`) walks every scrape index and reports whether the walk +plausibly reached the start of the source: + +- state is `done`, no `lastError`, and the message count is non-zero +- the message count matches the recorded Discord search total when there is one +- the oldest message is close to the source's creation time, derived from the channel ID + snowflake (`--grace ` sets the allowance, default 60) + +The age check is a heuristic: a channel can sit empty after creation, and a ranged scrape is +expected to stop early. A flagged scrape that a later full walk of the same source supersedes is +reported as `old` rather than a problem. Exit code is 1 when a scrape is genuinely incomplete. + ## Web UI smoke test The web UI starts with the daemon. The mock core is enough for this test. diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index 8b0b754..b2ffbde 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -657,6 +657,17 @@ export class ScrapeStore { .run(preview.hash, preview.bytes, preview.ext ?? "webp", id); } + /** Message span of the walk, for completeness checks. */ + dbSummary(): { messages: number; oldest: string | null; newest: string | null } { + const row = this.db + .prepare( + "SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest FROM messages" + ) + .get() as { messages: number; oldest: string | null; newest: string | null } | undefined; + + return { messages: row?.messages ?? 0, oldest: row?.oldest ?? null, newest: row?.newest ?? null }; + } + /** Preview state for the whole scrape and per kind, for the backfill UI. */ previewCounts(): { total: number; diff --git a/poc/package.json b/poc/package.json index 80abe65..664ef1c 100644 --- a/poc/package.json +++ b/poc/package.json @@ -12,6 +12,7 @@ "control:bridge": "tsx src/control-bridge.ts", "check": "tsc --noEmit", "control:mock": "tsx src/mock-core.ts", + "verify:scrapes": "tsx scripts/verify-scrapes.ts", "test:trigger": "tsx src/test-trigger.ts", "cdp:dispatch": "tsx src/cdp-dispatch.ts", "test:cache": "tsx src/test-cache.ts", diff --git a/poc/public/media.js b/poc/public/media.js index 75b1dee..1396030 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -866,7 +866,7 @@ for (const job of jobs) { const row = el("div", `vk-bot-job vk-bot-job-${job.state}`); const head = el("div", "vk-bot-job-head"); - head.append(el("span", "vk-bot-job-title", channelLabel(job.chatId))); + head.append(el("span", "vk-bot-job-title", scrapeName(job.chatId))); head.append(el("span", `vk-bot-state vk-bot-state-${job.state}`, job.state)); const parts = [`${job.pending} pending`]; @@ -953,7 +953,7 @@ if (status.failed) parts.push(`${status.failed} failed`); if (status.perMinute) parts.push(`${status.perMinute}/min`); if (status.pending > 0) parts.push(`ETA ${formatEta(status.etaMs)}`); - if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`); + if (status.currentScrapeId) parts.push(`on ${scrapeName(status.currentScrapeId)}`); head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 "))); titleLane.append(head); @@ -1005,7 +1005,7 @@ if (status.recovered) parts.push(`${status.recovered} recovered`); if (status.deferred) parts.push(`${status.deferred} deferred`); if (status.gone) parts.push(`${status.gone} unavailable`); - if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`); + if (status.currentScrapeId) parts.push(`on ${scrapeName(status.currentScrapeId)}`); head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 "))); recoverLane.append(head); @@ -2451,6 +2451,20 @@ return state.channels.find(channel => channel.id === channelId)?.label ?? channelId; } + /** Short scrape name: the first two words of the channel or group-chat name. */ + function scrapeName(channelId) { + const label = channelLabel(channelId); + const cleaned = label.includes("/#") + ? label.split("/#")[1] + : label.replace(/\s*\((group DM|DM)\)\s*$/i, ""); + const words = cleaned + .replace(/[_\-]+/g, " ") + .split(/\s+/) + .filter(Boolean) + .slice(0, 2); + return words.join(" ") || label; + } + async function openHistoryModal() { const list = historyModal.querySelector("#history-list"); list.replaceChildren(el("div", null, "loading...")); @@ -2469,7 +2483,9 @@ for (const scrape of scrapes) { const media = scrape.media ?? {}; const row = el("div", "vk-media-history-row"); - row.append(el("strong", null, channelLabel(scrape.channelId))); + row.append( + el("strong", null, `${scrapeName(scrape.channelId)} · ${scrape.id}`) + ); row.append( el( "span", @@ -2894,7 +2910,14 @@ list.replaceChildren(); for (const scrape of scrapes) { - list.append(el("div", null, `${scrape.id} · ${scrape.state} · ${formatBytes(scrape.bytes)}`)); + list.append( + el( + "div", + null, + `${scrapeName(scrape.channelId)} · ${scrape.id} · ${scrape.state} · ` + + formatBytes(scrape.bytes) + ) + ); } const bytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0); diff --git a/poc/scripts/verify-scrapes.ts b/poc/scripts/verify-scrapes.ts new file mode 100644 index 0000000..2614441 --- /dev/null +++ b/poc/scripts/verify-scrapes.ts @@ -0,0 +1,152 @@ +/* + * Scrape completeness check. + * + * Reads every scrape index and reports whether the walk plausibly reached the start of the + * source: + * - state is done, no lastError, and the message count is non-zero + * - the message count matches the Discord search total when the scrape recorded one + * - the oldest message is close to the channel's creation time (derived from its snowflake) + * + * The last check is a heuristic: a channel can legitimately sit empty for a while after it is + * created, and a ranged scrape (From date) is expected to stop early. Anything flagged deserves + * a look, not necessarily a re-run. + * + * Usage: pnpm verify:scrapes [--db ] [--grace ] + */ + +import { existsSync } from "node:fs"; +import { + isoFromSnowflake, + listScrapes, + resolveDbRoot, + ScrapeStore, + scrapeDir +} from "../../media-scrape/src/index.js"; + +const args = process.argv.slice(2); +const valueOf = (flag: string) => { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +}; + +const dbRoot = valueOf("--db") ?? resolveDbRoot(); +const graceDays = Number(valueOf("--grace") ?? 60); +const graceMs = (Number.isFinite(graceDays) ? graceDays : 60) * 86_400_000; + +interface Verdict { + id: string; + channelId: string; + state: string; + backend: string; + messages: number; + estimated: number | null; + oldest: string | null; + created: string | null; + flags: string[]; + /** Another scrape of the same source that completed and walked further. */ + supersededBy?: string; +} + +const results: Verdict[] = []; + +for (const meta of listScrapes(dbRoot)) { + if (meta.state === "deleted" || meta.state === "deleting") continue; + + const flags: string[] = []; + let messages = 0; + let oldest: string | null = null; + + const dir = scrapeDir(dbRoot, meta.id); + if (!existsSync(dir)) { + flags.push("directory missing"); + } else { + try { + const store = new ScrapeStore(dir); + try { + const row = store.dbSummary(); + messages = row.messages; + oldest = row.oldest; + } finally { + store.close(); + } + } catch (error) { + flags.push(`unreadable index: ${String(error)}`); + } + } + + if (meta.state !== "done") flags.push(`state ${meta.state}`); + if (meta.lastError) flags.push(`lastError: ${meta.lastError}`); + if (messages === 0) flags.push("no messages indexed"); + + if (meta.estimatedMessages != null && messages < meta.estimatedMessages * 0.98) { + flags.push(`indexed ${messages} of ~${meta.estimatedMessages} search results`); + } + + let created: string | null = null; + try { + created = isoFromSnowflake(meta.channelId); + } catch { + // not a snowflake; skip the age heuristic + } + + if (!meta.fromDate && created && oldest) { + const age = new Date(oldest).getTime() - new Date(created).getTime(); + if (Number.isFinite(age) && age > graceMs) { + flags.push(`oldest ${oldest.slice(0, 10)} vs source created ${created.slice(0, 10)}`); + } + } + + results.push({ + id: meta.id, + channelId: meta.channelId, + state: meta.state, + backend: meta.backend, + messages, + estimated: meta.estimatedMessages ?? null, + oldest, + created, + flags + }); +} + +// A flagged scrape that a later full walk of the same source supersedes is stale, not a problem. +for (const result of results) { + if (!result.flags.length) continue; + + const replacement = results.find( + candidate => + candidate !== result && + candidate.channelId === result.channelId && + candidate.flags.length === 0 && + candidate.messages > result.messages + ); + if (replacement) result.supersededBy = replacement.id; +} + +const complete = results.filter(result => result.flags.length === 0); +const superseded = results.filter(result => result.flags.length > 0 && result.supersededBy); +const incomplete = results.filter(result => result.flags.length > 0 && !result.supersededBy); + +console.log( + `scrape completeness: ${complete.length} complete, ${superseded.length} superseded, ` + + `${incomplete.length} incomplete` +); +console.log(`database root: ${dbRoot}\n`); + +for (const result of results) { + const mark = !result.flags.length ? "ok " : result.supersededBy ? "old " : "FLAG"; + const oldest = result.oldest ? result.oldest.slice(0, 10) : "----------"; + const created = result.created ? result.created.slice(0, 10) : "----------"; + console.log( + `${mark} ${result.id.padEnd(44)} ${result.state.padEnd(7)} ` + + `msgs ${String(result.messages).padStart(6)}/${String(result.estimated ?? "?").padStart(6)} ` + + `oldest ${oldest} (source ${created})` + + (result.supersededBy + ? ` <- superseded by ${result.supersededBy}` + : result.flags.length + ? ` <- ${result.flags.join("; ")}` + : "") + ); +} + +process.exitCode = incomplete.length ? 1 : 0;