Add friendly scrape names and a completeness check

AGENTS.md

@@ -117,9 +117,10 @@ pnpm testTypes    # tsc --noEmit
 pnpm lint
 
 cd poc
-pnpm check        # type check the daemon
+pnpm check            # type check the daemon
-pnpm test:comm    # unit tests
+pnpm test:comm        # unit tests
-pnpm test:ui      # optional browser smoke test for the Media tab
+pnpm test:ui          # optional browser smoke test for the Media tab
+pnpm verify:scrapes   # scrape completeness report (exit 1 when a scrape looks truncated)
 ```
 
 `pnpm test:ui` is optional. It needs a running daemon with media data and a Chromium: set

CURRENT.md

@@ -40,6 +40,15 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C59 Scrape names and the completeness check. Scrape listings (History, Clean media,
+    Bot-tab job/lane cards) now lead with the first two words of the channel or group-chat name
+    plus the scrape id. New `poc/scripts/verify-scrapes.ts` (`pnpm verify:scrapes`) checks every
+    index: done state, no lastError, messages non-zero, messages vs the Discord search total, and
+    the oldest message against the source's snowflake creation time (60-day grace). Scrapes a
+    later full walk supersedes are reported as `old`, not failures. Result: 10 complete, 4
+    superseded, 1 genuinely incomplete — the early DM test scrape
+    `805255536223715370-20260916215657` (10 messages, pagination stopped before the fix, never
+    re-scraped).
   - [x] C58 Titles from Discord's embed cache. Gone YouTube videos have no oEmbed answer, but
     Discord keeps the embed title (and, with the rebuilt core bundle, the author/channel). The
     title lane now asks the source message's embed first and only then the provider oEmbed; the

docs/testing.md

@@ -61,6 +61,20 @@ The mock also covers tiers, continuation, polls, reactions, and the action log.
 with `set_puppet_config` before a test. Inject a reaction with
 `TEST_TRIGGER_REACTION=1 TEST_REACTION_MESSAGE_ID=msg-1 TEST_REACTION_EMOJI="1️⃣" pnpm test:trigger`.
 
+## Scrape completeness
+
+`pnpm verify:scrapes` (from `poc/`) walks every scrape index and reports whether the walk
+plausibly reached the start of the source:
+
+- state is `done`, no `lastError`, and the message count is non-zero
+- the message count matches the recorded Discord search total when there is one
+- the oldest message is close to the source's creation time, derived from the channel ID
+  snowflake (`--grace <days>` sets the allowance, default 60)
+
+The age check is a heuristic: a channel can sit empty after creation, and a ranged scrape is
+expected to stop early. A flagged scrape that a later full walk of the same source supersedes is
+reported as `old` rather than a problem. Exit code is 1 when a scrape is genuinely incomplete.
+
 ## Web UI smoke test
 
 The web UI starts with the daemon. The mock core is enough for this test.

media-scrape/src/store.ts

@@ -657,6 +657,17 @@ export class ScrapeStore {
             .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
     }
 
+    /** Message span of the walk, for completeness checks. */
+    dbSummary(): { messages: number; oldest: string | null; newest: string | null } {
+        const row = this.db
+            .prepare(
+                "SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest FROM messages"
+            )
+            .get() as { messages: number; oldest: string | null; newest: string | null } | undefined;
+
+        return { messages: row?.messages ?? 0, oldest: row?.oldest ?? null, newest: row?.newest ?? null };
+    }
+
     /** Preview state for the whole scrape and per kind, for the backfill UI. */
     previewCounts(): {
         total: number;

poc/package.json

@@ -12,6 +12,7 @@
         "control:bridge": "tsx src/control-bridge.ts",
         "check": "tsc --noEmit",
         "control:mock": "tsx src/mock-core.ts",
+        "verify:scrapes": "tsx scripts/verify-scrapes.ts",
         "test:trigger": "tsx src/test-trigger.ts",
         "cdp:dispatch": "tsx src/cdp-dispatch.ts",
         "test:cache": "tsx src/test-cache.ts",

poc/public/media.js

@@ -866,7 +866,7 @@
         for (const job of jobs) {
             const row = el("div", `vk-bot-job vk-bot-job-${job.state}`);
             const head = el("div", "vk-bot-job-head");
-            head.append(el("span", "vk-bot-job-title", channelLabel(job.chatId)));
+            head.append(el("span", "vk-bot-job-title", scrapeName(job.chatId)));
             head.append(el("span", `vk-bot-state vk-bot-state-${job.state}`, job.state));
 
             const parts = [`${job.pending} pending`];
@@ -953,7 +953,7 @@
         if (status.failed) parts.push(`${status.failed} failed`);
         if (status.perMinute) parts.push(`${status.perMinute}/min`);
         if (status.pending > 0) parts.push(`ETA ${formatEta(status.etaMs)}`);
-        if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
+        if (status.currentScrapeId) parts.push(`on ${scrapeName(status.currentScrapeId)}`);
         head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
         titleLane.append(head);
 
@@ -1005,7 +1005,7 @@
         if (status.recovered) parts.push(`${status.recovered} recovered`);
         if (status.deferred) parts.push(`${status.deferred} deferred`);
         if (status.gone) parts.push(`${status.gone} unavailable`);
-        if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
+        if (status.currentScrapeId) parts.push(`on ${scrapeName(status.currentScrapeId)}`);
         head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
         recoverLane.append(head);
 
@@ -2451,6 +2451,20 @@
         return state.channels.find(channel => channel.id === channelId)?.label ?? channelId;
     }
 
+    /** Short scrape name: the first two words of the channel or group-chat name. */
+    function scrapeName(channelId) {
+        const label = channelLabel(channelId);
+        const cleaned = label.includes("/#")
+            ? label.split("/#")[1]
+            : label.replace(/\s*\((group DM|DM)\)\s*$/i, "");
+        const words = cleaned
+            .replace(/[_\-]+/g, " ")
+            .split(/\s+/)
+            .filter(Boolean)
+            .slice(0, 2);
+        return words.join(" ") || label;
+    }
+
     async function openHistoryModal() {
         const list = historyModal.querySelector("#history-list");
         list.replaceChildren(el("div", null, "loading..."));
@@ -2469,7 +2483,9 @@
             for (const scrape of scrapes) {
                 const media = scrape.media ?? {};
                 const row = el("div", "vk-media-history-row");
-                row.append(el("strong", null, channelLabel(scrape.channelId)));
+                row.append(
+                    el("strong", null, `${scrapeName(scrape.channelId)} · ${scrape.id}`)
+                );
                 row.append(
                     el(
                         "span",
@@ -2894,7 +2910,14 @@
 
         list.replaceChildren();
         for (const scrape of scrapes) {
-            list.append(el("div", null, `${scrape.id} · ${scrape.state} · ${formatBytes(scrape.bytes)}`));
+            list.append(
+                el(
+                    "div",
+                    null,
+                    `${scrapeName(scrape.channelId)} · ${scrape.id} · ${scrape.state} · ` +
+                        formatBytes(scrape.bytes)
+                )
+            );
         }
 
         const bytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);

poc/scripts/verify-scrapes.ts

@@ -0,0 +1,152 @@
+/*
+ * Scrape completeness check.
+ *
+ * Reads every scrape index and reports whether the walk plausibly reached the start of the
+ * source:
+ *   - state is done, no lastError, and the message count is non-zero
+ *   - the message count matches the Discord search total when the scrape recorded one
+ *   - the oldest message is close to the channel's creation time (derived from its snowflake)
+ *
+ * The last check is a heuristic: a channel can legitimately sit empty for a while after it is
+ * created, and a ranged scrape (From date) is expected to stop early. Anything flagged deserves
+ * a look, not necessarily a re-run.
+ *
+ * Usage: pnpm verify:scrapes [--db <path>] [--grace <days>]
+ */
+
+import { existsSync } from "node:fs";
+import {
+    isoFromSnowflake,
+    listScrapes,
+    resolveDbRoot,
+    ScrapeStore,
+    scrapeDir
+} from "../../media-scrape/src/index.js";
+
+const args = process.argv.slice(2);
+const valueOf = (flag: string) => {
+    const index = args.indexOf(flag);
+    return index >= 0 ? args[index + 1] : undefined;
+};
+
+const dbRoot = valueOf("--db") ?? resolveDbRoot();
+const graceDays = Number(valueOf("--grace") ?? 60);
+const graceMs = (Number.isFinite(graceDays) ? graceDays : 60) * 86_400_000;
+
+interface Verdict {
+    id: string;
+    channelId: string;
+    state: string;
+    backend: string;
+    messages: number;
+    estimated: number | null;
+    oldest: string | null;
+    created: string | null;
+    flags: string[];
+    /** Another scrape of the same source that completed and walked further. */
+    supersededBy?: string;
+}
+
+const results: Verdict[] = [];
+
+for (const meta of listScrapes(dbRoot)) {
+    if (meta.state === "deleted" || meta.state === "deleting") continue;
+
+    const flags: string[] = [];
+    let messages = 0;
+    let oldest: string | null = null;
+
+    const dir = scrapeDir(dbRoot, meta.id);
+    if (!existsSync(dir)) {
+        flags.push("directory missing");
+    } else {
+        try {
+            const store = new ScrapeStore(dir);
+            try {
+                const row = store.dbSummary();
+                messages = row.messages;
+                oldest = row.oldest;
+            } finally {
+                store.close();
+            }
+        } catch (error) {
+            flags.push(`unreadable index: ${String(error)}`);
+        }
+    }
+
+    if (meta.state !== "done") flags.push(`state ${meta.state}`);
+    if (meta.lastError) flags.push(`lastError: ${meta.lastError}`);
+    if (messages === 0) flags.push("no messages indexed");
+
+    if (meta.estimatedMessages != null && messages < meta.estimatedMessages * 0.98) {
+        flags.push(`indexed ${messages} of ~${meta.estimatedMessages} search results`);
+    }
+
+    let created: string | null = null;
+    try {
+        created = isoFromSnowflake(meta.channelId);
+    } catch {
+        // not a snowflake; skip the age heuristic
+    }
+
+    if (!meta.fromDate && created && oldest) {
+        const age = new Date(oldest).getTime() - new Date(created).getTime();
+        if (Number.isFinite(age) && age > graceMs) {
+            flags.push(`oldest ${oldest.slice(0, 10)} vs source created ${created.slice(0, 10)}`);
+        }
+    }
+
+    results.push({
+        id: meta.id,
+        channelId: meta.channelId,
+        state: meta.state,
+        backend: meta.backend,
+        messages,
+        estimated: meta.estimatedMessages ?? null,
+        oldest,
+        created,
+        flags
+    });
+}
+
+// A flagged scrape that a later full walk of the same source supersedes is stale, not a problem.
+for (const result of results) {
+    if (!result.flags.length) continue;
+
+    const replacement = results.find(
+        candidate =>
+            candidate !== result &&
+            candidate.channelId === result.channelId &&
+            candidate.flags.length === 0 &&
+            candidate.messages > result.messages
+    );
+    if (replacement) result.supersededBy = replacement.id;
+}
+
+const complete = results.filter(result => result.flags.length === 0);
+const superseded = results.filter(result => result.flags.length > 0 && result.supersededBy);
+const incomplete = results.filter(result => result.flags.length > 0 && !result.supersededBy);
+
+console.log(
+    `scrape completeness: ${complete.length} complete, ${superseded.length} superseded, ` +
+        `${incomplete.length} incomplete`
+);
+console.log(`database root: ${dbRoot}\n`);
+
+for (const result of results) {
+    const mark = !result.flags.length ? "ok  " : result.supersededBy ? "old " : "FLAG";
+    const oldest = result.oldest ? result.oldest.slice(0, 10) : "----------";
+    const created = result.created ? result.created.slice(0, 10) : "----------";
+    console.log(
+        `${mark} ${result.id.padEnd(44)} ${result.state.padEnd(7)} ` +
+            `msgs ${String(result.messages).padStart(6)}/${String(result.estimated ?? "?").padStart(6)} ` +
+            `oldest ${oldest} (source ${created})` +
+            (result.supersededBy
+                ? `  <- superseded by ${result.supersededBy}`
+                : result.flags.length
+                  ? `  <- ${result.flags.join("; ")}`
+                  : "")
+    );
+}
+
+process.exitCode = incomplete.length ? 1 : 0;