Add friendly scrape names and a completeness check

2026-09-21 10:40:53 +02:00
parent 19bf6e16fc
commit 3bba051971
7 changed files with 219 additions and 8 deletions
@@ -0,0 +1,152 @@
+/*
+ * Scrape completeness check.
+ *
+ * Reads every scrape index and reports whether the walk plausibly reached the start of the
+ * source:
+ *   - state is done, no lastError, and the message count is non-zero
+ *   - the message count matches the Discord search total when the scrape recorded one
+ *   - the oldest message is close to the channel's creation time (derived from its snowflake)
+ *
+ * The last check is a heuristic: a channel can legitimately sit empty for a while after it is
+ * created, and a ranged scrape (From date) is expected to stop early. Anything flagged deserves
+ * a look, not necessarily a re-run.
+ *
+ * Usage: pnpm verify:scrapes [--db <path>] [--grace <days>]
+ */
+
+import { existsSync } from "node:fs";
+import {
+    isoFromSnowflake,
+    listScrapes,
+    resolveDbRoot,
+    ScrapeStore,
+    scrapeDir
+} from "../../media-scrape/src/index.js";
+
+const args = process.argv.slice(2);
+const valueOf = (flag: string) => {
+    const index = args.indexOf(flag);
+    return index >= 0 ? args[index + 1] : undefined;
+};
+
+const dbRoot = valueOf("--db") ?? resolveDbRoot();
+const graceDays = Number(valueOf("--grace") ?? 60);
+const graceMs = (Number.isFinite(graceDays) ? graceDays : 60) * 86_400_000;
+
+interface Verdict {
+    id: string;
+    channelId: string;
+    state: string;
+    backend: string;
+    messages: number;
+    estimated: number | null;
+    oldest: string | null;
+    created: string | null;
+    flags: string[];
+    /** Another scrape of the same source that completed and walked further. */
+    supersededBy?: string;
+}
+
+const results: Verdict[] = [];
+
+for (const meta of listScrapes(dbRoot)) {
+    if (meta.state === "deleted" || meta.state === "deleting") continue;
+
+    const flags: string[] = [];
+    let messages = 0;
+    let oldest: string | null = null;
+
+    const dir = scrapeDir(dbRoot, meta.id);
+    if (!existsSync(dir)) {
+        flags.push("directory missing");
+    } else {
+        try {
+            const store = new ScrapeStore(dir);
+            try {
+                const row = store.dbSummary();
+                messages = row.messages;
+                oldest = row.oldest;
+            } finally {
+                store.close();
+            }
+        } catch (error) {
+            flags.push(`unreadable index: ${String(error)}`);
+        }
+    }
+
+    if (meta.state !== "done") flags.push(`state ${meta.state}`);
+    if (meta.lastError) flags.push(`lastError: ${meta.lastError}`);
+    if (messages === 0) flags.push("no messages indexed");
+
+    if (meta.estimatedMessages != null && messages < meta.estimatedMessages * 0.98) {
+        flags.push(`indexed ${messages} of ~${meta.estimatedMessages} search results`);
+    }
+
+    let created: string | null = null;
+    try {
+        created = isoFromSnowflake(meta.channelId);
+    } catch {
+        // not a snowflake; skip the age heuristic
+    }
+
+    if (!meta.fromDate && created && oldest) {
+        const age = new Date(oldest).getTime() - new Date(created).getTime();
+        if (Number.isFinite(age) && age > graceMs) {
+            flags.push(`oldest ${oldest.slice(0, 10)} vs source created ${created.slice(0, 10)}`);
+        }
+    }
+
+    results.push({
+        id: meta.id,
+        channelId: meta.channelId,
+        state: meta.state,
+        backend: meta.backend,
+        messages,
+        estimated: meta.estimatedMessages ?? null,
+        oldest,
+        created,
+        flags
+    });
+}
+
+// A flagged scrape that a later full walk of the same source supersedes is stale, not a problem.
+for (const result of results) {
+    if (!result.flags.length) continue;
+
+    const replacement = results.find(
+        candidate =>
+            candidate !== result &&
+            candidate.channelId === result.channelId &&
+            candidate.flags.length === 0 &&
+            candidate.messages > result.messages
+    );
+    if (replacement) result.supersededBy = replacement.id;
+}
+
+const complete = results.filter(result => result.flags.length === 0);
+const superseded = results.filter(result => result.flags.length > 0 && result.supersededBy);
+const incomplete = results.filter(result => result.flags.length > 0 && !result.supersededBy);
+
+console.log(
+    `scrape completeness: ${complete.length} complete, ${superseded.length} superseded, ` +
+        `${incomplete.length} incomplete`
+);
+console.log(`database root: ${dbRoot}\n`);
+
+for (const result of results) {
+    const mark = !result.flags.length ? "ok  " : result.supersededBy ? "old " : "FLAG";
+    const oldest = result.oldest ? result.oldest.slice(0, 10) : "----------";
+    const created = result.created ? result.created.slice(0, 10) : "----------";
+    console.log(
+        `${mark} ${result.id.padEnd(44)} ${result.state.padEnd(7)} ` +
+            `msgs ${String(result.messages).padStart(6)}/${String(result.estimated ?? "?").padStart(6)} ` +
+            `oldest ${oldest} (source ${created})` +
+            (result.supersededBy
+                ? `  <- superseded by ${result.supersededBy}`
+                : result.flags.length
+                  ? `  <- ${result.flags.join("; ")}`
+                  : "")
+    );
+}
+
+process.exitCode = incomplete.length ? 1 : 0;