Veskslop-t/poc/scripts/verify-scrapes.ts

/*
 * Scrape completeness check.
 *
 * Reads every scrape index and reports whether the walk plausibly reached the start of the
 * source:
 *   - state is done, no lastError, and the message count is non-zero
 *   - the message count matches the Discord search total when the scrape recorded one
 *   - the oldest message is close to the channel's creation time (derived from its snowflake)
 *
 * The last check is a heuristic: a channel can legitimately sit empty for a while after it is
 * created, and a ranged scrape (From date) is expected to stop early. Anything flagged deserves
 * a look, not necessarily a re-run.
 *
 * Usage: pnpm verify:scrapes [--db <path>] [--grace <days>]
 */

import { existsSync } from "node:fs";
import {
    isoFromSnowflake,
    listScrapes,
    resolveDbRoot,
    ScrapeStore,
    scrapeDir
} from "../../media-scrape/src/index.js";

const args = process.argv.slice(2);
const valueOf = (flag: string) => {
    const index = args.indexOf(flag);
    return index >= 0 ? args[index + 1] : undefined;
};

const dbRoot = valueOf("--db") ?? resolveDbRoot();
const graceDays = Number(valueOf("--grace") ?? 60);
const graceMs = (Number.isFinite(graceDays) ? graceDays : 60) * 86_400_000;

interface Verdict {
    id: string;
    channelId: string;
    state: string;
    backend: string;
    messages: number;
    estimated: number | null;
    oldest: string | null;
    created: string | null;
    flags: string[];
    /** Another scrape of the same source that completed and walked further. */
    supersededBy?: string;
}

const results: Verdict[] = [];

for (const meta of listScrapes(dbRoot)) {
    if (meta.state === "deleted" || meta.state === "deleting") continue;

    const flags: string[] = [];
    let messages = 0;
    let oldest: string | null = null;

    const dir = scrapeDir(dbRoot, meta.id);
    if (!existsSync(dir)) {
        flags.push("directory missing");
    } else {
        try {
            const store = new ScrapeStore(dir);
            try {
                const row = store.dbSummary();
                messages = row.messages;
                oldest = row.oldest;
            } finally {
                store.close();
            }
        } catch (error) {
            flags.push(`unreadable index: ${String(error)}`);
        }
    }

    if (meta.state !== "done") flags.push(`state ${meta.state}`);
    if (meta.lastError) flags.push(`lastError: ${meta.lastError}`);
    if (messages === 0) flags.push("no messages indexed");

    if (meta.estimatedMessages != null && messages < meta.estimatedMessages * 0.98) {
        flags.push(`indexed ${messages} of ~${meta.estimatedMessages} search results`);
    }

    let created: string | null = null;
    try {
        created = isoFromSnowflake(meta.channelId);
    } catch {
        // not a snowflake; skip the age heuristic
    }

    if (!meta.fromDate && created && oldest) {
        const age = new Date(oldest).getTime() - new Date(created).getTime();
        if (Number.isFinite(age) && age > graceMs) {
            flags.push(`oldest ${oldest.slice(0, 10)} vs source created ${created.slice(0, 10)}`);
        }
    }

    results.push({
        id: meta.id,
        channelId: meta.channelId,
        state: meta.state,
        backend: meta.backend,
        messages,
        estimated: meta.estimatedMessages ?? null,
        oldest,
        created,
        flags
    });
}

// A flagged scrape that a later full walk of the same source supersedes is stale, not a problem.
for (const result of results) {
    if (!result.flags.length) continue;

    const replacement = results.find(
        candidate =>
            candidate !== result &&
            candidate.channelId === result.channelId &&
            candidate.flags.length === 0 &&
            candidate.messages > result.messages
    );
    if (replacement) result.supersededBy = replacement.id;
}

const complete = results.filter(result => result.flags.length === 0);
const superseded = results.filter(result => result.flags.length > 0 && result.supersededBy);
const incomplete = results.filter(result => result.flags.length > 0 && !result.supersededBy);

console.log(
    `scrape completeness: ${complete.length} complete, ${superseded.length} superseded, ` +
        `${incomplete.length} incomplete`
);
console.log(`database root: ${dbRoot}\n`);

for (const result of results) {
    const mark = !result.flags.length ? "ok  " : result.supersededBy ? "old " : "FLAG";
    const oldest = result.oldest ? result.oldest.slice(0, 10) : "----------";
    const created = result.created ? result.created.slice(0, 10) : "----------";
    console.log(
        `${mark} ${result.id.padEnd(44)} ${result.state.padEnd(7)} ` +
            `msgs ${String(result.messages).padStart(6)}/${String(result.estimated ?? "?").padStart(6)} ` +
            `oldest ${oldest} (source ${created})` +
            (result.supersededBy
                ? `  <- superseded by ${result.supersededBy}`
                : result.flags.length
                  ? `  <- ${result.flags.join("; ")}`
                  : "")
    );
}

process.exitCode = incomplete.length ? 1 : 0;