Continue new scrapes where the last complete one started

This commit is contained in:
Fisto
2026-09-21 10:45:30 +02:00
parent 3bba051971
commit 816939288c
3 changed files with 77 additions and 8 deletions
+17 -5
View File
@@ -657,15 +657,27 @@ export class ScrapeStore {
.run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
}
/** Message span of the walk, for completeness checks. */
dbSummary(): { messages: number; oldest: string | null; newest: string | null } {
/** Message span of the walk, for completeness checks and incremental scrapes. */
dbSummary(): {
messages: number;
oldest: string | null;
newest: string | null;
newestId: string | null;
} {
const row = this.db
.prepare(
"SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest FROM messages"
"SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest, MAX(id) AS newestId FROM messages"
)
.get() as { messages: number; oldest: string | null; newest: string | null } | undefined;
.get() as
| { messages: number; oldest: string | null; newest: string | null; newestId: string | null }
| undefined;
return { messages: row?.messages ?? 0, oldest: row?.oldest ?? null, newest: row?.newest ?? null };
return {
messages: row?.messages ?? 0,
oldest: row?.oldest ?? null,
newest: row?.newest ?? null,
newestId: row?.newestId ?? null
};
}
/** Preview state for the whole scrape and per kind, for the backfill UI. */
+17
View File
@@ -1437,6 +1437,23 @@ void (async () => {
reopened.close();
});
await test("dbSummary reports the message span and newest id", () => {
const dir = join(root, "summary");
mkdirSync(dir, { recursive: true });
const store = new ScrapeStore(dir);
store.insertPage([
message("1800", { timestamp: "2026-09-16T00:00:00.000Z" }),
message("1801", { timestamp: "2026-09-17T00:00:00.000Z" })
]);
const summary = store.dbSummary();
assert.equal(summary.messages, 2);
assert.equal(summary.oldest, "2026-09-16T00:00:00.000Z");
assert.equal(summary.newest, "2026-09-17T00:00:00.000Z");
assert.equal(summary.newestId, "1801");
store.close();
});
await test("message links build for guilds and DMs", () => {
assert.equal(messageUrlFor("123", "456", "789"), "https://discord.com/channels/123/456/789");
assert.equal(messageUrlFor(null, "456", "789"), "https://discord.com/channels/@me/456/789");