Continue new scrapes where the last complete one started
This commit is contained in:
@@ -657,15 +657,27 @@ export class ScrapeStore {
|
||||
.run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
|
||||
}
|
||||
|
||||
/** Message span of the walk, for completeness checks. */
|
||||
dbSummary(): { messages: number; oldest: string | null; newest: string | null } {
|
||||
/** Message span of the walk, for completeness checks and incremental scrapes. */
|
||||
dbSummary(): {
|
||||
messages: number;
|
||||
oldest: string | null;
|
||||
newest: string | null;
|
||||
newestId: string | null;
|
||||
} {
|
||||
const row = this.db
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest FROM messages"
|
||||
"SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest, MAX(id) AS newestId FROM messages"
|
||||
)
|
||||
.get() as { messages: number; oldest: string | null; newest: string | null } | undefined;
|
||||
.get() as
|
||||
| { messages: number; oldest: string | null; newest: string | null; newestId: string | null }
|
||||
| undefined;
|
||||
|
||||
return { messages: row?.messages ?? 0, oldest: row?.oldest ?? null, newest: row?.newest ?? null };
|
||||
return {
|
||||
messages: row?.messages ?? 0,
|
||||
oldest: row?.oldest ?? null,
|
||||
newest: row?.newest ?? null,
|
||||
newestId: row?.newestId ?? null
|
||||
};
|
||||
}
|
||||
|
||||
/** Preview state for the whole scrape and per kind, for the backfill UI. */
|
||||
|
||||
@@ -1437,6 +1437,23 @@ void (async () => {
|
||||
reopened.close();
|
||||
});
|
||||
|
||||
await test("dbSummary reports the message span and newest id", () => {
|
||||
const dir = join(root, "summary");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const store = new ScrapeStore(dir);
|
||||
store.insertPage([
|
||||
message("1800", { timestamp: "2026-09-16T00:00:00.000Z" }),
|
||||
message("1801", { timestamp: "2026-09-17T00:00:00.000Z" })
|
||||
]);
|
||||
|
||||
const summary = store.dbSummary();
|
||||
assert.equal(summary.messages, 2);
|
||||
assert.equal(summary.oldest, "2026-09-16T00:00:00.000Z");
|
||||
assert.equal(summary.newest, "2026-09-17T00:00:00.000Z");
|
||||
assert.equal(summary.newestId, "1801");
|
||||
store.close();
|
||||
});
|
||||
|
||||
await test("message links build for guilds and DMs", () => {
|
||||
assert.equal(messageUrlFor("123", "456", "789"), "https://discord.com/channels/123/456/789");
|
||||
assert.equal(messageUrlFor(null, "456", "789"), "https://discord.com/channels/@me/456/789");
|
||||
|
||||
Reference in New Issue
Block a user