Add message counting and date-bounded scrapes

This commit is contained in:
Fisto
2026-09-16 12:06:34 +02:00
parent f14f067413
commit fae754fba0
21 changed files with 410 additions and 22 deletions
+45
View File
@@ -14,6 +14,7 @@ import { extractMedia } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js";
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
import { isNearLimit, planRetention } from "../src/retention.js";
import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
import type { CoreAttachment, CoreMessage } from "../src/types.js";
@@ -710,6 +711,50 @@ void (async () => {
store.close();
});
await test("snowflake helpers round-trip dates", () => {
const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
assert.ok(id);
assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z");
assert.equal(snowflakeFromIso("not a date"), null);
assert.equal(isoFromSnowflake("nope"), null);
});
await test("engine honors a from-date boundary", async () => {
const backend = new FakeBackend([
{ messages: [message("1000"), message("900"), message("800")] }
]);
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 10,
pageMinMs: 0,
pageMaxMs: 0,
sleep: async () => {},
random: () => 0,
log: () => {},
previews: false
});
const started = await engine.start({
channelId: "131",
guildId: null,
mode: "links",
backend,
afterId: "900"
});
await engine.processQueue();
const meta = engine.get(started.id);
assert.equal(meta?.state, "done");
assert.equal(meta?.messages, 1);
assert.equal(meta?.afterId, "900");
assert.equal(backend.calls.length, 1);
const store = new ScrapeStore(join(root, started.id));
assert.equal(store.queryMedia({}).length, 0);
store.close();
});
await test("retention marks the oldest completed scrapes up to the limit", () => {
const scrapes = [
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },