Add message counting and date-bounded scrapes

media-scrape/test/live-count.ts

@@ -0,0 +1,50 @@
+/*
+ * Live check: Discord search total_results for a channel or a date range.
+ *
+ * Usage (from poc/):
+ *   VESKSLOP_SLOPBOT_ENABLED=1 VESKSLOP_SLOPBOT_STATE=/path/to/slopbot.json \
+ *     tsx ../media-scrape/test/live-count.ts <channelId> <guildId> [afterIso] [beforeIso]
+ */
+
+import { loadSlopbotConfig, startSlopbot, stopSlopbot } from "../slopbot/index.js";
+
+function snowflakeFromIso(iso: string): string {
+    return String((BigInt(Date.parse(iso) - 1420070400000) << 22n));
+}
+
+async function main() {
+    const [channelId, guildId, after, before] = process.argv.slice(2);
+    if (!channelId || !guildId) {
+        console.error("usage: live-count.ts <channelId> <guildId> [afterIso] [beforeIso]");
+        process.exit(1);
+    }
+
+    const config = loadSlopbotConfig();
+    if (!config.enabled) {
+        console.error("slopbot disabled: set VESKSLOP_SLOPBOT_ENABLED=1 and a token");
+        process.exit(1);
+    }
+
+    const client = await startSlopbot(config);
+    try {
+        const query = new URLSearchParams({ channel_id: channelId, limit: "1" });
+        if (after) query.set("min_id", snowflakeFromIso(after));
+        if (before) query.set("max_id", snowflakeFromIso(before));
+
+        for (const [label, path] of [
+            ["guild", `/guilds/${guildId}/messages/search`],
+            ["channel", `/channels/${channelId}/messages/search`]
+        ] as const) {
+            try {
+                const data = (await client.rest.get(path, { query })) as Record<string, unknown>;
+                console.log(`${label}: total_results=${String(data?.total_results)} keys=${Object.keys(data ?? {}).join(",")}`);
+            } catch (error) {
+                console.log(`${label}: error ${(error as Error).message}`);
+            }
+        }
+    } finally {
+        await stopSlopbot(client);
+    }
+}
+
+void main();

media-scrape/test/run.ts

@@ -14,6 +14,7 @@ import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
+import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
 import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
 import type { CoreAttachment, CoreMessage } from "../src/types.js";
 
@@ -710,6 +711,50 @@ void (async () => {
         store.close();
     });
 
+    await test("snowflake helpers round-trip dates", () => {
+        const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
+        assert.ok(id);
+        assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z");
+        assert.equal(snowflakeFromIso("not a date"), null);
+        assert.equal(isoFromSnowflake("nope"), null);
+    });
+
+    await test("engine honors a from-date boundary", async () => {
+        const backend = new FakeBackend([
+            { messages: [message("1000"), message("900"), message("800")] }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false
+        });
+
+        const started = await engine.start({
+            channelId: "131",
+            guildId: null,
+            mode: "links",
+            backend,
+            afterId: "900"
+        });
+        await engine.processQueue();
+
+        const meta = engine.get(started.id);
+        assert.equal(meta?.state, "done");
+        assert.equal(meta?.messages, 1);
+        assert.equal(meta?.afterId, "900");
+        assert.equal(backend.calls.length, 1);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.equal(store.queryMedia({}).length, 0);
+        store.close();
+    });
+
     await test("retention marks the oldest completed scrapes up to the limit", () => {
         const scrapes = [
             { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },

media-scrape/test/slopbot.ts

@@ -6,6 +6,7 @@
 import assert from "node:assert/strict";
 import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js";
 import { mapSlopbotMessage } from "../slopbot/backend.js";
+import { countMessages } from "../slopbot/client.js";
 
 let failures = 0;
 
@@ -138,6 +139,35 @@ void (async () => {
         assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm");
     });
 
+    await test("countMessages reads guild search totals", async () => {
+        const calls: string[] = [];
+        const rest = {
+            async get(path: string, options?: { query?: URLSearchParams }) {
+                calls.push(`${path}?${options?.query?.toString() ?? ""}`);
+                return { total_results: 7 };
+            }
+        };
+
+        const count = await countMessages(rest, {
+            channelId: "1",
+            guildId: "2",
+            afterId: "100",
+            beforeId: "200"
+        });
+        assert.equal(count, 7);
+        assert.equal(calls[0], "/guilds/2/messages/search?channel_id=1&limit=1&min_id=100&max_id=200");
+
+        const failed = await countMessages(
+            {
+                async get() {
+                    throw new Error("nope");
+                }
+            },
+            { channelId: "1", guildId: "2" }
+        );
+        assert.equal(failed, null);
+    });
+
     await test("backend pages oldest first and passes before and limit", async () => {
         const fetcher = new FakeFetcher([
             [discordMessage({ id: "1002" }), discordMessage({ id: "1001" })],