Add message counting and date-bounded scrapes

media-scrape/slopbot/client.ts

@@ -54,6 +54,33 @@ export function slopbotStatus(client: Client | null): SlopbotStatus {
     };
 }
 
+/** The REST surface the search count needs. discord.js `client.rest` satisfies it. */
+export interface SlopbotRest {
+    get(path: string, options?: { query?: URLSearchParams }): Promise<unknown>;
+}
+
+/**
+ * Discord search `total_results` for a channel or range. One request.
+ * Null when the search fails (rate limit, permissions, or no index).
+ */
+export async function countMessages(
+    rest: SlopbotRest,
+    input: { channelId: string; guildId: string; afterId?: string | null; beforeId?: string | null }
+): Promise<number | null> {
+    const query = new URLSearchParams({ channel_id: input.channelId, limit: "1" });
+    if (input.afterId) query.set("min_id", input.afterId);
+    if (input.beforeId) query.set("max_id", input.beforeId);
+
+    try {
+        const data = (await rest.get(`/guilds/${input.guildId}/messages/search`, { query })) as {
+            total_results?: number;
+        };
+        return typeof data?.total_results === "number" ? data.total_results : null;
+    } catch {
+        return null;
+    }
+}
+
 /** The ScrapeBackend fetcher over a live client. */
 export function slopbotFetcher(client: Client): SlopbotFetcher {
     return {

media-scrape/slopbot/index.ts

@@ -13,7 +13,15 @@ export type SlopbotClient = Client;
 
 export { loadSlopbotConfig, type SlopbotConfig };
 export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js";
-export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js";
+export {
+    countMessages,
+    slopbotFetcher,
+    slopbotStatus,
+    startSlopbot,
+    stopSlopbot,
+    type SlopbotRest,
+    type SlopbotStatus
+} from "./client.js";
 
 /** View Channels + Read Message History. The bot reads only; it never sends. */
 const PERMISSIONS = String((1 << 10) | (1 << 16));

media-scrape/src/engine.ts

@@ -95,6 +95,11 @@ export class ScrapeEngine {
         mode: ScrapeMode;
         backend: ScrapeBackend;
         downloadDir?: string | null;
+        afterId?: string | null;
+        beforeId?: string | null;
+        fromDate?: string | null;
+        toDate?: string | null;
+        estimatedMessages?: number | null;
     }): Promise<ScrapeMeta> {
         const now = this.now();
         const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
@@ -108,7 +113,7 @@ export class ScrapeEngine {
             backend: input.backend.name,
             mode: input.mode,
             state: "queued",
-            cursor: null,
+            cursor: input.beforeId ?? null,
             createdAt: timestamp,
             updatedAt: timestamp,
             pages: 0,
@@ -116,7 +121,12 @@ export class ScrapeEngine {
             media: emptyMediaCounts(),
             errors: 0,
             lastError: null,
-            downloadDir: input.downloadDir ?? null
+            downloadDir: input.downloadDir ?? null,
+            afterId: input.afterId ?? null,
+            beforeId: input.beforeId ?? null,
+            fromDate: input.fromDate ?? null,
+            toDate: input.toDate ?? null,
+            estimatedMessages: input.estimatedMessages ?? null
         };
 
         writeScrapeMeta(dir, meta);
@@ -217,15 +227,25 @@ export class ScrapeEngine {
                     break;
                 }
 
-                const added = store.insertPage(page.messages);
+                // The from-date is a lower ID boundary: trim anything older and finish.
+                const kept = meta.afterId
+                    ? page.messages.filter(message => BigInt(message.id) > BigInt(meta.afterId as string))
+                    : page.messages;
+
+                if (kept.length === 0) {
+                    meta.state = "done";
+                    break;
+                }
+
+                const added = store.insertPage(kept);
                 for (const kind of MEDIA_KINDS) meta.media[kind] += added[kind];
                 meta.pages++;
-                meta.messages += page.messages.length;
-                meta.cursor = oldestId(page.messages) ?? meta.cursor;
+                meta.messages += kept.length;
+                meta.cursor = oldestId(kept) ?? meta.cursor;
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
-                if (page.messages.length < pageSize) {
+                if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
                 }

media-scrape/src/index.ts

@@ -5,6 +5,7 @@
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
+export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
 export {
     ASPECT_PRESETS,
     RESOLUTION_PRESETS,

media-scrape/src/snowflake.ts

@@ -0,0 +1,21 @@
+/*
+ * Discord snowflake helpers for date-bounded scrapes.
+ */
+
+const DISCORD_EPOCH = 1420070400000;
+
+/** The snowflake at an instant. Null for an unparsable date; "0" before the epoch. */
+export function snowflakeFromIso(iso: string): string | null {
+    const time = Date.parse(iso);
+    if (!Number.isFinite(time)) return null;
+
+    const value = BigInt(time - DISCORD_EPOCH) << 22n;
+    return value > 0n ? value.toString() : "0";
+}
+
+/** The instant a snowflake encodes, as an ISO string. Null when it is not a snowflake. */
+export function isoFromSnowflake(id: string | null | undefined): string | null {
+    if (!id || !/^\d{17,20}$/.test(id)) return null;
+    const time = Number(BigInt(id) >> 22n) + DISCORD_EPOCH;
+    return Number.isFinite(time) ? new Date(time).toISOString() : null;
+}

media-scrape/src/types.ts

@@ -96,6 +96,16 @@ export interface ScrapeMeta {
     finishedAt?: string | null;
     /** Absolute destination for downloaded files. Null means the local `media/` folder. */
     downloadDir?: string | null;
+    /** Lower message-ID boundary (exclusive), derived from the from-date. */
+    afterId?: string | null;
+    /** Upper message-ID boundary; the walk starts just below it. */
+    beforeId?: string | null;
+    /** From-date as ISO, for display in history. */
+    fromDate?: string | null;
+    /** To-date as ISO, for display in history. */
+    toDate?: string | null;
+    /** Discord search total for the range, when known (slopbot only). */
+    estimatedMessages?: number | null;
 }
 
 export function emptyMediaCounts(): Record<MediaKind, number> {

media-scrape/test/live-count.ts

@@ -0,0 +1,50 @@
+/*
+ * Live check: Discord search total_results for a channel or a date range.
+ *
+ * Usage (from poc/):
+ *   VESKSLOP_SLOPBOT_ENABLED=1 VESKSLOP_SLOPBOT_STATE=/path/to/slopbot.json \
+ *     tsx ../media-scrape/test/live-count.ts <channelId> <guildId> [afterIso] [beforeIso]
+ */
+
+import { loadSlopbotConfig, startSlopbot, stopSlopbot } from "../slopbot/index.js";
+
+function snowflakeFromIso(iso: string): string {
+    return String((BigInt(Date.parse(iso) - 1420070400000) << 22n));
+}
+
+async function main() {
+    const [channelId, guildId, after, before] = process.argv.slice(2);
+    if (!channelId || !guildId) {
+        console.error("usage: live-count.ts <channelId> <guildId> [afterIso] [beforeIso]");
+        process.exit(1);
+    }
+
+    const config = loadSlopbotConfig();
+    if (!config.enabled) {
+        console.error("slopbot disabled: set VESKSLOP_SLOPBOT_ENABLED=1 and a token");
+        process.exit(1);
+    }
+
+    const client = await startSlopbot(config);
+    try {
+        const query = new URLSearchParams({ channel_id: channelId, limit: "1" });
+        if (after) query.set("min_id", snowflakeFromIso(after));
+        if (before) query.set("max_id", snowflakeFromIso(before));
+
+        for (const [label, path] of [
+            ["guild", `/guilds/${guildId}/messages/search`],
+            ["channel", `/channels/${channelId}/messages/search`]
+        ] as const) {
+            try {
+                const data = (await client.rest.get(path, { query })) as Record<string, unknown>;
+                console.log(`${label}: total_results=${String(data?.total_results)} keys=${Object.keys(data ?? {}).join(",")}`);
+            } catch (error) {
+                console.log(`${label}: error ${(error as Error).message}`);
+            }
+        }
+    } finally {
+        await stopSlopbot(client);
+    }
+}
+
+void main();

media-scrape/test/run.ts

@@ -14,6 +14,7 @@ import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
+import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
 import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
 import type { CoreAttachment, CoreMessage } from "../src/types.js";
 
@@ -710,6 +711,50 @@ void (async () => {
         store.close();
     });
 
+    await test("snowflake helpers round-trip dates", () => {
+        const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
+        assert.ok(id);
+        assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z");
+        assert.equal(snowflakeFromIso("not a date"), null);
+        assert.equal(isoFromSnowflake("nope"), null);
+    });
+
+    await test("engine honors a from-date boundary", async () => {
+        const backend = new FakeBackend([
+            { messages: [message("1000"), message("900"), message("800")] }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 10,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false
+        });
+
+        const started = await engine.start({
+            channelId: "131",
+            guildId: null,
+            mode: "links",
+            backend,
+            afterId: "900"
+        });
+        await engine.processQueue();
+
+        const meta = engine.get(started.id);
+        assert.equal(meta?.state, "done");
+        assert.equal(meta?.messages, 1);
+        assert.equal(meta?.afterId, "900");
+        assert.equal(backend.calls.length, 1);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.equal(store.queryMedia({}).length, 0);
+        store.close();
+    });
+
     await test("retention marks the oldest completed scrapes up to the limit", () => {
         const scrapes = [
             { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },

media-scrape/test/slopbot.ts

@@ -6,6 +6,7 @@
 import assert from "node:assert/strict";
 import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js";
 import { mapSlopbotMessage } from "../slopbot/backend.js";
+import { countMessages } from "../slopbot/client.js";
 
 let failures = 0;
 
@@ -138,6 +139,35 @@ void (async () => {
         assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm");
     });
 
+    await test("countMessages reads guild search totals", async () => {
+        const calls: string[] = [];
+        const rest = {
+            async get(path: string, options?: { query?: URLSearchParams }) {
+                calls.push(`${path}?${options?.query?.toString() ?? ""}`);
+                return { total_results: 7 };
+            }
+        };
+
+        const count = await countMessages(rest, {
+            channelId: "1",
+            guildId: "2",
+            afterId: "100",
+            beforeId: "200"
+        });
+        assert.equal(count, 7);
+        assert.equal(calls[0], "/guilds/2/messages/search?channel_id=1&limit=1&min_id=100&max_id=200");
+
+        const failed = await countMessages(
+            {
+                async get() {
+                    throw new Error("nope");
+                }
+            },
+            { channelId: "1", guildId: "2" }
+        );
+        assert.equal(failed, null);
+    });
+
     await test("backend pages oldest first and passes before and limit", async () => {
         const fetcher = new FakeFetcher([
             [discordMessage({ id: "1002" }), discordMessage({ id: "1001" })],