Add message counting and date-bounded scrapes

media-scrape/src/engine.ts

@@ -95,6 +95,11 @@ export class ScrapeEngine {
         mode: ScrapeMode;
         backend: ScrapeBackend;
         downloadDir?: string | null;
+        afterId?: string | null;
+        beforeId?: string | null;
+        fromDate?: string | null;
+        toDate?: string | null;
+        estimatedMessages?: number | null;
     }): Promise<ScrapeMeta> {
         const now = this.now();
         const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
@@ -108,7 +113,7 @@ export class ScrapeEngine {
             backend: input.backend.name,
             mode: input.mode,
             state: "queued",
-            cursor: null,
+            cursor: input.beforeId ?? null,
             createdAt: timestamp,
             updatedAt: timestamp,
             pages: 0,
@@ -116,7 +121,12 @@ export class ScrapeEngine {
             media: emptyMediaCounts(),
             errors: 0,
             lastError: null,
-            downloadDir: input.downloadDir ?? null
+            downloadDir: input.downloadDir ?? null,
+            afterId: input.afterId ?? null,
+            beforeId: input.beforeId ?? null,
+            fromDate: input.fromDate ?? null,
+            toDate: input.toDate ?? null,
+            estimatedMessages: input.estimatedMessages ?? null
         };
 
         writeScrapeMeta(dir, meta);
@@ -217,15 +227,25 @@ export class ScrapeEngine {
                     break;
                 }
 
-                const added = store.insertPage(page.messages);
+                // The from-date is a lower ID boundary: trim anything older and finish.
+                const kept = meta.afterId
+                    ? page.messages.filter(message => BigInt(message.id) > BigInt(meta.afterId as string))
+                    : page.messages;
+
+                if (kept.length === 0) {
+                    meta.state = "done";
+                    break;
+                }
+
+                const added = store.insertPage(kept);
                 for (const kind of MEDIA_KINDS) meta.media[kind] += added[kind];
                 meta.pages++;
-                meta.messages += page.messages.length;
-                meta.cursor = oldestId(page.messages) ?? meta.cursor;
+                meta.messages += kept.length;
+                meta.cursor = oldestId(kept) ?? meta.cursor;
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
-                if (page.messages.length < pageSize) {
+                if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
                 }

media-scrape/src/index.ts

@@ -5,6 +5,7 @@
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
 export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
+export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
 export {
     ASPECT_PRESETS,
     RESOLUTION_PRESETS,

media-scrape/src/snowflake.ts

@@ -0,0 +1,21 @@
+/*
+ * Discord snowflake helpers for date-bounded scrapes.
+ */
+
+const DISCORD_EPOCH = 1420070400000;
+
+/** The snowflake at an instant. Null for an unparsable date; "0" before the epoch. */
+export function snowflakeFromIso(iso: string): string | null {
+    const time = Date.parse(iso);
+    if (!Number.isFinite(time)) return null;
+
+    const value = BigInt(time - DISCORD_EPOCH) << 22n;
+    return value > 0n ? value.toString() : "0";
+}
+
+/** The instant a snowflake encodes, as an ISO string. Null when it is not a snowflake. */
+export function isoFromSnowflake(id: string | null | undefined): string | null {
+    if (!id || !/^\d{17,20}$/.test(id)) return null;
+    const time = Number(BigInt(id) >> 22n) + DISCORD_EPOCH;
+    return Number.isFinite(time) ? new Date(time).toISOString() : null;
+}

media-scrape/src/types.ts

@@ -96,6 +96,16 @@ export interface ScrapeMeta {
     finishedAt?: string | null;
     /** Absolute destination for downloaded files. Null means the local `media/` folder. */
     downloadDir?: string | null;
+    /** Lower message-ID boundary (exclusive), derived from the from-date. */
+    afterId?: string | null;
+    /** Upper message-ID boundary; the walk starts just below it. */
+    beforeId?: string | null;
+    /** From-date as ISO, for display in history. */
+    fromDate?: string | null;
+    /** To-date as ISO, for display in history. */
+    toDate?: string | null;
+    /** Discord search total for the range, when known (slopbot only). */
+    estimatedMessages?: number | null;
 }
 
 export function emptyMediaCounts(): Record<MediaKind, number> {