Add message counting and date-bounded scrapes

media-scrape/src/engine.ts

@@ -95,6 +95,11 @@ export class ScrapeEngine {
         mode: ScrapeMode;
         backend: ScrapeBackend;
         downloadDir?: string | null;
+        afterId?: string | null;
+        beforeId?: string | null;
+        fromDate?: string | null;
+        toDate?: string | null;
+        estimatedMessages?: number | null;
     }): Promise<ScrapeMeta> {
         const now = this.now();
         const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
@@ -108,7 +113,7 @@ export class ScrapeEngine {
             backend: input.backend.name,
             mode: input.mode,
             state: "queued",
-            cursor: null,
+            cursor: input.beforeId ?? null,
             createdAt: timestamp,
             updatedAt: timestamp,
             pages: 0,
@@ -116,7 +121,12 @@ export class ScrapeEngine {
             media: emptyMediaCounts(),
             errors: 0,
             lastError: null,
-            downloadDir: input.downloadDir ?? null
+            downloadDir: input.downloadDir ?? null,
+            afterId: input.afterId ?? null,
+            beforeId: input.beforeId ?? null,
+            fromDate: input.fromDate ?? null,
+            toDate: input.toDate ?? null,
+            estimatedMessages: input.estimatedMessages ?? null
         };
 
         writeScrapeMeta(dir, meta);
@@ -217,15 +227,25 @@ export class ScrapeEngine {
                     break;
                 }
 
-                const added = store.insertPage(page.messages);
+                // The from-date is a lower ID boundary: trim anything older and finish.
+                const kept = meta.afterId
+                    ? page.messages.filter(message => BigInt(message.id) > BigInt(meta.afterId as string))
+                    : page.messages;
+
+                if (kept.length === 0) {
+                    meta.state = "done";
+                    break;
+                }
+
+                const added = store.insertPage(kept);
                 for (const kind of MEDIA_KINDS) meta.media[kind] += added[kind];
                 meta.pages++;
-                meta.messages += page.messages.length;
-                meta.cursor = oldestId(page.messages) ?? meta.cursor;
+                meta.messages += kept.length;
+                meta.cursor = oldestId(kept) ?? meta.cursor;
                 this.pageTimes.push(this.now());
                 this.persist(dir, meta);
 
-                if (page.messages.length < pageSize) {
+                if (kept.length < page.messages.length || page.messages.length < pageSize) {
                     meta.state = "done";
                     break;
                 }