Continue new scrapes where the last complete one started

2026-09-21 10:45:30 +02:00
parent 3bba051971
commit 816939288c
3 changed files with 77 additions and 8 deletions
@@ -632,8 +632,32 @@ export class MediaService {

        try {
            const mode = input.mode ?? "links";
-            const afterId = input.after ? snowflakeFromIso(input.after) : null;
-            const beforeId = input.before ? snowflakeFromIso(input.before) : null;
+            let afterId = input.after ? snowflakeFromIso(input.after) : null;
+            let beforeId = input.before ? snowflakeFromIso(input.before) : null;
+            let fromDate = input.after ?? null;
+
+            // Continuing a channel: stop where the best completed scrape of it starts, so a new
+            // walk only covers what is newer instead of re-fetching the whole history.
+            if (!afterId && !beforeId) {
+                const previous = this.previousCompleteScrape(input.channelId);
+                if (previous) {
+                    const store = new ScrapeStore(scrapeDir(this.rootDir, previous.id));
+                    try {
+                        const summary = store.dbSummary();
+                        if (summary.newestId) {
+                            afterId = summary.newestId;
+                            fromDate = summary.newest;
+                            this.deps.log?.(
+                                `continuing ${input.channelId} after ${summary.newestId} ` +
+                                    `(from scrape ${previous.id})`
+                            );
+                        }
+                    } finally {
+                        store.close();
+                    }
+                }
+            }
+
            let backend: ScrapeBackend;

            if (input.backend === "puppet") {
@@ -672,7 +696,7 @@ export class MediaService {
                downloadDir: input.downloadDir ?? null,
                afterId,
                beforeId,
-                fromDate: input.after ?? null,
+                fromDate,
                toDate: input.before ?? null,
                estimatedMessages
            });
@@ -747,6 +771,22 @@ export class MediaService {
        return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null;
    }

+    /** The completed scrape of a channel that walked the most, if any. */
+    private previousCompleteScrape(channelId: string): ScrapeMeta | null {
+        const candidates = this.engine
+            .list()
+            .filter(
+                meta =>
+                    meta.channelId === channelId &&
+                    meta.state === "done" &&
+                    meta.messages > 0 &&
+                    meta.mode === "links"
+            );
+
+        if (!candidates.length) return null;
+        return candidates.sort((a, b) => b.messages - a.messages)[0] ?? null;
+    }
+
    /** Continue a paused scrape from its cursor with the same backend family. */
    async resumeScrape(scrapeId: string): Promise<ScrapeMeta | null> {
        const meta = this.engine.get(scrapeId);