From 816939288cb00fcc68fb1e6c17acd46dc14e9de0 Mon Sep 17 00:00:00 2001
From: Fisto <fisto@fisto.cccp>
Date: Mon, 21 Sep 2026 10:45:30 +0200
Subject: [PATCH] Continue new scrapes where the last complete one started

---
 media-scrape/src/store.ts | 22 ++++++++++++++-----
 media-scrape/test/run.ts  | 17 +++++++++++++++
 poc/src/media.ts          | 46 ++++++++++++++++++++++++++++++++++++---
 3 files changed, 77 insertions(+), 8 deletions(-)

diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts
index b2ffbde..6543bcc 100644
--- a/media-scrape/src/store.ts
+++ b/media-scrape/src/store.ts
@@ -657,15 +657,27 @@ export class ScrapeStore {
             .run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
     }
 
-    /** Message span of the walk, for completeness checks. */
-    dbSummary(): { messages: number; oldest: string | null; newest: string | null } {
+    /** Message span of the walk, for completeness checks and incremental scrapes. */
+    dbSummary(): {
+        messages: number;
+        oldest: string | null;
+        newest: string | null;
+        newestId: string | null;
+    } {
         const row = this.db
             .prepare(
-                "SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest FROM messages"
+                "SELECT COUNT(*) AS messages, MIN(timestamp) AS oldest, MAX(timestamp) AS newest, MAX(id) AS newestId FROM messages"
             )
-            .get() as { messages: number; oldest: string | null; newest: string | null } | undefined;
+            .get() as
+            | { messages: number; oldest: string | null; newest: string | null; newestId: string | null }
+            | undefined;
 
-        return { messages: row?.messages ?? 0, oldest: row?.oldest ?? null, newest: row?.newest ?? null };
+        return {
+            messages: row?.messages ?? 0,
+            oldest: row?.oldest ?? null,
+            newest: row?.newest ?? null,
+            newestId: row?.newestId ?? null
+        };
     }
 
     /** Preview state for the whole scrape and per kind, for the backfill UI. */
diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts
index 63a44b8..2c047ad 100644
--- a/media-scrape/test/run.ts
+++ b/media-scrape/test/run.ts
@@ -1437,6 +1437,23 @@ void (async () => {
         reopened.close();
     });
 
+    await test("dbSummary reports the message span and newest id", () => {
+        const dir = join(root, "summary");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+        store.insertPage([
+            message("1800", { timestamp: "2026-09-16T00:00:00.000Z" }),
+            message("1801", { timestamp: "2026-09-17T00:00:00.000Z" })
+        ]);
+
+        const summary = store.dbSummary();
+        assert.equal(summary.messages, 2);
+        assert.equal(summary.oldest, "2026-09-16T00:00:00.000Z");
+        assert.equal(summary.newest, "2026-09-17T00:00:00.000Z");
+        assert.equal(summary.newestId, "1801");
+        store.close();
+    });
+
     await test("message links build for guilds and DMs", () => {
         assert.equal(messageUrlFor("123", "456", "789"), "https://discord.com/channels/123/456/789");
         assert.equal(messageUrlFor(null, "456", "789"), "https://discord.com/channels/@me/456/789");
diff --git a/poc/src/media.ts b/poc/src/media.ts
index 87141bf..1335b35 100644
--- a/poc/src/media.ts
+++ b/poc/src/media.ts
@@ -632,8 +632,32 @@ export class MediaService {
 
         try {
             const mode = input.mode ?? "links";
-            const afterId = input.after ? snowflakeFromIso(input.after) : null;
-            const beforeId = input.before ? snowflakeFromIso(input.before) : null;
+            let afterId = input.after ? snowflakeFromIso(input.after) : null;
+            let beforeId = input.before ? snowflakeFromIso(input.before) : null;
+            let fromDate = input.after ?? null;
+
+            // Continuing a channel: stop where the best completed scrape of it starts, so a new
+            // walk only covers what is newer instead of re-fetching the whole history.
+            if (!afterId && !beforeId) {
+                const previous = this.previousCompleteScrape(input.channelId);
+                if (previous) {
+                    const store = new ScrapeStore(scrapeDir(this.rootDir, previous.id));
+                    try {
+                        const summary = store.dbSummary();
+                        if (summary.newestId) {
+                            afterId = summary.newestId;
+                            fromDate = summary.newest;
+                            this.deps.log?.(
+                                `continuing ${input.channelId} after ${summary.newestId} ` +
+                                    `(from scrape ${previous.id})`
+                            );
+                        }
+                    } finally {
+                        store.close();
+                    }
+                }
+            }
+
             let backend: ScrapeBackend;
 
             if (input.backend === "puppet") {
@@ -672,7 +696,7 @@ export class MediaService {
                 downloadDir: input.downloadDir ?? null,
                 afterId,
                 beforeId,
-                fromDate: input.after ?? null,
+                fromDate,
                 toDate: input.before ?? null,
                 estimatedMessages
             });
@@ -747,6 +771,22 @@ export class MediaService {
         return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null;
     }
 
+    /** The completed scrape of a channel that walked the most, if any. */
+    private previousCompleteScrape(channelId: string): ScrapeMeta | null {
+        const candidates = this.engine
+            .list()
+            .filter(
+                meta =>
+                    meta.channelId === channelId &&
+                    meta.state === "done" &&
+                    meta.messages > 0 &&
+                    meta.mode === "links"
+            );
+
+        if (!candidates.length) return null;
+        return candidates.sort((a, b) => b.messages - a.messages)[0] ?? null;
+    }
+
     /** Continue a paused scrape from its cursor with the same backend family. */
     async resumeScrape(scrapeId: string): Promise<ScrapeMeta | null> {
         const meta = this.engine.get(scrapeId);