Serialize walks and backfill, split the ETA into start and run

media-scrape/src/backfill.ts

@@ -57,6 +57,12 @@ export interface BackfillJobStatus {
     perMinute: number;
     /** Estimated ms to clear the pending rows. Null when there is nothing to do. */
     etaMs: number | null;
+    /** Ms until this job starts, waiting for the jobs ahead of it. Null when not scheduled. */
+    etaStartMs: number | null;
+    /** Ms this job needs once it starts. */
+    etaRunMs: number | null;
+    /** Start plus run: when this job is expected to finish. */
+    etaTotalMs: number | null;
 }
 
 export interface BackfillTotals {
@@ -68,6 +74,9 @@ export interface BackfillTotals {
     bytes: number;
     perMinute: number;
     etaMs: number | null;
+    etaStartMs: number | null;
+    etaRunMs: number | null;
+    etaTotalMs: number | null;
     byBackend: Record<BackendName, { pending: number; cached: number; failed: number; total: number }>;
 }
 
@@ -107,6 +116,8 @@ const defaultSleep = (ms: number) => new Promise<void>(resolve => setTimeout(res
 
 export class BackfillQueue {
     private readonly jobs = new Map<string, Job>();
+    /** Start order: jobs claimed by the scheduler in this sequence, smallest first at start(). */
+    private readonly order: string[] = [];
     private active: Promise<void> = Promise.resolve();
     private limitedUntil = 0;
 
@@ -217,10 +228,32 @@ export class BackfillQueue {
                 startedAt: tracked?.startedAt ? new Date(tracked.startedAt).toISOString() : null,
                 finishedAt: tracked?.finishedAt ? new Date(tracked.finishedAt).toISOString() : null,
                 perMinute: Math.round(perMinute * 10) / 10,
-                etaMs
+                etaMs,
+                etaStartMs: null,
+                etaRunMs: null,
+                etaTotalMs: null
             });
         }
 
+        // Waiting times follow the execution order: each job starts after the ones ahead of it.
+        const executionOrder = [...this.order];
+        for (const job of jobs) {
+            if (!executionOrder.includes(job.chatId)) executionOrder.push(job.chatId);
+        }
+
+        let ahead = 0;
+        for (const chatId of executionOrder) {
+            const job = jobs.find(candidate => candidate.chatId === chatId);
+            if (!job) continue;
+
+            const scheduled = job.state === "running" || job.state === "queued" || job.state === "paused";
+            const runMs = job.pending > 0 ? (job.pending / Math.max(0.01, job.perMinute)) * 60_000 : 0;
+            job.etaStartMs = scheduled ? Math.round(ahead) : null;
+            job.etaRunMs = Math.round(runMs);
+            job.etaTotalMs = job.etaStartMs === null ? null : Math.round(ahead + runMs);
+            if (scheduled) ahead += runMs;
+        }
+
         jobs.sort((a, b) => b.pending - a.pending);
 
         // Totals span every scrape on disk, not just the jobs the list shows.
@@ -268,6 +301,9 @@ export class BackfillQueue {
                 bytes,
                 perMinute: Math.round(rate * 10) / 10,
                 etaMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
+                etaStartMs: pending > 0 ? 0 : null,
+                etaRunMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
+                etaTotalMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
                 byBackend
             },
             settings,
@@ -318,6 +354,7 @@ export class BackfillQueue {
             tracked.startedAt = this.now();
             tracked.finishedAt = null;
             this.jobs.set(job.chatId, tracked);
+            if (!this.order.includes(job.chatId)) this.order.push(job.chatId);
             this.options.log?.(`backfill: started ${job.chatId} (${job.pending} pending)`);
         }
 
@@ -350,7 +387,11 @@ export class BackfillQueue {
     private schedule(): void {
         this.active = this.active.then(async () => {
             for (;;) {
-                const job = [...this.jobs.values()].find(
+                const ordered = this.order
+                    .map(chatId => this.jobs.get(chatId))
+                    .filter((candidate): candidate is Job => candidate !== undefined);
+                const rest = [...this.jobs.values()].filter(candidate => !this.order.includes(candidate.chatId));
+                const job = [...ordered, ...rest].find(
                     candidate => candidate.state === "running" && !candidate.claimed
                 );
                 if (!job) return;
@@ -383,6 +424,13 @@ export class BackfillQueue {
                 this.limitedUntil = 0;
             }
 
+            // A walk holds the media-traffic lock for its whole run. Wait it out without
+            // blocking a pause request.
+            while (!job.stop && this.engine.isWalking()) {
+                await this.sleep(5_000);
+            }
+            if (job.stop) break;
+
             const settings = this.options.getSettings();
             const result: PreviewBatchResult = await this.engine.previewBatch(
                 member.scrapeId,

media-scrape/src/engine.ts

@@ -116,6 +116,11 @@ export class ScrapeEngine {
     private readonly pageTimes: number[] = [];
     private rateLimitUntil = 0;
     private processingPromise: Promise<void> | null = null;
+    /**
+     * One media-traffic lock: a walk holds it for its whole run, a backfill batch for the batch.
+     * Walks and backfill therefore never fetch at the same time.
+     */
+    private trafficLock: Promise<void> = Promise.resolve();
 
     constructor(private readonly options: ScrapeEngineOptions) {}
 
@@ -186,6 +191,32 @@ export class ScrapeEngine {
         }
     }
 
+    /** True while a walk is draining. The backfill queue waits on this before a batch. */
+    isWalking(): boolean {
+        return this.processingPromise !== null;
+    }
+
+    /** Resolves when every queued walk has finished. */
+    async whenIdle(): Promise<void> {
+        while (this.processingPromise) await this.processingPromise;
+    }
+
+    /**
+     * Take the media-traffic lock. Resolves with the release function. Walks take it for a whole
+     * run; backfill batches for one batch, so the two never overlap and the pacing settings hold
+     * for the total traffic, not per action.
+     */
+    async acquireTraffic(): Promise<() => void> {
+        let release!: () => void;
+        const next = new Promise<void>(resolve => {
+            release = resolve;
+        });
+        const previous = this.trafficLock;
+        this.trafficLock = previous.then(() => next);
+        await previous;
+        return release;
+    }
+
     get(id: string): ScrapeMeta | null {
         const active = this.active.get(id);
         if (active) return { ...active };
@@ -248,12 +279,14 @@ export class ScrapeEngine {
 
         const dir = join(this.options.rootDir, scrapeId);
         const store = new ScrapeStore(dir);
+        const releaseTraffic = await this.acquireTraffic();
         const before = meta.previewed ?? 0;
 
         try {
             await this.runPreviews(store, dir, meta);
         } finally {
             store.close();
+            releaseTraffic();
         }
 
         return (meta.previewed ?? 0) - before;
@@ -283,6 +316,7 @@ export class ScrapeEngine {
         const pageSize = this.options.pageSize ?? 100;
         const perCycle = this.options.download?.previewBatchSize ?? 25;
         let pagesSincePreview = 0;
+        const releaseTraffic = await this.acquireTraffic();
 
         try {
             if (this.stopRequested.has(meta.id)) {
@@ -388,6 +422,7 @@ export class ScrapeEngine {
             this.persist(dir, meta);
             store.close();
             this.active.set(meta.id, meta);
+            releaseTraffic();
         }
     }
 
@@ -514,11 +549,13 @@ export class ScrapeEngine {
         const meta = this.active.get(scrapeId) ?? readScrapeMeta(dir);
         if (!meta) return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, remaining: 0, stopped: false };
 
+        const releaseTraffic = await this.acquireTraffic();
         const store = new ScrapeStore(dir);
         try {
             return await this.previewRows(store, dir, meta, Math.max(1, limit), this.previewOptions(meta));
         } finally {
             store.close();
+            releaseTraffic();
         }
     }