Serialize walks and backfill, split the ETA into start and run

media-scrape/src/backfill.ts

@@ -57,6 +57,12 @@ export interface BackfillJobStatus {
     perMinute: number;
     /** Estimated ms to clear the pending rows. Null when there is nothing to do. */
     etaMs: number | null;
+    /** Ms until this job starts, waiting for the jobs ahead of it. Null when not scheduled. */
+    etaStartMs: number | null;
+    /** Ms this job needs once it starts. */
+    etaRunMs: number | null;
+    /** Start plus run: when this job is expected to finish. */
+    etaTotalMs: number | null;
 }
 
 export interface BackfillTotals {
@@ -68,6 +74,9 @@ export interface BackfillTotals {
     bytes: number;
     perMinute: number;
     etaMs: number | null;
+    etaStartMs: number | null;
+    etaRunMs: number | null;
+    etaTotalMs: number | null;
     byBackend: Record<BackendName, { pending: number; cached: number; failed: number; total: number }>;
 }
 
@@ -107,6 +116,8 @@ const defaultSleep = (ms: number) => new Promise<void>(resolve => setTimeout(res
 
 export class BackfillQueue {
     private readonly jobs = new Map<string, Job>();
+    /** Start order: jobs claimed by the scheduler in this sequence, smallest first at start(). */
+    private readonly order: string[] = [];
     private active: Promise<void> = Promise.resolve();
     private limitedUntil = 0;
 
@@ -217,10 +228,32 @@ export class BackfillQueue {
                 startedAt: tracked?.startedAt ? new Date(tracked.startedAt).toISOString() : null,
                 finishedAt: tracked?.finishedAt ? new Date(tracked.finishedAt).toISOString() : null,
                 perMinute: Math.round(perMinute * 10) / 10,
-                etaMs
+                etaMs,
+                etaStartMs: null,
+                etaRunMs: null,
+                etaTotalMs: null
             });
         }
 
+        // Waiting times follow the execution order: each job starts after the ones ahead of it.
+        const executionOrder = [...this.order];
+        for (const job of jobs) {
+            if (!executionOrder.includes(job.chatId)) executionOrder.push(job.chatId);
+        }
+
+        let ahead = 0;
+        for (const chatId of executionOrder) {
+            const job = jobs.find(candidate => candidate.chatId === chatId);
+            if (!job) continue;
+
+            const scheduled = job.state === "running" || job.state === "queued" || job.state === "paused";
+            const runMs = job.pending > 0 ? (job.pending / Math.max(0.01, job.perMinute)) * 60_000 : 0;
+            job.etaStartMs = scheduled ? Math.round(ahead) : null;
+            job.etaRunMs = Math.round(runMs);
+            job.etaTotalMs = job.etaStartMs === null ? null : Math.round(ahead + runMs);
+            if (scheduled) ahead += runMs;
+        }
+
         jobs.sort((a, b) => b.pending - a.pending);
 
         // Totals span every scrape on disk, not just the jobs the list shows.
@@ -268,6 +301,9 @@ export class BackfillQueue {
                 bytes,
                 perMinute: Math.round(rate * 10) / 10,
                 etaMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
+                etaStartMs: pending > 0 ? 0 : null,
+                etaRunMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
+                etaTotalMs: pending > 0 && rate > 0 ? Math.round((pending / rate) * 60_000) : null,
                 byBackend
             },
             settings,
@@ -318,6 +354,7 @@ export class BackfillQueue {
             tracked.startedAt = this.now();
             tracked.finishedAt = null;
             this.jobs.set(job.chatId, tracked);
+            if (!this.order.includes(job.chatId)) this.order.push(job.chatId);
             this.options.log?.(`backfill: started ${job.chatId} (${job.pending} pending)`);
         }
 
@@ -350,7 +387,11 @@ export class BackfillQueue {
     private schedule(): void {
         this.active = this.active.then(async () => {
             for (;;) {
-                const job = [...this.jobs.values()].find(
+                const ordered = this.order
+                    .map(chatId => this.jobs.get(chatId))
+                    .filter((candidate): candidate is Job => candidate !== undefined);
+                const rest = [...this.jobs.values()].filter(candidate => !this.order.includes(candidate.chatId));
+                const job = [...ordered, ...rest].find(
                     candidate => candidate.state === "running" && !candidate.claimed
                 );
                 if (!job) return;
@@ -383,6 +424,13 @@ export class BackfillQueue {
                 this.limitedUntil = 0;
             }
 
+            // A walk holds the media-traffic lock for its whole run. Wait it out without
+            // blocking a pause request.
+            while (!job.stop && this.engine.isWalking()) {
+                await this.sleep(5_000);
+            }
+            if (job.stop) break;
+
             const settings = this.options.getSettings();
             const result: PreviewBatchResult = await this.engine.previewBatch(
                 member.scrapeId,