diff --git a/docs/media-scraper.md b/docs/media-scraper.md index c44a929..6921cc3 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -158,9 +158,10 @@ Previews run after every walk: before their previews were cached. One job per chat groups every paused, done, or failed scrape of that chat, so a paused-and-resumed scrape stays in one job. Jobs run one at a time, in batches, and report pending/cached/failed counts, per-kind pending, throughput, and an ETA. The - Bot tab drives them: **Start backfill**, per-job Start/Pause, and **Pause all**. On daemon boot - jobs start automatically when the Backfilling setting **Start automatically on daemon boot** is - on. + Bot tab drives them: **Start backfill**, per-job Start/Pause, and **Pause all**. Jobs with the + least pending work start first, so small chats clear instead of waiting behind a long one; a job + that is started but waiting for the queue reads `queued`. On daemon boot jobs start automatically + when the Backfilling setting **Start automatically on daemon boot** is on. - Transient fetch failures keep the row pending and count up `preview_attempts`. After the **Tries before giving up** setting (default 3) the row is marked `unreachable after N attempt(s)`, so dead hosts (Gfycat, Streamable) stop burning batches forever. diff --git a/media-scrape/src/backfill.ts b/media-scrape/src/backfill.ts index 9be3a99..80ffd70 100644 --- a/media-scrape/src/backfill.ts +++ b/media-scrape/src/backfill.ts @@ -40,7 +40,8 @@ export interface BackfillMember { export interface BackfillJobStatus { chatId: string; guildId: string | null; - state: "idle" | "running" | "paused" | "done"; + /** `queued` means started, but another job holds the queue. */ + state: "idle" | "queued" | "running" | "paused" | "done"; members: BackfillMember[]; pending: number; cached: number; @@ -188,6 +189,8 @@ export class BackfillQueue { const groups = this.plan(); const jobs: BackfillJobStatus[] = []; + const anyClaimed = [...this.jobs.values()].some(candidate => candidate.claimed); + for (const [chatId, group] of groups) { // Hide chats that are fully backfilled unless their job did work this session. const tracked = this.jobs.get(chatId); @@ -196,11 +199,13 @@ export class BackfillQueue { const perMinute = tracked?.perMinute ?? this.estimatedPerMinute(settings, gaps); const etaMs = pending > 0 && perMinute > 0 ? Math.round((pending / perMinute) * 60_000) : null; + const state = + tracked?.state === "running" && anyClaimed && !tracked.claimed ? "queued" : (tracked?.state ?? "idle"); jobs.push({ chatId, guildId: group.guildId, - state: tracked?.state ?? "idle", + state, members: group.members, pending, cached: group.members.reduce((sum, member) => sum + member.cached, 0), @@ -280,11 +285,14 @@ export class BackfillQueue { return perRow > 0 ? 60_000 / perRow : 6000; } - /** Start every job with pending rows, or one chat's job. */ + /** Start every job with pending rows, or one chat's job. Smallest jobs first: they finish. */ start(chatId?: string): void { const status = this.list(); + const order = chatId + ? status.jobs + : [...status.jobs].sort((a, b) => a.pending - b.pending); - for (const job of status.jobs) { + for (const job of order) { if (chatId && job.chatId !== chatId) continue; if (job.pending <= 0) continue; diff --git a/poc/public/media.css b/poc/public/media.css index e351340..bc80ecd 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -740,3 +740,8 @@ color: color-mix(in srgb, var(--color-neutral-400) 55%, transparent); font-style: italic; } + +.vk-bot-state-queued { + background: color-mix(in srgb, var(--color-neutral-400) 18%, transparent); + color: var(--color-neutral-300); +} diff --git a/poc/public/media.js b/poc/public/media.js index 34b4622..da17c23 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -615,7 +615,7 @@ row.append(progress); const actions = el("div", "vk-bot-job-actions"); - if (job.state === "running") { + if (job.state === "running" || job.state === "queued") { actions.append(button("Pause", () => void backfillAction("pause", job.chatId))); } else if (job.pending > 0) { actions.append(button("Start", () => void backfillAction("start", job.chatId)));