Add the Bot-tab backfill control room and media strip rules

This commit is contained in:
Fisto
2026-09-17 09:04:27 +02:00
parent 16ce809270
commit 6ae2ff5ab3
14 changed files with 949 additions and 85 deletions
+199
View File
@@ -541,3 +541,202 @@
color: #fff;
font: 600 9px/1 ui-monospace, Menlo, monospace;
}
/* Bot tab: backfill jobs, stats, and settings. */
.vk-media-bot {
display: flex;
flex-direction: column;
gap: 12px;
flex: 1;
min-height: 0;
overflow-y: auto;
}
.vk-media-bot[hidden] {
display: none;
}
.vk-bot-limited {
padding: 8px 10px;
border-radius: 6px;
font: 500 11px/1.4 ui-monospace, Menlo, monospace;
color: #ffb0b0;
background: color-mix(in srgb, #ff5555 14%, transparent);
border: 1px solid color-mix(in srgb, #ff5555 35%, transparent);
}
.vk-bot-limited[hidden] {
display: none;
}
.vk-bot-section {
border: 1px solid var(--color-divider);
border-radius: 6px;
padding: 8px 10px;
background: var(--color-surface);
}
.vk-bot-head {
display: flex;
align-items: center;
justify-content: space-between;
gap: 8px;
margin-bottom: 6px;
}
.vk-bot-title {
font-weight: 600;
font-size: 12px;
}
.vk-bot-actions {
display: flex;
gap: 6px;
}
.vk-bot-summary {
display: flex;
flex-direction: column;
gap: 2px;
margin-bottom: 8px;
}
.vk-bot-summary-line {
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-300);
}
.vk-bot-jobs {
display: flex;
flex-direction: column;
gap: 6px;
}
.vk-bot-empty {
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-500);
}
.vk-bot-job {
border: 1px solid var(--color-neutral-800);
border-radius: 5px;
padding: 6px 8px;
display: flex;
flex-direction: column;
gap: 4px;
}
.vk-bot-job-running {
border-color: color-mix(in srgb, var(--color-accent, #4f8cff) 45%, transparent);
}
.vk-bot-job-head {
display: flex;
align-items: center;
gap: 8px;
flex-wrap: wrap;
}
.vk-bot-job-title {
flex: 1;
font-weight: 600;
font-size: 11px;
}
.vk-bot-job-meta {
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
color: var(--color-neutral-400);
}
.vk-bot-state {
font: 600 10px/1.4 ui-monospace, Menlo, monospace;
text-transform: uppercase;
padding: 0 5px;
border-radius: 3px;
background: var(--color-neutral-800);
color: var(--color-neutral-300);
}
.vk-bot-state-running {
background: color-mix(in srgb, var(--color-accent, #4f8cff) 25%, transparent);
color: var(--color-accent-300, #9fc1ff);
}
.vk-bot-state-paused {
background: color-mix(in srgb, #ffaa33 20%, transparent);
color: #ffc880;
}
.vk-bot-state-done {
background: color-mix(in srgb, #44cc88 20%, transparent);
color: #8fe0b8;
}
.vk-bot-job-member {
font: 400 10px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-400);
word-break: break-all;
}
.vk-bot-progress {
height: 4px;
border-radius: 2px;
background: var(--color-neutral-800);
overflow: hidden;
}
.vk-bot-progress-fill {
height: 100%;
background: var(--color-accent-400, #4f8cff);
}
.vk-bot-job-actions {
display: flex;
gap: 6px;
}
.vk-bot-settings {
display: flex;
flex-direction: column;
gap: 6px;
}
.vk-bot-group-title {
cursor: pointer;
font: 600 11px/1.6 ui-monospace, Menlo, monospace;
color: var(--color-neutral-300);
}
.vk-bot-group-body {
display: flex;
flex-direction: column;
gap: 4px;
padding: 6px 0 2px 12px;
}
.vk-bot-field {
display: flex;
align-items: center;
justify-content: space-between;
gap: 10px;
}
.vk-bot-field-label {
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
color: var(--color-neutral-400);
}
.vk-bot-save {
align-self: flex-start;
margin-top: 4px;
}
.vk-media-limited {
color: #ff8080;
}
.vk-media-done-note {
color: color-mix(in srgb, var(--color-neutral-400) 55%, transparent);
font-style: italic;
}
+388 -9
View File
@@ -35,7 +35,11 @@
zoom: Number(localStorage.getItem("vk-media-zoom") ?? 50),
draft: null,
prefetched: null,
pageSize: 60
pageSize: 60,
backfill: null,
preferences: null,
/** Scrapes that finished while this page was open, until the tab is reloaded. */
completed: new Set()
};
const overlay = document.createElement("div");
@@ -52,6 +56,35 @@
<div id="media-status" class="vk-media-status"></div>
<div id="media-scrapes" class="vk-media-scrapes"></div>
<div id="media-grid" class="vk-media-grid"></div>
<div id="media-bot" class="vk-media-bot" hidden>
<div id="bot-limited" class="vk-bot-limited" hidden></div>
<div class="vk-bot-section">
<div class="vk-bot-head">
<span class="vk-bot-title">Backfill</span>
<span class="vk-bot-actions">
<button id="backfill-start" class="btn btn-primary vk-small-btn" type="button">
Start backfill
</button>
<button id="backfill-pause" class="btn btn-secondary vk-small-btn" type="button">
Pause all
</button>
</span>
</div>
<div id="backfill-summary" class="vk-bot-summary"></div>
<div id="backfill-jobs" class="vk-bot-jobs"></div>
</div>
<div class="vk-bot-section">
<div class="vk-bot-head"><span class="vk-bot-title">Settings</span></div>
<div id="bot-tunables" class="vk-bot-settings"></div>
</div>
<div class="vk-bot-section">
<div class="vk-bot-head"><span class="vk-bot-title">Errors</span></div>
<div id="bot-errors" class="vk-bot-errors"></div>
</div>
</div>
</div>
`;
document.body.append(overlay);
@@ -61,6 +94,12 @@
const statusLine = overlay.querySelector("#media-status");
const scrapeList = overlay.querySelector("#media-scrapes");
const grid = overlay.querySelector("#media-grid");
const botPanel = overlay.querySelector("#media-bot");
const botLimited = overlay.querySelector("#bot-limited");
const backfillSummary = overlay.querySelector("#backfill-summary");
const backfillJobs = overlay.querySelector("#backfill-jobs");
const tunablesHost = overlay.querySelector("#bot-tunables");
const botErrors = overlay.querySelector("#bot-errors");
const filterDrawer = el("div", "vk-media-drawer");
filterDrawer.hidden = true;
@@ -177,12 +216,27 @@
}
async function loadScrapes() {
const wasActive = new Set(
state.scrapes.filter(scrape => scrape.state === "running" || scrape.state === "queued").map(s => s.id)
);
try {
state.scrapes = await window.VK.api.get("/api/media/scrapes");
state.status = await window.VK.api.get("/api/media/status");
} catch (error) {
setStatus(`media status failed: ${error.message}`);
}
// A scrape that was running and is now done counts once for the strip notice.
const ids = new Set(state.scrapes.map(scrape => scrape.id));
for (const scrape of state.scrapes) {
if (wasActive.has(scrape.id) && scrape.state === "done") state.completed.add(scrape.id);
}
state.completed = new Set([...state.completed].filter(id => ids.has(id)));
const dismissed = prefs.dismissed.filter(id => ids.has(id));
if (dismissed.length !== prefs.dismissed.length) prefs.setDismissed(dismissed);
renderScrapes();
}
@@ -327,11 +381,23 @@
function renderScrapes() {
scrapeList.replaceChildren();
// Only live jobs: a paused scrape is not running and belongs in History, not the strip.
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
for (const scrape of active) {
const row = el("div", "vk-media-scrape");
// The strip shows live jobs plus the newest paused scrape. Stop on a paused one only
// hides it here; it stays resumable, and the next paused scrape can take its place.
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
const paused = state.scrapes
.filter(scrape => scrape.state === "paused" && !prefs.dismissed.includes(scrape.id))
.sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : -1))
.slice(0, 1);
const limitedUntil = state.status?.limitedUntil ? new Date(state.status.limitedUntil).getTime() : 0;
const limited = limitedUntil > Date.now();
const limitedText = limited
? ` · API limited, retry in ${Math.max(0, Math.ceil((limitedUntil - Date.now()) / 1000))}s`
: "";
for (const scrape of [...active, ...paused]) {
const row = el("div", `vk-media-scrape${limited ? " vk-media-limited" : ""}`);
const media = scrape.media ?? {};
if (scrape.state === "running" || scrape.state === "queued") {
@@ -354,12 +420,32 @@
`${channelLabel(scrape.channelId)} · ${scrape.state} · ${pageText}${percent} · ${scrape.messages}m · ` +
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0}` +
(scrape.downloaded ? ` · files ${scrape.downloaded}` : "") +
(scrape.previewed ? ` · previews ${scrape.previewed}` : "")
(scrape.previewed ? ` · previews ${scrape.previewed}` : "") +
limitedText
)
);
if (scrape.state === "paused") {
row.append(
button("Resume", async () => {
try {
await window.VK.api.post("/api/media/scrape/resume", { id: scrape.id });
} catch (error) {
setStatus(`resume failed: ${error.message}`);
}
void loadScrapes();
})
);
}
row.append(
button("Stop", async () => {
if (scrape.state === "paused") {
// Hidden here, still resumable in the Bot tab.
prefs.setDismissed([...prefs.dismissed, scrape.id]);
renderScrapes();
return;
}
await window.VK.api.post("/api/media/scrape/stop", { id: scrape.id });
void loadScrapes();
})
@@ -367,6 +453,285 @@
scrapeList.append(row);
}
// Session notice only: it clears on reload, not because the data changed.
if (state.completed.size) {
scrapeList.append(
el(
"div",
"vk-media-scrape vk-media-done-note",
`${state.completed.size} ${state.completed.size === 1 ? "scrape" : "scrapes"} completed`
)
);
}
}
const prefs = {
get dismissed() {
try {
const raw = JSON.parse(localStorage.getItem("vk-media-dismissed") ?? "[]");
return Array.isArray(raw) ? raw.map(String) : [];
} catch {
return [];
}
},
setDismissed(ids) {
localStorage.setItem("vk-media-dismissed", JSON.stringify([...new Set(ids.map(String))]));
}
};
// ---- Backfill (Bot tab) ----
const TUNABLE_GROUPS = {
backfill: {
title: "Backfilling",
fields: [
["batchSize", "Batch size (rows)", 1, 500, 1],
["batchPauseMs", "Cooldown between batches (ms)", 0, 600000, 100],
["maxAttempts", "Tries before giving up", 1, 100, 1],
["previewMinGapMs", "Preview gap min (ms)", 0, 60000, 10],
["previewMaxGapMs", "Preview gap max (ms)", 0, 60000, 10],
["maxBytes", "Max preview size (MiB)", 0.0625, 2048, 0.5, "mib"],
["auto", "Start automatically on daemon boot", null, null, null, "bool"]
]
},
scrape: {
title: "Scraping",
fields: [
["walkCyclePages", "Preview every N walk pages", 0, 100, 1],
["maxFileBytes", "Max file size (MiB)", 1, 8192, 1, "mib"],
["downloadMinGapMs", "Download gap min (ms)", 0, 600000, 100],
["downloadMaxGapMs", "Download gap max (ms)", 0, 600000, 100],
["maxRetries", "Page fetch retries", 0, 10, 1]
]
},
timers: {
title: "Timers",
fields: [
["pageMinMs", "Walk page gap min (ms)", 0, 600000, 100],
["pageMaxMs", "Walk page gap max (ms)", 0, 600000, 100],
["rateLimitPauseMs", "Rate-limit cooldown (ms)", 0, 3600000, 5000]
]
}
};
function formatEta(ms) {
if (ms === null || ms === undefined || !Number.isFinite(ms)) return "\u2014";
const seconds = Math.max(0, Math.round(ms / 1000));
if (seconds < 90) return `${seconds}s`;
const minutes = Math.round(seconds / 60);
if (minutes < 90) return `${minutes}m`;
const hours = Math.floor(minutes / 60);
const rest = minutes % 60;
return rest ? `${hours}h ${rest}m` : `${hours}h`;
}
function pendingKinds(member) {
const by = member.pendingByKind ?? {};
return ["video", "image", "gif", "audio"]
.filter(kind => by[kind] > 0)
.map(kind => `${by[kind]} ${kind}`)
.join(" \u00b7 ");
}
function renderBackfill() {
const status = state.backfill;
if (!status) return;
const until = status.limitedUntil ? new Date(status.limitedUntil).getTime() : 0;
const limited = until > Date.now();
botLimited.hidden = !limited;
if (limited) {
botLimited.textContent =
`API timed out, increase timers \u2014 media traffic paused for ` +
formatEta(until - Date.now());
}
const totals = status.totals ?? {};
const backend = totals.byBackend ?? {};
const bot = backend.slopbot ?? { pending: 0, cached: 0, failed: 0, total: 0 };
const puppet = backend.puppet ?? { pending: 0, cached: 0, failed: 0, total: 0 };
backfillSummary.replaceChildren(
el(
"div",
"vk-bot-summary-line",
`${totals.pending ?? 0} pending \u00b7 ${totals.cached ?? 0} cached \u00b7 ` +
`${totals.failed ?? 0} failed \u00b7 ${totals.downloaded ?? 0} downloaded \u00b7 ` +
`${formatBytes(totals.bytes ?? 0)} stored`
),
el(
"div",
"vk-bot-summary-line",
`ETA ${formatEta(totals.etaMs)} at ${status.estimatedPerMinute ?? 0} rows/min ` +
`(current settings)`
),
el(
"div",
"vk-bot-summary-line vk-dim",
`Bot: ${bot.pending} pending, ${bot.cached} cached, ${bot.failed} failed \u00b7 ` +
`Puppet: ${puppet.pending} pending, ${puppet.cached} cached, ${puppet.failed} failed`
)
);
backfillJobs.replaceChildren();
const jobs = (status.jobs ?? []).filter(job => job.pending > 0 || job.state !== "idle");
if (!jobs.length) {
backfillJobs.append(el("div", "vk-bot-empty", "Nothing waiting to backfill."));
return;
}
for (const job of jobs) {
const row = el("div", `vk-bot-job vk-bot-job-${job.state}`);
const head = el("div", "vk-bot-job-head");
head.append(el("span", "vk-bot-job-title", channelLabel(job.chatId)));
head.append(el("span", `vk-bot-state vk-bot-state-${job.state}`, job.state));
const parts = [`${job.pending} pending`];
if (job.cached) parts.push(`${job.cached} cached`);
if (job.failed) parts.push(`${job.failed} failed`);
if (job.pending > 0) parts.push(`ETA ${formatEta(job.etaMs)}`);
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
row.append(head);
for (const member of job.members) {
if (member.pending === 0 && member.state !== "running") continue;
const kinds = pendingKinds(member);
row.append(
el(
"div",
"vk-bot-job-member",
`${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` +
`${member.backend} \u00b7 ${member.pending} pending` +
(kinds ? ` (${kinds})` : "")
)
);
}
const progress = el("div", "vk-bot-progress");
const fill = el("div", "vk-bot-progress-fill");
fill.style.width = `${Math.min(100, Math.round((job.cached / Math.max(1, job.total)) * 100))}%`;
progress.append(fill);
row.append(progress);
const actions = el("div", "vk-bot-job-actions");
if (job.state === "running") {
actions.append(button("Pause", () => void backfillAction("pause", job.chatId)));
} else if (job.pending > 0) {
actions.append(button("Start", () => void backfillAction("start", job.chatId)));
}
row.append(actions);
backfillJobs.append(row);
}
}
async function backfillAction(action, chatId) {
try {
state.backfill = await window.VK.api.post(
`/api/media/backfill/${action}`,
chatId ? { chatId } : {}
);
renderBackfill();
} catch (error) {
setStatus(`backfill ${action} failed: ${error.message}`);
}
}
async function refreshBackfill() {
try {
state.backfill = await window.VK.api.get("/api/media/backfill");
} catch {
state.backfill = null;
}
renderBackfill();
}
function tunableValue(raw, key, scale) {
const value = raw?.[key];
if (scale === "bool") return value !== false;
const number = Number(value);
if (!Number.isFinite(number)) return null;
return scale === "mib" ? Math.round((number / 1024 ** 2) * 100) / 100 : number;
}
function renderTunables() {
const tunables = state.preferences?.media?.tunables;
if (!tunables) return;
tunablesHost.replaceChildren();
for (const [group, spec] of Object.entries(TUNABLE_GROUPS)) {
const details = el("details", "vk-bot-group");
details.open = group === "backfill";
details.append(el("summary", "vk-bot-group-title", spec.title));
const body = el("div", "vk-bot-group-body");
for (const [key, label, min, max, step, scale] of spec.fields) {
const row = el("label", "vk-bot-field");
row.append(el("span", "vk-bot-field-label", label));
const input = document.createElement("input");
input.dataset.group = group;
input.dataset.key = key;
input.dataset.scale = scale ?? "";
if (scale === "bool") {
input.type = "checkbox";
input.checked = tunables[group]?.[key] !== false;
} else {
input.type = "number";
input.className = "vk-media-input";
input.min = String(min);
input.max = String(max);
input.step = String(step);
input.value = String(tunableValue(tunables[group] ?? {}, key, scale) ?? "");
}
row.append(input);
body.append(row);
}
const save = button("Save", () => void saveTunables(group));
save.classList.add("vk-bot-save");
body.append(save);
details.append(body);
tunablesHost.append(details);
}
}
async function saveTunables(group) {
const patch = {};
for (const input of tunablesHost.querySelectorAll(`input[data-group="${group}"]`)) {
const scale = input.dataset.scale;
if (scale === "bool") {
patch[input.dataset.key] = input.checked;
continue;
}
const value = Number(input.value);
if (!Number.isFinite(value)) continue;
patch[input.dataset.key] = scale === "mib" ? Math.round(value * 1024 ** 2) : Math.round(value);
}
try {
await window.VK.api.post("/api/preferences", { media: { tunables: { [group]: patch } } });
setStatus(`${TUNABLE_GROUPS[group].title.toLowerCase()} settings saved`);
await refreshPreferences();
void refreshBackfill();
} catch (error) {
setStatus(`save failed: ${error.message}`);
}
}
async function refreshPreferences() {
try {
state.preferences = await window.VK.api.get("/api/preferences");
} catch {
return;
}
renderTunables();
}
// ---- Bot errors ----
@@ -387,7 +752,7 @@
}
function renderErrors() {
scrapeList.replaceChildren();
botErrors.replaceChildren();
const head = el("div", "vk-media-scrape");
head.append(el("span", null, `${(state.errors ?? []).length} error(s)`));
@@ -408,7 +773,7 @@
scrapeList.append(head);
for (const entry of state.errors ?? []) {
scrapeList.append(
botErrors.append(
el(
"div",
"vk-media-scrape vk-media-error",
@@ -417,7 +782,7 @@
);
}
if (!(state.errors ?? []).length) scrapeList.append(el("div", "vk-media-scrape", "No errors."));
if (!(state.errors ?? []).length) botErrors.append(el("div", "vk-media-scrape", "No errors."));
}
async function refreshErrors() {
@@ -972,6 +1337,8 @@
title.textContent = state.view === "bot" ? "Slopbot" : "Media archive";
toolbar.hidden = false;
grid.hidden = state.view === "bot";
botPanel.hidden = state.view !== "bot";
scrapeList.hidden = state.view === "bot";
renderToolbar();
if (state.view === "bot") {
@@ -979,6 +1346,8 @@
statusLine.textContent = bot?.connected
? `connected as ${bot.user} in ${bot.guilds.length} guild(s)`
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
if (!state.preferences) void refreshPreferences();
void refreshBackfill();
void loadScrapes().then(renderScrapes);
void refreshErrors();
} else {
@@ -993,10 +1362,15 @@
overlay.querySelector("#media-close").addEventListener("click", hide);
overlay.querySelector("#media-refresh").addEventListener("click", () => {
state.completed.clear();
void loadScrapes();
if (state.view === "media") void loadMedia(true);
if (state.view === "bot") void refreshBackfill();
});
overlay.querySelector("#backfill-start").addEventListener("click", () => void backfillAction("start"));
overlay.querySelector("#backfill-pause").addEventListener("click", () => void backfillAction("pause"));
const scopeMedia = document.querySelector("#scope-media");
const scopeBot = document.querySelector("#scope-bot");
scopeMedia?.addEventListener("click", () => show("media"));
@@ -1019,6 +1393,11 @@
if (state.view === "media") void loadScrapes();
}, 4000);
// Backfill progress while the Bot tab is open.
setInterval(() => {
if (state.view === "bot") void refreshBackfill();
}, 3000);
// Infinite scroll: fetch the next page before the grid bottom is reached.
grid.addEventListener("scroll", () => {
if (state.loading || !state.loaded) return;