Add the Bot-tab backfill control room and media strip rules
This commit is contained in:
Fisto
+388
-9
@@ -35,7 +35,11 @@
|
||||
zoom: Number(localStorage.getItem("vk-media-zoom") ?? 50),
|
||||
draft: null,
|
||||
prefetched: null,
|
||||
pageSize: 60
|
||||
pageSize: 60,
|
||||
backfill: null,
|
||||
preferences: null,
|
||||
/** Scrapes that finished while this page was open, until the tab is reloaded. */
|
||||
completed: new Set()
|
||||
};
|
||||
|
||||
const overlay = document.createElement("div");
|
||||
@@ -52,6 +56,35 @@
|
||||
<div id="media-status" class="vk-media-status"></div>
|
||||
<div id="media-scrapes" class="vk-media-scrapes"></div>
|
||||
<div id="media-grid" class="vk-media-grid"></div>
|
||||
<div id="media-bot" class="vk-media-bot" hidden>
|
||||
<div id="bot-limited" class="vk-bot-limited" hidden></div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head">
|
||||
<span class="vk-bot-title">Backfill</span>
|
||||
<span class="vk-bot-actions">
|
||||
<button id="backfill-start" class="btn btn-primary vk-small-btn" type="button">
|
||||
Start backfill
|
||||
</button>
|
||||
<button id="backfill-pause" class="btn btn-secondary vk-small-btn" type="button">
|
||||
Pause all
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<div id="backfill-summary" class="vk-bot-summary"></div>
|
||||
<div id="backfill-jobs" class="vk-bot-jobs"></div>
|
||||
</div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head"><span class="vk-bot-title">Settings</span></div>
|
||||
<div id="bot-tunables" class="vk-bot-settings"></div>
|
||||
</div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head"><span class="vk-bot-title">Errors</span></div>
|
||||
<div id="bot-errors" class="vk-bot-errors"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
document.body.append(overlay);
|
||||
@@ -61,6 +94,12 @@
|
||||
const statusLine = overlay.querySelector("#media-status");
|
||||
const scrapeList = overlay.querySelector("#media-scrapes");
|
||||
const grid = overlay.querySelector("#media-grid");
|
||||
const botPanel = overlay.querySelector("#media-bot");
|
||||
const botLimited = overlay.querySelector("#bot-limited");
|
||||
const backfillSummary = overlay.querySelector("#backfill-summary");
|
||||
const backfillJobs = overlay.querySelector("#backfill-jobs");
|
||||
const tunablesHost = overlay.querySelector("#bot-tunables");
|
||||
const botErrors = overlay.querySelector("#bot-errors");
|
||||
|
||||
const filterDrawer = el("div", "vk-media-drawer");
|
||||
filterDrawer.hidden = true;
|
||||
@@ -177,12 +216,27 @@
|
||||
}
|
||||
|
||||
async function loadScrapes() {
|
||||
const wasActive = new Set(
|
||||
state.scrapes.filter(scrape => scrape.state === "running" || scrape.state === "queued").map(s => s.id)
|
||||
);
|
||||
|
||||
try {
|
||||
state.scrapes = await window.VK.api.get("/api/media/scrapes");
|
||||
state.status = await window.VK.api.get("/api/media/status");
|
||||
} catch (error) {
|
||||
setStatus(`media status failed: ${error.message}`);
|
||||
}
|
||||
|
||||
// A scrape that was running and is now done counts once for the strip notice.
|
||||
const ids = new Set(state.scrapes.map(scrape => scrape.id));
|
||||
for (const scrape of state.scrapes) {
|
||||
if (wasActive.has(scrape.id) && scrape.state === "done") state.completed.add(scrape.id);
|
||||
}
|
||||
state.completed = new Set([...state.completed].filter(id => ids.has(id)));
|
||||
|
||||
const dismissed = prefs.dismissed.filter(id => ids.has(id));
|
||||
if (dismissed.length !== prefs.dismissed.length) prefs.setDismissed(dismissed);
|
||||
|
||||
renderScrapes();
|
||||
}
|
||||
|
||||
@@ -327,11 +381,23 @@
|
||||
|
||||
function renderScrapes() {
|
||||
scrapeList.replaceChildren();
|
||||
// Only live jobs: a paused scrape is not running and belongs in History, not the strip.
|
||||
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
|
||||
|
||||
for (const scrape of active) {
|
||||
const row = el("div", "vk-media-scrape");
|
||||
// The strip shows live jobs plus the newest paused scrape. Stop on a paused one only
|
||||
// hides it here; it stays resumable, and the next paused scrape can take its place.
|
||||
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
|
||||
const paused = state.scrapes
|
||||
.filter(scrape => scrape.state === "paused" && !prefs.dismissed.includes(scrape.id))
|
||||
.sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : -1))
|
||||
.slice(0, 1);
|
||||
|
||||
const limitedUntil = state.status?.limitedUntil ? new Date(state.status.limitedUntil).getTime() : 0;
|
||||
const limited = limitedUntil > Date.now();
|
||||
const limitedText = limited
|
||||
? ` · API limited, retry in ${Math.max(0, Math.ceil((limitedUntil - Date.now()) / 1000))}s`
|
||||
: "";
|
||||
|
||||
for (const scrape of [...active, ...paused]) {
|
||||
const row = el("div", `vk-media-scrape${limited ? " vk-media-limited" : ""}`);
|
||||
const media = scrape.media ?? {};
|
||||
|
||||
if (scrape.state === "running" || scrape.state === "queued") {
|
||||
@@ -354,12 +420,32 @@
|
||||
`${channelLabel(scrape.channelId)} · ${scrape.state} · ${pageText}${percent} · ${scrape.messages}m · ` +
|
||||
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0}` +
|
||||
(scrape.downloaded ? ` · files ${scrape.downloaded}` : "") +
|
||||
(scrape.previewed ? ` · previews ${scrape.previewed}` : "")
|
||||
(scrape.previewed ? ` · previews ${scrape.previewed}` : "") +
|
||||
limitedText
|
||||
)
|
||||
);
|
||||
|
||||
if (scrape.state === "paused") {
|
||||
row.append(
|
||||
button("Resume", async () => {
|
||||
try {
|
||||
await window.VK.api.post("/api/media/scrape/resume", { id: scrape.id });
|
||||
} catch (error) {
|
||||
setStatus(`resume failed: ${error.message}`);
|
||||
}
|
||||
void loadScrapes();
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
row.append(
|
||||
button("Stop", async () => {
|
||||
if (scrape.state === "paused") {
|
||||
// Hidden here, still resumable in the Bot tab.
|
||||
prefs.setDismissed([...prefs.dismissed, scrape.id]);
|
||||
renderScrapes();
|
||||
return;
|
||||
}
|
||||
await window.VK.api.post("/api/media/scrape/stop", { id: scrape.id });
|
||||
void loadScrapes();
|
||||
})
|
||||
@@ -367,6 +453,285 @@
|
||||
|
||||
scrapeList.append(row);
|
||||
}
|
||||
|
||||
// Session notice only: it clears on reload, not because the data changed.
|
||||
if (state.completed.size) {
|
||||
scrapeList.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-media-scrape vk-media-done-note",
|
||||
`${state.completed.size} ${state.completed.size === 1 ? "scrape" : "scrapes"} completed`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const prefs = {
|
||||
get dismissed() {
|
||||
try {
|
||||
const raw = JSON.parse(localStorage.getItem("vk-media-dismissed") ?? "[]");
|
||||
return Array.isArray(raw) ? raw.map(String) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
setDismissed(ids) {
|
||||
localStorage.setItem("vk-media-dismissed", JSON.stringify([...new Set(ids.map(String))]));
|
||||
}
|
||||
};
|
||||
|
||||
// ---- Backfill (Bot tab) ----
|
||||
|
||||
const TUNABLE_GROUPS = {
|
||||
backfill: {
|
||||
title: "Backfilling",
|
||||
fields: [
|
||||
["batchSize", "Batch size (rows)", 1, 500, 1],
|
||||
["batchPauseMs", "Cooldown between batches (ms)", 0, 600000, 100],
|
||||
["maxAttempts", "Tries before giving up", 1, 100, 1],
|
||||
["previewMinGapMs", "Preview gap min (ms)", 0, 60000, 10],
|
||||
["previewMaxGapMs", "Preview gap max (ms)", 0, 60000, 10],
|
||||
["maxBytes", "Max preview size (MiB)", 0.0625, 2048, 0.5, "mib"],
|
||||
["auto", "Start automatically on daemon boot", null, null, null, "bool"]
|
||||
]
|
||||
},
|
||||
scrape: {
|
||||
title: "Scraping",
|
||||
fields: [
|
||||
["walkCyclePages", "Preview every N walk pages", 0, 100, 1],
|
||||
["maxFileBytes", "Max file size (MiB)", 1, 8192, 1, "mib"],
|
||||
["downloadMinGapMs", "Download gap min (ms)", 0, 600000, 100],
|
||||
["downloadMaxGapMs", "Download gap max (ms)", 0, 600000, 100],
|
||||
["maxRetries", "Page fetch retries", 0, 10, 1]
|
||||
]
|
||||
},
|
||||
timers: {
|
||||
title: "Timers",
|
||||
fields: [
|
||||
["pageMinMs", "Walk page gap min (ms)", 0, 600000, 100],
|
||||
["pageMaxMs", "Walk page gap max (ms)", 0, 600000, 100],
|
||||
["rateLimitPauseMs", "Rate-limit cooldown (ms)", 0, 3600000, 5000]
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
function formatEta(ms) {
|
||||
if (ms === null || ms === undefined || !Number.isFinite(ms)) return "\u2014";
|
||||
const seconds = Math.max(0, Math.round(ms / 1000));
|
||||
if (seconds < 90) return `${seconds}s`;
|
||||
const minutes = Math.round(seconds / 60);
|
||||
if (minutes < 90) return `${minutes}m`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
const rest = minutes % 60;
|
||||
return rest ? `${hours}h ${rest}m` : `${hours}h`;
|
||||
}
|
||||
|
||||
function pendingKinds(member) {
|
||||
const by = member.pendingByKind ?? {};
|
||||
return ["video", "image", "gif", "audio"]
|
||||
.filter(kind => by[kind] > 0)
|
||||
.map(kind => `${by[kind]} ${kind}`)
|
||||
.join(" \u00b7 ");
|
||||
}
|
||||
|
||||
function renderBackfill() {
|
||||
const status = state.backfill;
|
||||
if (!status) return;
|
||||
|
||||
const until = status.limitedUntil ? new Date(status.limitedUntil).getTime() : 0;
|
||||
const limited = until > Date.now();
|
||||
botLimited.hidden = !limited;
|
||||
if (limited) {
|
||||
botLimited.textContent =
|
||||
`API timed out, increase timers \u2014 media traffic paused for ` +
|
||||
formatEta(until - Date.now());
|
||||
}
|
||||
|
||||
const totals = status.totals ?? {};
|
||||
const backend = totals.byBackend ?? {};
|
||||
const bot = backend.slopbot ?? { pending: 0, cached: 0, failed: 0, total: 0 };
|
||||
const puppet = backend.puppet ?? { pending: 0, cached: 0, failed: 0, total: 0 };
|
||||
|
||||
backfillSummary.replaceChildren(
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line",
|
||||
`${totals.pending ?? 0} pending \u00b7 ${totals.cached ?? 0} cached \u00b7 ` +
|
||||
`${totals.failed ?? 0} failed \u00b7 ${totals.downloaded ?? 0} downloaded \u00b7 ` +
|
||||
`${formatBytes(totals.bytes ?? 0)} stored`
|
||||
),
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line",
|
||||
`ETA ${formatEta(totals.etaMs)} at ${status.estimatedPerMinute ?? 0} rows/min ` +
|
||||
`(current settings)`
|
||||
),
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line vk-dim",
|
||||
`Bot: ${bot.pending} pending, ${bot.cached} cached, ${bot.failed} failed \u00b7 ` +
|
||||
`Puppet: ${puppet.pending} pending, ${puppet.cached} cached, ${puppet.failed} failed`
|
||||
)
|
||||
);
|
||||
|
||||
backfillJobs.replaceChildren();
|
||||
const jobs = (status.jobs ?? []).filter(job => job.pending > 0 || job.state !== "idle");
|
||||
if (!jobs.length) {
|
||||
backfillJobs.append(el("div", "vk-bot-empty", "Nothing waiting to backfill."));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const job of jobs) {
|
||||
const row = el("div", `vk-bot-job vk-bot-job-${job.state}`);
|
||||
const head = el("div", "vk-bot-job-head");
|
||||
head.append(el("span", "vk-bot-job-title", channelLabel(job.chatId)));
|
||||
head.append(el("span", `vk-bot-state vk-bot-state-${job.state}`, job.state));
|
||||
|
||||
const parts = [`${job.pending} pending`];
|
||||
if (job.cached) parts.push(`${job.cached} cached`);
|
||||
if (job.failed) parts.push(`${job.failed} failed`);
|
||||
if (job.pending > 0) parts.push(`ETA ${formatEta(job.etaMs)}`);
|
||||
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
|
||||
row.append(head);
|
||||
|
||||
for (const member of job.members) {
|
||||
if (member.pending === 0 && member.state !== "running") continue;
|
||||
const kinds = pendingKinds(member);
|
||||
row.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-job-member",
|
||||
`${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` +
|
||||
`${member.backend} \u00b7 ${member.pending} pending` +
|
||||
(kinds ? ` (${kinds})` : "")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const progress = el("div", "vk-bot-progress");
|
||||
const fill = el("div", "vk-bot-progress-fill");
|
||||
fill.style.width = `${Math.min(100, Math.round((job.cached / Math.max(1, job.total)) * 100))}%`;
|
||||
progress.append(fill);
|
||||
row.append(progress);
|
||||
|
||||
const actions = el("div", "vk-bot-job-actions");
|
||||
if (job.state === "running") {
|
||||
actions.append(button("Pause", () => void backfillAction("pause", job.chatId)));
|
||||
} else if (job.pending > 0) {
|
||||
actions.append(button("Start", () => void backfillAction("start", job.chatId)));
|
||||
}
|
||||
row.append(actions);
|
||||
|
||||
backfillJobs.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
async function backfillAction(action, chatId) {
|
||||
try {
|
||||
state.backfill = await window.VK.api.post(
|
||||
`/api/media/backfill/${action}`,
|
||||
chatId ? { chatId } : {}
|
||||
);
|
||||
renderBackfill();
|
||||
} catch (error) {
|
||||
setStatus(`backfill ${action} failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshBackfill() {
|
||||
try {
|
||||
state.backfill = await window.VK.api.get("/api/media/backfill");
|
||||
} catch {
|
||||
state.backfill = null;
|
||||
}
|
||||
renderBackfill();
|
||||
}
|
||||
|
||||
function tunableValue(raw, key, scale) {
|
||||
const value = raw?.[key];
|
||||
if (scale === "bool") return value !== false;
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return null;
|
||||
return scale === "mib" ? Math.round((number / 1024 ** 2) * 100) / 100 : number;
|
||||
}
|
||||
|
||||
function renderTunables() {
|
||||
const tunables = state.preferences?.media?.tunables;
|
||||
if (!tunables) return;
|
||||
|
||||
tunablesHost.replaceChildren();
|
||||
|
||||
for (const [group, spec] of Object.entries(TUNABLE_GROUPS)) {
|
||||
const details = el("details", "vk-bot-group");
|
||||
details.open = group === "backfill";
|
||||
details.append(el("summary", "vk-bot-group-title", spec.title));
|
||||
|
||||
const body = el("div", "vk-bot-group-body");
|
||||
for (const [key, label, min, max, step, scale] of spec.fields) {
|
||||
const row = el("label", "vk-bot-field");
|
||||
row.append(el("span", "vk-bot-field-label", label));
|
||||
|
||||
const input = document.createElement("input");
|
||||
input.dataset.group = group;
|
||||
input.dataset.key = key;
|
||||
input.dataset.scale = scale ?? "";
|
||||
|
||||
if (scale === "bool") {
|
||||
input.type = "checkbox";
|
||||
input.checked = tunables[group]?.[key] !== false;
|
||||
} else {
|
||||
input.type = "number";
|
||||
input.className = "vk-media-input";
|
||||
input.min = String(min);
|
||||
input.max = String(max);
|
||||
input.step = String(step);
|
||||
input.value = String(tunableValue(tunables[group] ?? {}, key, scale) ?? "");
|
||||
}
|
||||
|
||||
row.append(input);
|
||||
body.append(row);
|
||||
}
|
||||
|
||||
const save = button("Save", () => void saveTunables(group));
|
||||
save.classList.add("vk-bot-save");
|
||||
body.append(save);
|
||||
details.append(body);
|
||||
tunablesHost.append(details);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveTunables(group) {
|
||||
const patch = {};
|
||||
|
||||
for (const input of tunablesHost.querySelectorAll(`input[data-group="${group}"]`)) {
|
||||
const scale = input.dataset.scale;
|
||||
if (scale === "bool") {
|
||||
patch[input.dataset.key] = input.checked;
|
||||
continue;
|
||||
}
|
||||
|
||||
const value = Number(input.value);
|
||||
if (!Number.isFinite(value)) continue;
|
||||
patch[input.dataset.key] = scale === "mib" ? Math.round(value * 1024 ** 2) : Math.round(value);
|
||||
}
|
||||
|
||||
try {
|
||||
await window.VK.api.post("/api/preferences", { media: { tunables: { [group]: patch } } });
|
||||
setStatus(`${TUNABLE_GROUPS[group].title.toLowerCase()} settings saved`);
|
||||
await refreshPreferences();
|
||||
void refreshBackfill();
|
||||
} catch (error) {
|
||||
setStatus(`save failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshPreferences() {
|
||||
try {
|
||||
state.preferences = await window.VK.api.get("/api/preferences");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
renderTunables();
|
||||
}
|
||||
|
||||
// ---- Bot errors ----
|
||||
@@ -387,7 +752,7 @@
|
||||
}
|
||||
|
||||
function renderErrors() {
|
||||
scrapeList.replaceChildren();
|
||||
botErrors.replaceChildren();
|
||||
|
||||
const head = el("div", "vk-media-scrape");
|
||||
head.append(el("span", null, `${(state.errors ?? []).length} error(s)`));
|
||||
@@ -408,7 +773,7 @@
|
||||
scrapeList.append(head);
|
||||
|
||||
for (const entry of state.errors ?? []) {
|
||||
scrapeList.append(
|
||||
botErrors.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-media-scrape vk-media-error",
|
||||
@@ -417,7 +782,7 @@
|
||||
);
|
||||
}
|
||||
|
||||
if (!(state.errors ?? []).length) scrapeList.append(el("div", "vk-media-scrape", "No errors."));
|
||||
if (!(state.errors ?? []).length) botErrors.append(el("div", "vk-media-scrape", "No errors."));
|
||||
}
|
||||
|
||||
async function refreshErrors() {
|
||||
@@ -972,6 +1337,8 @@
|
||||
title.textContent = state.view === "bot" ? "Slopbot" : "Media archive";
|
||||
toolbar.hidden = false;
|
||||
grid.hidden = state.view === "bot";
|
||||
botPanel.hidden = state.view !== "bot";
|
||||
scrapeList.hidden = state.view === "bot";
|
||||
renderToolbar();
|
||||
|
||||
if (state.view === "bot") {
|
||||
@@ -979,6 +1346,8 @@
|
||||
statusLine.textContent = bot?.connected
|
||||
? `connected as ${bot.user} in ${bot.guilds.length} guild(s)`
|
||||
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
|
||||
if (!state.preferences) void refreshPreferences();
|
||||
void refreshBackfill();
|
||||
void loadScrapes().then(renderScrapes);
|
||||
void refreshErrors();
|
||||
} else {
|
||||
@@ -993,10 +1362,15 @@
|
||||
|
||||
overlay.querySelector("#media-close").addEventListener("click", hide);
|
||||
overlay.querySelector("#media-refresh").addEventListener("click", () => {
|
||||
state.completed.clear();
|
||||
void loadScrapes();
|
||||
if (state.view === "media") void loadMedia(true);
|
||||
if (state.view === "bot") void refreshBackfill();
|
||||
});
|
||||
|
||||
overlay.querySelector("#backfill-start").addEventListener("click", () => void backfillAction("start"));
|
||||
overlay.querySelector("#backfill-pause").addEventListener("click", () => void backfillAction("pause"));
|
||||
|
||||
const scopeMedia = document.querySelector("#scope-media");
|
||||
const scopeBot = document.querySelector("#scope-bot");
|
||||
scopeMedia?.addEventListener("click", () => show("media"));
|
||||
@@ -1019,6 +1393,11 @@
|
||||
if (state.view === "media") void loadScrapes();
|
||||
}, 4000);
|
||||
|
||||
// Backfill progress while the Bot tab is open.
|
||||
setInterval(() => {
|
||||
if (state.view === "bot") void refreshBackfill();
|
||||
}, 3000);
|
||||
|
||||
// Infinite scroll: fetch the next page before the grid bottom is reached.
|
||||
grid.addEventListener("scroll", () => {
|
||||
if (state.loading || !state.loaded) return;
|
||||
|
||||
Reference in New Issue
Block a user