Add the Bot-tab backfill control room and media strip rules
This commit is contained in:
Fisto
@@ -541,3 +541,202 @@
|
||||
color: #fff;
|
||||
font: 600 9px/1 ui-monospace, Menlo, monospace;
|
||||
}
|
||||
|
||||
/* Bot tab: backfill jobs, stats, and settings. */
|
||||
|
||||
.vk-media-bot {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.vk-media-bot[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.vk-bot-limited {
|
||||
padding: 8px 10px;
|
||||
border-radius: 6px;
|
||||
font: 500 11px/1.4 ui-monospace, Menlo, monospace;
|
||||
color: #ffb0b0;
|
||||
background: color-mix(in srgb, #ff5555 14%, transparent);
|
||||
border: 1px solid color-mix(in srgb, #ff5555 35%, transparent);
|
||||
}
|
||||
|
||||
.vk-bot-limited[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.vk-bot-section {
|
||||
border: 1px solid var(--color-divider);
|
||||
border-radius: 6px;
|
||||
padding: 8px 10px;
|
||||
background: var(--color-surface);
|
||||
}
|
||||
|
||||
.vk-bot-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.vk-bot-title {
|
||||
font-weight: 600;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.vk-bot-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-bot-summary {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.vk-bot-summary-line {
|
||||
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-300);
|
||||
}
|
||||
|
||||
.vk-bot-jobs {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-bot-empty {
|
||||
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-500);
|
||||
}
|
||||
|
||||
.vk-bot-job {
|
||||
border: 1px solid var(--color-neutral-800);
|
||||
border-radius: 5px;
|
||||
padding: 6px 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.vk-bot-job-running {
|
||||
border-color: color-mix(in srgb, var(--color-accent, #4f8cff) 45%, transparent);
|
||||
}
|
||||
|
||||
.vk-bot-job-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.vk-bot-job-title {
|
||||
flex: 1;
|
||||
font-weight: 600;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.vk-bot-job-meta {
|
||||
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-400);
|
||||
}
|
||||
|
||||
.vk-bot-state {
|
||||
font: 600 10px/1.4 ui-monospace, Menlo, monospace;
|
||||
text-transform: uppercase;
|
||||
padding: 0 5px;
|
||||
border-radius: 3px;
|
||||
background: var(--color-neutral-800);
|
||||
color: var(--color-neutral-300);
|
||||
}
|
||||
|
||||
.vk-bot-state-running {
|
||||
background: color-mix(in srgb, var(--color-accent, #4f8cff) 25%, transparent);
|
||||
color: var(--color-accent-300, #9fc1ff);
|
||||
}
|
||||
|
||||
.vk-bot-state-paused {
|
||||
background: color-mix(in srgb, #ffaa33 20%, transparent);
|
||||
color: #ffc880;
|
||||
}
|
||||
|
||||
.vk-bot-state-done {
|
||||
background: color-mix(in srgb, #44cc88 20%, transparent);
|
||||
color: #8fe0b8;
|
||||
}
|
||||
|
||||
.vk-bot-job-member {
|
||||
font: 400 10px/1.5 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-400);
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.vk-bot-progress {
|
||||
height: 4px;
|
||||
border-radius: 2px;
|
||||
background: var(--color-neutral-800);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.vk-bot-progress-fill {
|
||||
height: 100%;
|
||||
background: var(--color-accent-400, #4f8cff);
|
||||
}
|
||||
|
||||
.vk-bot-job-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-bot-settings {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-bot-group-title {
|
||||
cursor: pointer;
|
||||
font: 600 11px/1.6 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-300);
|
||||
}
|
||||
|
||||
.vk-bot-group-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
padding: 6px 0 2px 12px;
|
||||
}
|
||||
|
||||
.vk-bot-field {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.vk-bot-field-label {
|
||||
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-400);
|
||||
}
|
||||
|
||||
.vk-bot-save {
|
||||
align-self: flex-start;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.vk-media-limited {
|
||||
color: #ff8080;
|
||||
}
|
||||
|
||||
.vk-media-done-note {
|
||||
color: color-mix(in srgb, var(--color-neutral-400) 55%, transparent);
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
+388
-9
@@ -35,7 +35,11 @@
|
||||
zoom: Number(localStorage.getItem("vk-media-zoom") ?? 50),
|
||||
draft: null,
|
||||
prefetched: null,
|
||||
pageSize: 60
|
||||
pageSize: 60,
|
||||
backfill: null,
|
||||
preferences: null,
|
||||
/** Scrapes that finished while this page was open, until the tab is reloaded. */
|
||||
completed: new Set()
|
||||
};
|
||||
|
||||
const overlay = document.createElement("div");
|
||||
@@ -52,6 +56,35 @@
|
||||
<div id="media-status" class="vk-media-status"></div>
|
||||
<div id="media-scrapes" class="vk-media-scrapes"></div>
|
||||
<div id="media-grid" class="vk-media-grid"></div>
|
||||
<div id="media-bot" class="vk-media-bot" hidden>
|
||||
<div id="bot-limited" class="vk-bot-limited" hidden></div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head">
|
||||
<span class="vk-bot-title">Backfill</span>
|
||||
<span class="vk-bot-actions">
|
||||
<button id="backfill-start" class="btn btn-primary vk-small-btn" type="button">
|
||||
Start backfill
|
||||
</button>
|
||||
<button id="backfill-pause" class="btn btn-secondary vk-small-btn" type="button">
|
||||
Pause all
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<div id="backfill-summary" class="vk-bot-summary"></div>
|
||||
<div id="backfill-jobs" class="vk-bot-jobs"></div>
|
||||
</div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head"><span class="vk-bot-title">Settings</span></div>
|
||||
<div id="bot-tunables" class="vk-bot-settings"></div>
|
||||
</div>
|
||||
|
||||
<div class="vk-bot-section">
|
||||
<div class="vk-bot-head"><span class="vk-bot-title">Errors</span></div>
|
||||
<div id="bot-errors" class="vk-bot-errors"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
document.body.append(overlay);
|
||||
@@ -61,6 +94,12 @@
|
||||
const statusLine = overlay.querySelector("#media-status");
|
||||
const scrapeList = overlay.querySelector("#media-scrapes");
|
||||
const grid = overlay.querySelector("#media-grid");
|
||||
const botPanel = overlay.querySelector("#media-bot");
|
||||
const botLimited = overlay.querySelector("#bot-limited");
|
||||
const backfillSummary = overlay.querySelector("#backfill-summary");
|
||||
const backfillJobs = overlay.querySelector("#backfill-jobs");
|
||||
const tunablesHost = overlay.querySelector("#bot-tunables");
|
||||
const botErrors = overlay.querySelector("#bot-errors");
|
||||
|
||||
const filterDrawer = el("div", "vk-media-drawer");
|
||||
filterDrawer.hidden = true;
|
||||
@@ -177,12 +216,27 @@
|
||||
}
|
||||
|
||||
async function loadScrapes() {
|
||||
const wasActive = new Set(
|
||||
state.scrapes.filter(scrape => scrape.state === "running" || scrape.state === "queued").map(s => s.id)
|
||||
);
|
||||
|
||||
try {
|
||||
state.scrapes = await window.VK.api.get("/api/media/scrapes");
|
||||
state.status = await window.VK.api.get("/api/media/status");
|
||||
} catch (error) {
|
||||
setStatus(`media status failed: ${error.message}`);
|
||||
}
|
||||
|
||||
// A scrape that was running and is now done counts once for the strip notice.
|
||||
const ids = new Set(state.scrapes.map(scrape => scrape.id));
|
||||
for (const scrape of state.scrapes) {
|
||||
if (wasActive.has(scrape.id) && scrape.state === "done") state.completed.add(scrape.id);
|
||||
}
|
||||
state.completed = new Set([...state.completed].filter(id => ids.has(id)));
|
||||
|
||||
const dismissed = prefs.dismissed.filter(id => ids.has(id));
|
||||
if (dismissed.length !== prefs.dismissed.length) prefs.setDismissed(dismissed);
|
||||
|
||||
renderScrapes();
|
||||
}
|
||||
|
||||
@@ -327,11 +381,23 @@
|
||||
|
||||
function renderScrapes() {
|
||||
scrapeList.replaceChildren();
|
||||
// Only live jobs: a paused scrape is not running and belongs in History, not the strip.
|
||||
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
|
||||
|
||||
for (const scrape of active) {
|
||||
const row = el("div", "vk-media-scrape");
|
||||
// The strip shows live jobs plus the newest paused scrape. Stop on a paused one only
|
||||
// hides it here; it stays resumable, and the next paused scrape can take its place.
|
||||
const active = state.scrapes.filter(scrape => ["queued", "running"].includes(scrape.state));
|
||||
const paused = state.scrapes
|
||||
.filter(scrape => scrape.state === "paused" && !prefs.dismissed.includes(scrape.id))
|
||||
.sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : -1))
|
||||
.slice(0, 1);
|
||||
|
||||
const limitedUntil = state.status?.limitedUntil ? new Date(state.status.limitedUntil).getTime() : 0;
|
||||
const limited = limitedUntil > Date.now();
|
||||
const limitedText = limited
|
||||
? ` · API limited, retry in ${Math.max(0, Math.ceil((limitedUntil - Date.now()) / 1000))}s`
|
||||
: "";
|
||||
|
||||
for (const scrape of [...active, ...paused]) {
|
||||
const row = el("div", `vk-media-scrape${limited ? " vk-media-limited" : ""}`);
|
||||
const media = scrape.media ?? {};
|
||||
|
||||
if (scrape.state === "running" || scrape.state === "queued") {
|
||||
@@ -354,12 +420,32 @@
|
||||
`${channelLabel(scrape.channelId)} · ${scrape.state} · ${pageText}${percent} · ${scrape.messages}m · ` +
|
||||
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0}` +
|
||||
(scrape.downloaded ? ` · files ${scrape.downloaded}` : "") +
|
||||
(scrape.previewed ? ` · previews ${scrape.previewed}` : "")
|
||||
(scrape.previewed ? ` · previews ${scrape.previewed}` : "") +
|
||||
limitedText
|
||||
)
|
||||
);
|
||||
|
||||
if (scrape.state === "paused") {
|
||||
row.append(
|
||||
button("Resume", async () => {
|
||||
try {
|
||||
await window.VK.api.post("/api/media/scrape/resume", { id: scrape.id });
|
||||
} catch (error) {
|
||||
setStatus(`resume failed: ${error.message}`);
|
||||
}
|
||||
void loadScrapes();
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
row.append(
|
||||
button("Stop", async () => {
|
||||
if (scrape.state === "paused") {
|
||||
// Hidden here, still resumable in the Bot tab.
|
||||
prefs.setDismissed([...prefs.dismissed, scrape.id]);
|
||||
renderScrapes();
|
||||
return;
|
||||
}
|
||||
await window.VK.api.post("/api/media/scrape/stop", { id: scrape.id });
|
||||
void loadScrapes();
|
||||
})
|
||||
@@ -367,6 +453,285 @@
|
||||
|
||||
scrapeList.append(row);
|
||||
}
|
||||
|
||||
// Session notice only: it clears on reload, not because the data changed.
|
||||
if (state.completed.size) {
|
||||
scrapeList.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-media-scrape vk-media-done-note",
|
||||
`${state.completed.size} ${state.completed.size === 1 ? "scrape" : "scrapes"} completed`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const prefs = {
|
||||
get dismissed() {
|
||||
try {
|
||||
const raw = JSON.parse(localStorage.getItem("vk-media-dismissed") ?? "[]");
|
||||
return Array.isArray(raw) ? raw.map(String) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
setDismissed(ids) {
|
||||
localStorage.setItem("vk-media-dismissed", JSON.stringify([...new Set(ids.map(String))]));
|
||||
}
|
||||
};
|
||||
|
||||
// ---- Backfill (Bot tab) ----
|
||||
|
||||
const TUNABLE_GROUPS = {
|
||||
backfill: {
|
||||
title: "Backfilling",
|
||||
fields: [
|
||||
["batchSize", "Batch size (rows)", 1, 500, 1],
|
||||
["batchPauseMs", "Cooldown between batches (ms)", 0, 600000, 100],
|
||||
["maxAttempts", "Tries before giving up", 1, 100, 1],
|
||||
["previewMinGapMs", "Preview gap min (ms)", 0, 60000, 10],
|
||||
["previewMaxGapMs", "Preview gap max (ms)", 0, 60000, 10],
|
||||
["maxBytes", "Max preview size (MiB)", 0.0625, 2048, 0.5, "mib"],
|
||||
["auto", "Start automatically on daemon boot", null, null, null, "bool"]
|
||||
]
|
||||
},
|
||||
scrape: {
|
||||
title: "Scraping",
|
||||
fields: [
|
||||
["walkCyclePages", "Preview every N walk pages", 0, 100, 1],
|
||||
["maxFileBytes", "Max file size (MiB)", 1, 8192, 1, "mib"],
|
||||
["downloadMinGapMs", "Download gap min (ms)", 0, 600000, 100],
|
||||
["downloadMaxGapMs", "Download gap max (ms)", 0, 600000, 100],
|
||||
["maxRetries", "Page fetch retries", 0, 10, 1]
|
||||
]
|
||||
},
|
||||
timers: {
|
||||
title: "Timers",
|
||||
fields: [
|
||||
["pageMinMs", "Walk page gap min (ms)", 0, 600000, 100],
|
||||
["pageMaxMs", "Walk page gap max (ms)", 0, 600000, 100],
|
||||
["rateLimitPauseMs", "Rate-limit cooldown (ms)", 0, 3600000, 5000]
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
function formatEta(ms) {
|
||||
if (ms === null || ms === undefined || !Number.isFinite(ms)) return "\u2014";
|
||||
const seconds = Math.max(0, Math.round(ms / 1000));
|
||||
if (seconds < 90) return `${seconds}s`;
|
||||
const minutes = Math.round(seconds / 60);
|
||||
if (minutes < 90) return `${minutes}m`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
const rest = minutes % 60;
|
||||
return rest ? `${hours}h ${rest}m` : `${hours}h`;
|
||||
}
|
||||
|
||||
function pendingKinds(member) {
|
||||
const by = member.pendingByKind ?? {};
|
||||
return ["video", "image", "gif", "audio"]
|
||||
.filter(kind => by[kind] > 0)
|
||||
.map(kind => `${by[kind]} ${kind}`)
|
||||
.join(" \u00b7 ");
|
||||
}
|
||||
|
||||
function renderBackfill() {
|
||||
const status = state.backfill;
|
||||
if (!status) return;
|
||||
|
||||
const until = status.limitedUntil ? new Date(status.limitedUntil).getTime() : 0;
|
||||
const limited = until > Date.now();
|
||||
botLimited.hidden = !limited;
|
||||
if (limited) {
|
||||
botLimited.textContent =
|
||||
`API timed out, increase timers \u2014 media traffic paused for ` +
|
||||
formatEta(until - Date.now());
|
||||
}
|
||||
|
||||
const totals = status.totals ?? {};
|
||||
const backend = totals.byBackend ?? {};
|
||||
const bot = backend.slopbot ?? { pending: 0, cached: 0, failed: 0, total: 0 };
|
||||
const puppet = backend.puppet ?? { pending: 0, cached: 0, failed: 0, total: 0 };
|
||||
|
||||
backfillSummary.replaceChildren(
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line",
|
||||
`${totals.pending ?? 0} pending \u00b7 ${totals.cached ?? 0} cached \u00b7 ` +
|
||||
`${totals.failed ?? 0} failed \u00b7 ${totals.downloaded ?? 0} downloaded \u00b7 ` +
|
||||
`${formatBytes(totals.bytes ?? 0)} stored`
|
||||
),
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line",
|
||||
`ETA ${formatEta(totals.etaMs)} at ${status.estimatedPerMinute ?? 0} rows/min ` +
|
||||
`(current settings)`
|
||||
),
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-summary-line vk-dim",
|
||||
`Bot: ${bot.pending} pending, ${bot.cached} cached, ${bot.failed} failed \u00b7 ` +
|
||||
`Puppet: ${puppet.pending} pending, ${puppet.cached} cached, ${puppet.failed} failed`
|
||||
)
|
||||
);
|
||||
|
||||
backfillJobs.replaceChildren();
|
||||
const jobs = (status.jobs ?? []).filter(job => job.pending > 0 || job.state !== "idle");
|
||||
if (!jobs.length) {
|
||||
backfillJobs.append(el("div", "vk-bot-empty", "Nothing waiting to backfill."));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const job of jobs) {
|
||||
const row = el("div", `vk-bot-job vk-bot-job-${job.state}`);
|
||||
const head = el("div", "vk-bot-job-head");
|
||||
head.append(el("span", "vk-bot-job-title", channelLabel(job.chatId)));
|
||||
head.append(el("span", `vk-bot-state vk-bot-state-${job.state}`, job.state));
|
||||
|
||||
const parts = [`${job.pending} pending`];
|
||||
if (job.cached) parts.push(`${job.cached} cached`);
|
||||
if (job.failed) parts.push(`${job.failed} failed`);
|
||||
if (job.pending > 0) parts.push(`ETA ${formatEta(job.etaMs)}`);
|
||||
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
|
||||
row.append(head);
|
||||
|
||||
for (const member of job.members) {
|
||||
if (member.pending === 0 && member.state !== "running") continue;
|
||||
const kinds = pendingKinds(member);
|
||||
row.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-bot-job-member",
|
||||
`${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` +
|
||||
`${member.backend} \u00b7 ${member.pending} pending` +
|
||||
(kinds ? ` (${kinds})` : "")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const progress = el("div", "vk-bot-progress");
|
||||
const fill = el("div", "vk-bot-progress-fill");
|
||||
fill.style.width = `${Math.min(100, Math.round((job.cached / Math.max(1, job.total)) * 100))}%`;
|
||||
progress.append(fill);
|
||||
row.append(progress);
|
||||
|
||||
const actions = el("div", "vk-bot-job-actions");
|
||||
if (job.state === "running") {
|
||||
actions.append(button("Pause", () => void backfillAction("pause", job.chatId)));
|
||||
} else if (job.pending > 0) {
|
||||
actions.append(button("Start", () => void backfillAction("start", job.chatId)));
|
||||
}
|
||||
row.append(actions);
|
||||
|
||||
backfillJobs.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
async function backfillAction(action, chatId) {
|
||||
try {
|
||||
state.backfill = await window.VK.api.post(
|
||||
`/api/media/backfill/${action}`,
|
||||
chatId ? { chatId } : {}
|
||||
);
|
||||
renderBackfill();
|
||||
} catch (error) {
|
||||
setStatus(`backfill ${action} failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshBackfill() {
|
||||
try {
|
||||
state.backfill = await window.VK.api.get("/api/media/backfill");
|
||||
} catch {
|
||||
state.backfill = null;
|
||||
}
|
||||
renderBackfill();
|
||||
}
|
||||
|
||||
function tunableValue(raw, key, scale) {
|
||||
const value = raw?.[key];
|
||||
if (scale === "bool") return value !== false;
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return null;
|
||||
return scale === "mib" ? Math.round((number / 1024 ** 2) * 100) / 100 : number;
|
||||
}
|
||||
|
||||
function renderTunables() {
|
||||
const tunables = state.preferences?.media?.tunables;
|
||||
if (!tunables) return;
|
||||
|
||||
tunablesHost.replaceChildren();
|
||||
|
||||
for (const [group, spec] of Object.entries(TUNABLE_GROUPS)) {
|
||||
const details = el("details", "vk-bot-group");
|
||||
details.open = group === "backfill";
|
||||
details.append(el("summary", "vk-bot-group-title", spec.title));
|
||||
|
||||
const body = el("div", "vk-bot-group-body");
|
||||
for (const [key, label, min, max, step, scale] of spec.fields) {
|
||||
const row = el("label", "vk-bot-field");
|
||||
row.append(el("span", "vk-bot-field-label", label));
|
||||
|
||||
const input = document.createElement("input");
|
||||
input.dataset.group = group;
|
||||
input.dataset.key = key;
|
||||
input.dataset.scale = scale ?? "";
|
||||
|
||||
if (scale === "bool") {
|
||||
input.type = "checkbox";
|
||||
input.checked = tunables[group]?.[key] !== false;
|
||||
} else {
|
||||
input.type = "number";
|
||||
input.className = "vk-media-input";
|
||||
input.min = String(min);
|
||||
input.max = String(max);
|
||||
input.step = String(step);
|
||||
input.value = String(tunableValue(tunables[group] ?? {}, key, scale) ?? "");
|
||||
}
|
||||
|
||||
row.append(input);
|
||||
body.append(row);
|
||||
}
|
||||
|
||||
const save = button("Save", () => void saveTunables(group));
|
||||
save.classList.add("vk-bot-save");
|
||||
body.append(save);
|
||||
details.append(body);
|
||||
tunablesHost.append(details);
|
||||
}
|
||||
}
|
||||
|
||||
async function saveTunables(group) {
|
||||
const patch = {};
|
||||
|
||||
for (const input of tunablesHost.querySelectorAll(`input[data-group="${group}"]`)) {
|
||||
const scale = input.dataset.scale;
|
||||
if (scale === "bool") {
|
||||
patch[input.dataset.key] = input.checked;
|
||||
continue;
|
||||
}
|
||||
|
||||
const value = Number(input.value);
|
||||
if (!Number.isFinite(value)) continue;
|
||||
patch[input.dataset.key] = scale === "mib" ? Math.round(value * 1024 ** 2) : Math.round(value);
|
||||
}
|
||||
|
||||
try {
|
||||
await window.VK.api.post("/api/preferences", { media: { tunables: { [group]: patch } } });
|
||||
setStatus(`${TUNABLE_GROUPS[group].title.toLowerCase()} settings saved`);
|
||||
await refreshPreferences();
|
||||
void refreshBackfill();
|
||||
} catch (error) {
|
||||
setStatus(`save failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshPreferences() {
|
||||
try {
|
||||
state.preferences = await window.VK.api.get("/api/preferences");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
renderTunables();
|
||||
}
|
||||
|
||||
// ---- Bot errors ----
|
||||
@@ -387,7 +752,7 @@
|
||||
}
|
||||
|
||||
function renderErrors() {
|
||||
scrapeList.replaceChildren();
|
||||
botErrors.replaceChildren();
|
||||
|
||||
const head = el("div", "vk-media-scrape");
|
||||
head.append(el("span", null, `${(state.errors ?? []).length} error(s)`));
|
||||
@@ -408,7 +773,7 @@
|
||||
scrapeList.append(head);
|
||||
|
||||
for (const entry of state.errors ?? []) {
|
||||
scrapeList.append(
|
||||
botErrors.append(
|
||||
el(
|
||||
"div",
|
||||
"vk-media-scrape vk-media-error",
|
||||
@@ -417,7 +782,7 @@
|
||||
);
|
||||
}
|
||||
|
||||
if (!(state.errors ?? []).length) scrapeList.append(el("div", "vk-media-scrape", "No errors."));
|
||||
if (!(state.errors ?? []).length) botErrors.append(el("div", "vk-media-scrape", "No errors."));
|
||||
}
|
||||
|
||||
async function refreshErrors() {
|
||||
@@ -972,6 +1337,8 @@
|
||||
title.textContent = state.view === "bot" ? "Slopbot" : "Media archive";
|
||||
toolbar.hidden = false;
|
||||
grid.hidden = state.view === "bot";
|
||||
botPanel.hidden = state.view !== "bot";
|
||||
scrapeList.hidden = state.view === "bot";
|
||||
renderToolbar();
|
||||
|
||||
if (state.view === "bot") {
|
||||
@@ -979,6 +1346,8 @@
|
||||
statusLine.textContent = bot?.connected
|
||||
? `connected as ${bot.user} in ${bot.guilds.length} guild(s)`
|
||||
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
|
||||
if (!state.preferences) void refreshPreferences();
|
||||
void refreshBackfill();
|
||||
void loadScrapes().then(renderScrapes);
|
||||
void refreshErrors();
|
||||
} else {
|
||||
@@ -993,10 +1362,15 @@
|
||||
|
||||
overlay.querySelector("#media-close").addEventListener("click", hide);
|
||||
overlay.querySelector("#media-refresh").addEventListener("click", () => {
|
||||
state.completed.clear();
|
||||
void loadScrapes();
|
||||
if (state.view === "media") void loadMedia(true);
|
||||
if (state.view === "bot") void refreshBackfill();
|
||||
});
|
||||
|
||||
overlay.querySelector("#backfill-start").addEventListener("click", () => void backfillAction("start"));
|
||||
overlay.querySelector("#backfill-pause").addEventListener("click", () => void backfillAction("pause"));
|
||||
|
||||
const scopeMedia = document.querySelector("#scope-media");
|
||||
const scopeBot = document.querySelector("#scope-bot");
|
||||
scopeMedia?.addEventListener("click", () => show("media"));
|
||||
@@ -1019,6 +1393,11 @@
|
||||
if (state.view === "media") void loadScrapes();
|
||||
}, 4000);
|
||||
|
||||
// Backfill progress while the Bot tab is open.
|
||||
setInterval(() => {
|
||||
if (state.view === "bot") void refreshBackfill();
|
||||
}, 3000);
|
||||
|
||||
// Infinite scroll: fetch the next page before the grid bottom is reached.
|
||||
grid.addEventListener("scroll", () => {
|
||||
if (state.loading || !state.loaded) return;
|
||||
|
||||
@@ -42,13 +42,6 @@ let puppet: Puppet | undefined;
|
||||
let coreConnected = false;
|
||||
let connecting = false;
|
||||
|
||||
const media = new MediaService({
|
||||
getCore: () => core,
|
||||
isCoreConnected: () => coreConnected,
|
||||
getMediaPrefs: () => preferences.get().media,
|
||||
log: line => console.log(`[media] ${line}`)
|
||||
});
|
||||
|
||||
async function connectLoop(): Promise<void> {
|
||||
if (connecting) return;
|
||||
connecting = true;
|
||||
@@ -185,6 +178,14 @@ const preferences = new PreferencesStore(
|
||||
);
|
||||
const stt = new Stt();
|
||||
|
||||
// After `preferences`: the media engine and backfill queue read their tunables at construction.
|
||||
const media = new MediaService({
|
||||
getCore: () => core,
|
||||
isCoreConnected: () => coreConnected,
|
||||
getMediaPrefs: () => preferences.get().media,
|
||||
log: line => console.log(`[media] ${line}`)
|
||||
});
|
||||
|
||||
const audio = new AudioService({
|
||||
cacheDir: process.env.VESKSLOP_CACHE_DIR,
|
||||
transcripts,
|
||||
|
||||
+89
-27
@@ -8,6 +8,7 @@
|
||||
import { rmSync } from "node:fs";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import {
|
||||
BackfillQueue,
|
||||
betterCandidate,
|
||||
dirSizeBytes,
|
||||
isNearLimit,
|
||||
@@ -19,6 +20,8 @@ import {
|
||||
resolveDbRoot,
|
||||
type MediaKind,
|
||||
type MediaRow,
|
||||
type BackfillSettings,
|
||||
type BackfillStatus,
|
||||
type ScrapeBackend,
|
||||
type ScrapeMeta
|
||||
} from "../../media-scrape/src/index.js";
|
||||
@@ -153,11 +156,14 @@ const FILE_CONTENT_TYPES: Record<string, string> = {
|
||||
export interface MediaStatus {
|
||||
slopbot: SlopbotStatus;
|
||||
scrapes: ScrapeMeta[];
|
||||
/** ISO time while a Discord rate limit holds media traffic. */
|
||||
limitedUntil: string | null;
|
||||
}
|
||||
|
||||
export class MediaService {
|
||||
private readonly rootDir = resolveDbRoot();
|
||||
private readonly engine: ScrapeEngine;
|
||||
private readonly backfill: BackfillQueue;
|
||||
|
||||
private bot: BotClient | null = null;
|
||||
private botFetcher: SlopbotFetcher | null = null;
|
||||
@@ -167,50 +173,102 @@ export class MediaService {
|
||||
private readonly errors: MediaErrorEntry[] = [];
|
||||
|
||||
constructor(private readonly deps: MediaServiceDeps) {
|
||||
const number = (value: string | undefined, fallback: number) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
};
|
||||
const tunables = this.deps.getMediaPrefs().tunables;
|
||||
|
||||
this.engine = new ScrapeEngine({
|
||||
rootDir: this.rootDir,
|
||||
log: line => this.deps.log?.(line),
|
||||
onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
|
||||
walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
|
||||
rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
|
||||
previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3),
|
||||
walkCyclePages: tunables.scrape.walkCyclePages,
|
||||
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
|
||||
pageMinMs: tunables.timers.pageMinMs,
|
||||
pageMaxMs: tunables.timers.pageMaxMs,
|
||||
maxRetries: tunables.scrape.maxRetries,
|
||||
previewMaxAttempts: tunables.backfill.maxAttempts,
|
||||
download: {
|
||||
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
|
||||
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
|
||||
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
|
||||
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
|
||||
previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25),
|
||||
previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000),
|
||||
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
|
||||
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
|
||||
maxFileBytes: tunables.scrape.maxFileBytes,
|
||||
previewMaxBytes: tunables.backfill.maxBytes,
|
||||
previewMinGapMs: tunables.backfill.previewMinGapMs,
|
||||
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
|
||||
previewBatchSize: tunables.backfill.batchSize,
|
||||
previewBatchPauseMs: tunables.backfill.batchPauseMs,
|
||||
minGapMs: tunables.scrape.downloadMinGapMs,
|
||||
maxGapMs: tunables.scrape.downloadMaxGapMs
|
||||
}
|
||||
});
|
||||
|
||||
this.backfill = new BackfillQueue(this.engine, {
|
||||
rootDir: this.rootDir,
|
||||
getSettings: () => this.backfillSettings(),
|
||||
getGaps: () => {
|
||||
const live = this.deps.getMediaPrefs().tunables;
|
||||
return { minGapMs: live.backfill.previewMinGapMs, maxGapMs: live.backfill.previewMaxGapMs };
|
||||
},
|
||||
log: line => this.deps.log?.(line)
|
||||
});
|
||||
|
||||
const stale = this.engine.reconcileStaleJobs();
|
||||
if (stale.length) {
|
||||
this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`);
|
||||
}
|
||||
|
||||
// Preview passes only run when a walk ends; fill in scrapes that paused early.
|
||||
void this.backfillPreviews();
|
||||
// Preview passes only run when a walk ends; the queue fills in scrapes that paused early.
|
||||
this.backfill.autoStart();
|
||||
}
|
||||
|
||||
private async backfillPreviews(): Promise<void> {
|
||||
for (const meta of this.engine.list()) {
|
||||
if (meta.state === "running" || meta.state === "queued") continue;
|
||||
private backfillSettings(): BackfillSettings {
|
||||
const tunables = this.deps.getMediaPrefs().tunables;
|
||||
return {
|
||||
batchSize: tunables.backfill.batchSize,
|
||||
batchPauseMs: tunables.backfill.batchPauseMs,
|
||||
auto: tunables.backfill.auto
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const added = await this.engine.backfillPreviews(meta.id);
|
||||
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
|
||||
} catch (error) {
|
||||
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
|
||||
/** Push preference changes into the live engine and backfill queue. */
|
||||
applyTunables(): void {
|
||||
const tunables = this.deps.getMediaPrefs().tunables;
|
||||
this.engine.updateOptions({
|
||||
walkCyclePages: tunables.scrape.walkCyclePages,
|
||||
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
|
||||
pageMinMs: tunables.timers.pageMinMs,
|
||||
pageMaxMs: tunables.timers.pageMaxMs,
|
||||
maxRetries: tunables.scrape.maxRetries,
|
||||
previewMaxAttempts: tunables.backfill.maxAttempts,
|
||||
download: {
|
||||
maxFileBytes: tunables.scrape.maxFileBytes,
|
||||
previewMaxBytes: tunables.backfill.maxBytes,
|
||||
previewMinGapMs: tunables.backfill.previewMinGapMs,
|
||||
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
|
||||
previewBatchSize: tunables.backfill.batchSize,
|
||||
previewBatchPauseMs: tunables.backfill.batchPauseMs,
|
||||
minGapMs: tunables.scrape.downloadMinGapMs,
|
||||
maxGapMs: tunables.scrape.downloadMaxGapMs
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Backfill jobs, totals, and pacing, plus the engine's live cooldown. */
|
||||
backfillStatus(): BackfillStatus {
|
||||
const status = this.backfill.list();
|
||||
const limit = this.engine.rateLimitStatus();
|
||||
|
||||
return {
|
||||
...status,
|
||||
limitedUntil: limit.active
|
||||
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
|
||||
: status.limitedUntil
|
||||
};
|
||||
}
|
||||
|
||||
startBackfill(chatId?: string): BackfillStatus {
|
||||
this.backfill.start(chatId);
|
||||
return this.backfillStatus();
|
||||
}
|
||||
|
||||
pauseBackfill(chatId?: string): BackfillStatus {
|
||||
this.backfill.pause(chatId);
|
||||
return this.backfillStatus();
|
||||
}
|
||||
|
||||
/** Log slopbot in on first use. False when it is disabled or the login failed. */
|
||||
@@ -425,9 +483,13 @@ export class MediaService {
|
||||
}
|
||||
|
||||
status(): MediaStatus {
|
||||
const limit = this.engine.rateLimitStatus();
|
||||
return {
|
||||
slopbot: slopbotStatus(this.bot),
|
||||
scrapes: this.engine.list()
|
||||
scrapes: this.engine.list(),
|
||||
limitedUntil: limit.active
|
||||
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
|
||||
: null
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+117
-2
@@ -76,6 +76,43 @@ export interface AudioPreferences {
|
||||
duck: boolean;
|
||||
}
|
||||
|
||||
/** Pacing and limit knobs for preview backfilling, scraping, and cooldowns. */
|
||||
export interface MediaTunables {
|
||||
backfill: {
|
||||
/** Preview rows per batch. */
|
||||
batchSize: number;
|
||||
/** Cooldown between batches, ms. */
|
||||
batchPauseMs: number;
|
||||
/** Transient failures allowed before a row is closed as unreachable. */
|
||||
maxAttempts: number;
|
||||
/** Gap between preview fetches, ms. */
|
||||
previewMinGapMs: number;
|
||||
previewMaxGapMs: number;
|
||||
/** Largest preview to cache, bytes. */
|
||||
maxBytes: number;
|
||||
/** Start backfill jobs automatically on daemon boot. */
|
||||
auto: boolean;
|
||||
};
|
||||
scrape: {
|
||||
/** Walk pages between inline preview cycles. 0 defers previews to the end. */
|
||||
walkCyclePages: number;
|
||||
/** Largest file to download, bytes. */
|
||||
maxFileBytes: number;
|
||||
/** Gap between file downloads, ms. */
|
||||
downloadMinGapMs: number;
|
||||
downloadMaxGapMs: number;
|
||||
/** Page fetch retries before a scrape fails. */
|
||||
maxRetries: number;
|
||||
};
|
||||
timers: {
|
||||
/** Gap between walk pages, ms. */
|
||||
pageMinMs: number;
|
||||
pageMaxMs: number;
|
||||
/** Cooldown after a Discord rate limit, ms. */
|
||||
rateLimitPauseMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface MediaPreferences {
|
||||
/** Combined size limit for the media DB root: databases, downloads, previews. */
|
||||
limitBytes: number;
|
||||
@@ -83,6 +120,7 @@ export interface MediaPreferences {
|
||||
warnPercent: number;
|
||||
/** No-delete overrun allowance in percent, 0-100. */
|
||||
overrunPercent: number;
|
||||
tunables: MediaTunables;
|
||||
}
|
||||
|
||||
export interface WebPreferences {
|
||||
@@ -115,13 +153,89 @@ const DEFAULTS: WebPreferences = {
|
||||
media: {
|
||||
limitBytes: 30 * 1024 * 1024 * 1024,
|
||||
warnPercent: 90,
|
||||
overrunPercent: 5
|
||||
overrunPercent: 5,
|
||||
tunables: {
|
||||
backfill: {
|
||||
batchSize: 25,
|
||||
batchPauseMs: 60_000,
|
||||
maxAttempts: 3,
|
||||
previewMinGapMs: 200,
|
||||
previewMaxGapMs: 600,
|
||||
maxBytes: 5 * 1024 * 1024,
|
||||
auto: true
|
||||
},
|
||||
scrape: {
|
||||
walkCyclePages: 5,
|
||||
maxFileBytes: 50 * 1024 * 1024,
|
||||
downloadMinGapMs: 1000,
|
||||
downloadMaxGapMs: 3000,
|
||||
maxRetries: 2
|
||||
},
|
||||
timers: {
|
||||
pageMinMs: 1200,
|
||||
pageMaxMs: 3500,
|
||||
rateLimitPauseMs: 60_000
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const MEDIA_LIMIT_MIN = 256 * 1024 * 1024;
|
||||
const MEDIA_LIMIT_MAX = 10 * 1024 * 1024 * 1024 * 1024;
|
||||
|
||||
function intIn(value: unknown, min: number, max: number, fallback: number): number {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.round(number)));
|
||||
}
|
||||
|
||||
function normalizeTunables(raw: any, fallback: MediaTunables): MediaTunables {
|
||||
const backfill = raw?.backfill ?? {};
|
||||
const scrape = raw?.scrape ?? {};
|
||||
const timers = raw?.timers ?? {};
|
||||
|
||||
const previewMinGapMs = intIn(backfill.previewMinGapMs, 0, 60_000, fallback.backfill.previewMinGapMs);
|
||||
const previewMaxGapMs = intIn(
|
||||
backfill.previewMaxGapMs,
|
||||
previewMinGapMs,
|
||||
60_000,
|
||||
fallback.backfill.previewMaxGapMs
|
||||
);
|
||||
const downloadMinGapMs = intIn(scrape.downloadMinGapMs, 0, 600_000, fallback.scrape.downloadMinGapMs);
|
||||
const downloadMaxGapMs = intIn(
|
||||
scrape.downloadMaxGapMs,
|
||||
downloadMinGapMs,
|
||||
600_000,
|
||||
fallback.scrape.downloadMaxGapMs
|
||||
);
|
||||
const pageMinMs = intIn(timers.pageMinMs, 0, 600_000, fallback.timers.pageMinMs);
|
||||
const pageMaxMs = intIn(timers.pageMaxMs, pageMinMs, 600_000, fallback.timers.pageMaxMs);
|
||||
|
||||
return {
|
||||
backfill: {
|
||||
batchSize: intIn(backfill.batchSize, 1, 500, fallback.backfill.batchSize),
|
||||
batchPauseMs: intIn(backfill.batchPauseMs, 0, 3_600_000, fallback.backfill.batchPauseMs),
|
||||
maxAttempts: intIn(backfill.maxAttempts, 1, 100, fallback.backfill.maxAttempts),
|
||||
previewMinGapMs,
|
||||
previewMaxGapMs,
|
||||
maxBytes: intIn(backfill.maxBytes, 65_536, 2 * 1024 ** 3, fallback.backfill.maxBytes),
|
||||
auto: typeof backfill.auto === "boolean" ? backfill.auto : fallback.backfill.auto
|
||||
},
|
||||
scrape: {
|
||||
walkCyclePages: intIn(scrape.walkCyclePages, 0, 100, fallback.scrape.walkCyclePages),
|
||||
maxFileBytes: intIn(scrape.maxFileBytes, 1024 ** 2, 8 * 1024 ** 3, fallback.scrape.maxFileBytes),
|
||||
downloadMinGapMs,
|
||||
downloadMaxGapMs,
|
||||
maxRetries: intIn(scrape.maxRetries, 0, 10, fallback.scrape.maxRetries)
|
||||
},
|
||||
timers: {
|
||||
pageMinMs,
|
||||
pageMaxMs,
|
||||
rateLimitPauseMs: intIn(timers.rateLimitPauseMs, 0, 3_600_000, fallback.timers.rateLimitPauseMs)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences {
|
||||
const limit = Number(raw?.limitBytes);
|
||||
const warn = Number(raw?.warnPercent);
|
||||
@@ -136,7 +250,8 @@ function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences
|
||||
overrunPercent:
|
||||
Number.isFinite(overrun) && overrun >= 0 && overrun <= 100
|
||||
? Math.round(overrun)
|
||||
: fallback.overrunPercent
|
||||
: fallback.overrunPercent,
|
||||
tunables: normalizeTunables(raw?.tunables, fallback.tunables)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+28
-2
@@ -478,7 +478,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
: current.warnPercent,
|
||||
overrunPercent: Number.isFinite(Number(mediaPrefs?.overrunPercent))
|
||||
? Number(mediaPrefs.overrunPercent)
|
||||
: current.overrunPercent
|
||||
: current.overrunPercent,
|
||||
tunables: mediaPrefs?.tunables ?? current.tunables
|
||||
};
|
||||
}
|
||||
|
||||
@@ -487,7 +488,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
deps.audio.setDuckEnabled(patch.audio.duck);
|
||||
}
|
||||
|
||||
res.json({ ok: true, result: deps.preferences.update(patch) });
|
||||
const updated = deps.preferences.update(patch);
|
||||
if (mediaPrefs !== undefined) deps.media.applyTunables();
|
||||
res.json({ ok: true, result: updated });
|
||||
})
|
||||
);
|
||||
|
||||
@@ -773,6 +776,29 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/media/backfill",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: deps.media.backfillStatus() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/media/backfill/start",
|
||||
wrap(async (req, res) => {
|
||||
const chatId = req.body?.chatId ? String(req.body.chatId) : undefined;
|
||||
res.json({ ok: true, result: deps.media.startBackfill(chatId) });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/media/backfill/pause",
|
||||
wrap(async (req, res) => {
|
||||
const chatId = req.body?.chatId ? String(req.body.chatId) : undefined;
|
||||
res.json({ ok: true, result: deps.media.pauseBackfill(chatId) });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/media/:scrapeId/:itemId/thumb",
|
||||
wrap(async (req, res) => {
|
||||
|
||||
@@ -126,6 +126,29 @@ try {
|
||||
step("prompt after reset", resetPrompt);
|
||||
check("reset returns the fresh view", resetPrompt);
|
||||
|
||||
// Bot tab: backfill progress, stats, and the three settings groups.
|
||||
await page.click("#scope-bot");
|
||||
await page.waitForSelector("#media-bot:not([hidden])", { timeout: 10000 });
|
||||
await page.waitForFunction(
|
||||
() => document.querySelector("#backfill-summary")?.textContent?.includes("pending"),
|
||||
{ timeout: 20000 }
|
||||
);
|
||||
|
||||
const summaryLines = await page.locator(".vk-bot-summary-line").count();
|
||||
step("backfill summary lines", summaryLines);
|
||||
check("backfill summary renders", summaryLines >= 3, summaryLines);
|
||||
step("backfill totals", (await page.locator("#backfill-summary").textContent())?.trim().slice(0, 160));
|
||||
step("backend split", (await page.locator(".vk-bot-summary-line.vk-dim").first().textContent())?.trim());
|
||||
|
||||
const groups = await page.locator("#bot-tunables details").count();
|
||||
step("tunable groups", groups);
|
||||
check("three tunable groups", groups === 3, groups);
|
||||
|
||||
const fields = await page.locator("#bot-tunables input").count();
|
||||
step("tunable fields", fields);
|
||||
check("tunable fields render", fields >= 12, fields);
|
||||
check("start button visible", await page.locator("#backfill-start").isVisible());
|
||||
|
||||
step("console errors", results.consoleErrors.length);
|
||||
check("no console errors", results.consoleErrors.length === 0, results.consoleErrors.slice(0, 3));
|
||||
|
||||
|
||||
Reference in New Issue
Block a user