zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run the title catch-up in its own YouTube lane

Browse Source
This commit is contained in:
Fisto
2026-09-17 14:25:08 +02:00
parent 8befc05591
commit 9481286169
16 changed files with 495 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/public/media.css
+11
View File
@@ -791,3 +791,14 @@
.vk-chip-strike {
text-decoration: line-through;
}
/* The YouTube title catch-up lane, inside the Bot tab backfill section. */
.vk-bot-lane {
border: 1px dashed var(--color-neutral-700);
border-radius: 5px;
padding: 6px 8px;
margin-bottom: 8px;
display: flex;
flex-direction: column;
gap: 4px;
}
poc/public/media.js
+63 -3
View File
@@ -40,6 +40,7 @@
loadError: false,
pageSize: 60,
backfill: null,
titles: null,
preferences: null,
/** Scrapes that finished while this page was open, until the tab is reloaded. */
completed: new Set()
@@ -75,6 +76,7 @@
</span>
</div>
<div id="backfill-summary" class="vk-bot-summary"></div>
<div id="title-lane" class="vk-bot-lane"></div>
<div id="backfill-jobs" class="vk-bot-jobs"></div>
</div>
@@ -101,6 +103,7 @@
const botLimited = overlay.querySelector("#bot-limited");
const backfillSummary = overlay.querySelector("#backfill-summary");
const backfillJobs = overlay.querySelector("#backfill-jobs");
const titleLane = overlay.querySelector("#title-lane");
const tunablesHost = overlay.querySelector("#bot-tunables");
const botErrors = overlay.querySelector("#bot-errors");
@@ -790,7 +793,6 @@
"vk-bot-job-member",
`${member.scrapeId} \u00b7 ${member.state} \u00b7 ${member.mode} \u00b7 ` +
`${member.backend} \u00b7 ${member.pending} pending` +
(member.pendingTitles ? ` + ${member.pendingTitles} titles` : "") +
(kinds ? ` (${kinds})` : "")
)
);
@@ -826,6 +828,57 @@
}
}
function renderTitleLane(status) {
if (!status) return;
titleLane.replaceChildren();
const head = el("div", "vk-bot-job-head");
head.append(el("span", "vk-bot-job-title", "Titles (YouTube)"));
head.append(el("span", `vk-bot-state vk-bot-state-${status.state}`, status.state));
const parts = [`${status.pending} pending`, `${status.resolved} titles`];
if (status.failed) parts.push(`${status.failed} failed`);
if (status.perMinute) parts.push(`${status.perMinute}/min`);
if (status.pending > 0) parts.push(`ETA ${formatEta(status.etaMs)}`);
if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
titleLane.append(head);
const actions = el("div", "vk-bot-job-actions");
if (status.state === "running") {
actions.append(button("Pause", () => void titleAction("pause")));
} else if (status.pending > 0) {
actions.append(button("Start", () => void titleAction("start")));
}
titleLane.append(actions);
const note = el(
"div",
"vk-bot-job-member",
"One-time catch-up for rows indexed before titles were stored; talks to YouTube only, " +
"100 per batch, 10 s between batches."
);
titleLane.append(note);
}
async function titleAction(action) {
try {
state.titles = await window.VK.api.post(`/api/media/titles/${action}`, {});
renderTitleLane(state.titles);
} catch (error) {
setStatus(`titles ${action} failed: ${error.message}`);
}
}
async function refreshTitles() {
try {
state.titles = await window.VK.api.get("/api/media/titles");
} catch {
state.titles = null;
}
renderTitleLane(state.titles);
}
async function refreshBackfill() {
try {
state.backfill = await window.VK.api.get("/api/media/backfill");
@@ -1765,6 +1818,7 @@
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
if (!state.preferences) void refreshPreferences();
void refreshBackfill();
void refreshTitles();
void loadScrapes().then(renderScrapes);
void refreshErrors();
} else {
@@ -1782,7 +1836,10 @@
state.completed.clear();
void loadScrapes();
if (state.view === "media") void loadMedia(true);
if (state.view === "bot") void refreshBackfill();
if (state.view === "bot") {
void refreshBackfill();
void refreshTitles();
}
});
overlay.querySelector("#backfill-start").addEventListener("click", () => void backfillAction("start"));
@@ -1812,7 +1869,10 @@
// Backfill progress while the Bot tab is open.
setInterval(() => {
if (state.view === "bot") void refreshBackfill();
if (state.view === "bot") {
void refreshBackfill();
void refreshTitles();
}
}, 3000);
// Infinite scroll: the buffer entering the viewport starts the next page.
poc/src/media.ts
+25
View File
@@ -17,6 +17,8 @@ import {
ScrapeEngine,
ScrapeStore,
scrapeDir,
TitleLane,
type TitleLaneStatus,
snowflakeFromIso,
resolveDbRoot,
type MediaKind,
@@ -171,6 +173,7 @@ export class MediaService {
private readonly rootDir = resolveDbRoot();
private readonly engine: ScrapeEngine;
private readonly backfill: BackfillQueue;
private readonly titles: TitleLane;
private bot: BotClient | null = null;
private botFetcher: SlopbotFetcher | null = null;
@@ -214,6 +217,14 @@ export class MediaService {
log: line => this.deps.log?.(line)
});
// The title catch-up lane talks to YouTube only, so it runs beside the backfill queue.
// It auto-starts because it is a one-time catch-up: with nothing pending it exits at once.
this.titles = new TitleLane(this.engine, {
rootDir: this.rootDir,
log: line => this.deps.log?.(line)
});
this.titles.start();
const stale = this.engine.reconcileStaleJobs();
if (stale.length) {
this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`);
@@ -268,6 +279,20 @@ export class MediaService {
};
}
titleStatus(): TitleLaneStatus {
return this.titles.status();
}
startTitles(): TitleLaneStatus {
this.titles.start();
return this.titles.status();
}
pauseTitles(): TitleLaneStatus {
this.titles.pause();
return this.titles.status();
}
startBackfill(chatId?: string): BackfillStatus {
this.backfill.start(chatId);
return this.backfillStatus();
poc/src/webui.ts
+21
View File
@@ -835,6 +835,27 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
})
);
app.get(
"/api/media/titles",
wrap(async (_req, res) => {
res.json({ ok: true, result: deps.media.titleStatus() });
})
);
app.post(
"/api/media/titles/start",
wrap(async (_req, res) => {
res.json({ ok: true, result: deps.media.startTitles() });
})
);
app.post(
"/api/media/titles/pause",
wrap(async (_req, res) => {
res.json({ ok: true, result: deps.media.pauseTitles() });
})
);
app.post(
"/api/media/backfill/start",
wrap(async (req, res) => {
poc/test/media-ui-smoke.mjs
+6
View File
@@ -202,6 +202,12 @@ try {
check("media toolbar hidden on the bot tab", !(await page.locator("#media-toolbar").isVisible()));
check("media filter drawer hidden on the bot tab", !(await page.locator(".vk-media-drawer").isVisible()));
await page.waitForFunction(
() => document.querySelector("#title-lane")?.textContent?.includes("Titles (YouTube)"),
{ timeout: 15000 }
);
check("title lane card renders", (await page.locator("#title-lane .vk-bot-job-head").count()) === 1);
const summaryLines = await page.locator(".vk-bot-summary-line").count();
step("backfill summary lines", summaryLines);
check("backfill summary renders", summaryLines >= 3, summaryLines);