zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run the title catch-up in its own YouTube lane

Browse Source
This commit is contained in:
Fisto
2026-09-17 14:25:08 +02:00
parent 8befc05591
commit 9481286169
16 changed files with 495 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/backfill.ts
+2 -11
View File
@@ -29,8 +29,6 @@ export interface BackfillMember {
backend: BackendName;
mode: ScrapeMode;
pending: number;
/** YouTube-like rows still missing a title. */
pendingTitles: number;
cached: number;
failed: number;
total: number;
@@ -154,7 +152,6 @@ export class BackfillQueue {
backend: meta.backend,
mode: meta.mode,
pending: 0,
pendingTitles: 0,
cached: 0,
failed: 0,
total: 0,
@@ -168,7 +165,6 @@ export class BackfillQueue {
try {
const counts = store.previewCounts();
member.pending = counts.pending;
member.pendingTitles = counts.titlePending;
member.cached = counts.cached;
member.failed = counts.failed;
member.total = counts.total;
@@ -209,10 +205,7 @@ export class BackfillQueue {
for (const [chatId, group] of groups) {
// Hide chats that are fully backfilled unless their job did work this session.
const tracked = this.jobs.get(chatId);
const pending = group.members.reduce(
(sum, member) => sum + member.pending + member.pendingTitles,
0
);
const pending = group.members.reduce((sum, member) => sum + member.pending, 0);
if (!tracked && pending === 0) continue;
const perMinute = tracked?.perMinute ?? this.estimatedPerMinute(settings, gaps);
@@ -418,9 +411,7 @@ export class BackfillQueue {
while (!job.stop) {
const group = this.plan(false).get(job.chatId);
const member = group?.members.find(
candidate => candidate.pending > 0 || candidate.pendingTitles > 0
);
const member = group?.members.find(candidate => candidate.pending > 0);
if (!member) break;
const limit = this.engine.rateLimitStatus();
media-scrape/src/engine.ts
+63 -42
View File
@@ -46,14 +46,21 @@ export interface ScrapeBackend {
fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage>;
}
/** One batch of title resolution, as returned by `titleBatch`. */
export interface TitleBatchResult {
titled: number;
/** Rows closed out because the provider never returned a title. */
failed: number;
remaining: number;
stopped: boolean;
}
/** One batch of preview work, as returned by `previewBatch`. */
export interface PreviewBatchResult {
cached: number;
skipped: number;
failed: number;
ratelimited: number;
/** Titles resolved in this batch. */
titled: number;
/** Rows still pending after the batch. */
remaining: number;
/** True when a stop request interrupted the batch. */
@@ -550,7 +557,7 @@ export class ScrapeEngine {
const dir = join(this.options.rootDir, scrapeId);
const meta = this.active.get(scrapeId) ?? readScrapeMeta(dir);
if (!meta) {
return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, titled: 0, remaining: 0, stopped: false };
return { cached: 0, skipped: 0, failed: 0, ratelimited: 0, remaining: 0, stopped: false };
}
const releaseTraffic = await this.acquireTraffic();
@@ -563,6 +570,56 @@ export class ScrapeEngine {
}
}
/**
* Resolve YouTube titles for up to `limit` rows. No media-traffic lock: these requests go to
* YouTube's oEmbed, never Discord, so the lane can run beside a walk. The caller paces.
*/
async titleBatch(scrapeId: string, limit: number): Promise<TitleBatchResult> {
const dir = join(this.options.rootDir, scrapeId);
const meta = this.active.get(scrapeId) ?? readScrapeMeta(dir);
const result: TitleBatchResult = { titled: 0, failed: 0, remaining: 0, stopped: false };
if (!meta) return result;
const store = new ScrapeStore(dir);
const options = this.previewOptions(meta);
try {
for (const row of store.pendingTitles(Math.max(1, limit))) {
if (this.stopRequested.has(meta.id)) {
meta.state = "paused";
result.stopped = true;
break;
}
const title = await resolveEmbedTitle(row.url, options);
if (title) {
store.markTitle(row.id, title);
result.titled++;
continue;
}
const attempts = store.bumpTitleAttempt(row.id);
const maxAttempts = this.options.previewMaxAttempts ?? 3;
if (attempts >= maxAttempts) {
store.markTitleError(row.id, `no title after ${attempts} attempt(s)`);
result.failed++;
this.log(
`${meta.id}: title #${row.id} gave up after ${attempts} attempts (${row.url.slice(0, 60)})`
);
} else {
this.log(
`${meta.id}: title #${row.id} unresolved (${attempts}/${maxAttempts}) (${row.url.slice(0, 60)})`
);
}
}
result.remaining = store.pendingTitles(5000).length;
} finally {
store.close();
}
return result;
}
/** One batch of preview rows. Mutates the store and the meta counters. */
private async previewRows(
store: ScrapeStore,
@@ -576,47 +633,11 @@ export class ScrapeEngine {
skipped: 0,
failed: 0,
ratelimited: 0,
titled: 0,
remaining: 0,
stopped: false
};
// Title work first, then previews with what is left of the batch. YouTube rows that
// never got a title share the queue with the preview pass.
const titleBudget = store.pendingTitles(1).length > 0 && limit > 1 ? Math.ceil(limit / 2) : limit;
for (const row of store.pendingTitles(titleBudget)) {
if (this.stopRequested.has(meta.id)) {
meta.state = "paused";
result.stopped = true;
break;
}
const title = await resolveEmbedTitle(row.url, options);
if (title) {
store.markTitle(row.id, title);
result.titled++;
} else {
const attempts = store.bumpTitleAttempt(row.id);
const maxAttempts = this.options.previewMaxAttempts ?? 3;
if (attempts >= maxAttempts) {
store.markTitleError(row.id, `no title after ${attempts} attempt(s)`);
this.log(`${meta.id}: title #${row.id} gave up after ${attempts} attempts (${row.url.slice(0, 60)})`);
} else {
this.log(`${meta.id}: title #${row.id} unresolved (${attempts}/${maxAttempts}) (${row.url.slice(0, 60)})`);
}
}
this.persist(dir, meta);
await this.sleep(previewGapMs(options));
}
if (result.stopped) {
result.remaining = store.pendingPreviews(5000, true).length;
return result;
}
const previewLimit = Math.max(1, limit - result.titled);
const pending = store.pendingPreviews(previewLimit, true);
const pending = store.pendingPreviews(limit, true);
for (const row of pending) {
if (this.stopRequested.has(meta.id)) {
@@ -752,10 +773,10 @@ export class ScrapeEngine {
}
result.remaining = store.pendingPreviews(5000, true).length;
if (pending.length || result.titled) {
if (pending.length) {
this.log(
`${meta.id}: preview pass done: ${result.cached} cached, ${result.skipped} skipped, ` +
`${result.failed} failed, ${result.titled} titled, ${result.remaining} still pending`
`${result.failed} failed, ${result.remaining} still pending`
);
}
media-scrape/src/index.ts
+1
View File
@@ -37,6 +37,7 @@ export {
type BackfillStatus,
type BackfillTotals
} from "./backfill.js";
export { TitleLane, type TitleLaneOptions, type TitleLaneStatus } from "./titles.js";
export {
ScrapeEngine,
type FetchPage,
media-scrape/src/store.ts
+20 -7
View File
@@ -538,6 +538,10 @@ export class ScrapeStore {
failed: number;
/** YouTube-like rows still missing a title. */
titlePending: number;
/** Rows with a stored title. */
titled: number;
/** Rows closed out without a title. */
titleFailed: number;
byKind: Record<MediaKind, { total: number; pending: number; cached: number; failed: number }>;
} {
const empty = () => ({ total: 0, pending: 0, cached: 0, failed: 0 });
@@ -583,15 +587,24 @@ export class ScrapeStore {
const titles = this.db
.prepare(
`SELECT COUNT(*) AS count FROM media
WHERE title IS NULL AND title_error IS NULL
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
OR lower(url) LIKE '%youtube.com/%'
OR lower(url) LIKE '%youtu.be/%')`
`SELECT
SUM(title IS NULL AND title_error IS NULL
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
OR lower(url) LIKE '%youtube.com/%'
OR lower(url) LIKE '%youtu.be/%')) AS pending,
SUM(title IS NOT NULL) AS titled,
SUM(title IS NULL AND title_error IS NOT NULL) AS failed
FROM media`
)
.get() as { count: number | null } | undefined;
.get() as { pending: number | null; titled: number | null; failed: number | null } | undefined;
return { ...totals, titlePending: titles?.count ?? 0, byKind };
return {
...totals,
titlePending: titles?.pending ?? 0,
titled: titles?.titled ?? 0,
titleFailed: titles?.failed ?? 0,
byKind
};
}
/** Count one transient preview failure and return the new total. */
media-scrape/src/titles.ts
+202
View File
@@ -0,0 +1,202 @@
/*
* The YouTube title catch-up lane.
*
* Titles normally arrive with the scrape (Discord embeds carry them), so this lane only exists
* for rows indexed before titles were stored. It talks to YouTube's oEmbed directly, never to
* Discord, so it takes no media-traffic lock and can run beside a walk. Pacing is hard-coded
* (100 rows per batch, 10 s between batches): the lane is a one-time catch-up, not a tuned knob.
*/
import { ScrapeEngine } from "./engine.js";
import { listScrapes, ScrapeStore, scrapeDir } from "./store.js";
export interface TitleLaneStatus {
state: "idle" | "running" | "paused" | "done";
pending: number;
resolved: number;
failed: number;
/** Rows handled since the lane started. */
processed: number;
startedAt: string | null;
finishedAt: string | null;
perMinute: number | null;
etaMs: number | null;
/** The scrape being worked on right now. */
currentScrapeId: string | null;
}
export interface TitleLaneOptions {
rootDir: string;
log?: (line: string) => void;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
/** Hard-coded in production; tests override. */
batchSize?: number;
batchPauseMs?: number;
}
const BATCH_SIZE = 100;
const BATCH_PAUSE_MS = 10_000;
interface LaneState {
state: TitleLaneStatus["state"];
stop: boolean;
startedAt: number | null;
finishedAt: number | null;
processed: number;
perMinute: number | null;
currentScrapeId: string | null;
}
const defaultSleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms));
export class TitleLane {
private readonly state: LaneState = {
state: "idle",
stop: false,
startedAt: null,
finishedAt: null,
processed: 0,
perMinute: null,
currentScrapeId: null
};
private active: Promise<void> = Promise.resolve();
constructor(
private readonly engine: ScrapeEngine,
private readonly options: TitleLaneOptions
) {}
private now(): number {
return this.options.now?.() ?? Date.now();
}
private sleep(ms: number): Promise<void> {
return (this.options.sleep ?? defaultSleep)(ms);
}
/** Totals across every scrape, with the lane's own run state. */
status(): TitleLaneStatus {
let pending = 0;
let resolved = 0;
let failed = 0;
for (const meta of listScrapes(this.options.rootDir)) {
if (meta.state === "deleted" || meta.state === "deleting" || meta.state === "pending-delete") {
continue;
}
try {
const store = new ScrapeStore(scrapeDir(this.options.rootDir, meta.id));
try {
const counts = store.previewCounts();
pending += counts.titlePending;
resolved += counts.titled ?? 0;
failed += counts.titleFailed ?? 0;
} finally {
store.close();
}
} catch {
// an unreadable index counts as nothing to do
}
}
const runMs = this.state.startedAt ? Math.max(1, this.now() - this.state.startedAt) : 0;
const perMinute =
this.state.perMinute ??
(runMs && this.state.processed ? this.state.processed / (runMs / 60_000) : null);
return {
state: this.state.state,
pending,
resolved,
failed,
processed: this.state.processed,
startedAt: this.state.startedAt ? new Date(this.state.startedAt).toISOString() : null,
finishedAt: this.state.finishedAt ? new Date(this.state.finishedAt).toISOString() : null,
perMinute: perMinute === null ? null : Math.round(perMinute * 10) / 10,
etaMs:
pending > 0 && perMinute && perMinute > 0
? Math.round((pending / perMinute) * 60_000)
: null,
currentScrapeId: this.state.currentScrapeId
};
}
start(): void {
this.state.stop = false;
this.state.state = "running";
this.state.startedAt = this.state.startedAt ?? this.now();
this.state.finishedAt = null;
this.options.log?.("titles: lane started");
this.schedule();
}
pause(): void {
if (this.state.state !== "running") return;
this.state.stop = true;
this.state.state = "paused";
this.options.log?.("titles: pausing");
}
/** Resolves when the lane has stopped for now. */
idle(): Promise<void> {
return this.active;
}
private schedule(): void {
this.active = this.active.then(async () => {
if (this.state.state !== "running") return;
await this.run();
});
}
private async run(): Promise<void> {
const batchSize = Math.max(1, this.options.batchSize ?? BATCH_SIZE);
const batchPauseMs = Math.max(0, this.options.batchPauseMs ?? BATCH_PAUSE_MS);
while (!this.state.stop) {
const meta = listScrapes(this.options.rootDir).find(candidate => {
if (candidate.state === "deleted" || candidate.state === "deleting") return false;
try {
const store = new ScrapeStore(scrapeDir(this.options.rootDir, candidate.id));
try {
return store.previewCounts().titlePending > 0;
} finally {
store.close();
}
} catch {
return false;
}
});
if (!meta) break;
this.state.currentScrapeId = meta.id;
const result = await this.engine.titleBatch(meta.id, batchSize);
this.state.processed += result.titled + result.failed;
const elapsed = Math.max(1, this.now() - (this.state.startedAt ?? this.now()));
this.state.perMinute = this.state.processed / (elapsed / 60_000);
this.options.log?.(
`titles: ${meta.id} batch done: +${result.titled} titled, ${result.failed} closed, ` +
`${result.remaining} left in scrape`
);
if (this.state.stop) break;
if (result.stopped) {
this.state.state = "paused";
break;
}
if (batchPauseMs > 0) await this.sleep(batchPauseMs);
}
this.state.currentScrapeId = null;
if (this.state.state === "running") this.state.state = this.state.stop ? "paused" : "done";
this.state.finishedAt = this.state.state === "done" ? this.now() : null;
this.options.log?.(`titles: lane ${this.state.state} (${this.state.processed} rows this run)`);
}
}
media-scrape/test/run.ts
+55 -3
View File
@@ -21,6 +21,7 @@ import {
import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
import { BackfillQueue } from "../src/backfill.js";
import { TitleLane } from "../src/titles.js";
import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js";
import { betterCandidate, sortCandidates } from "../src/dedupe.js";
@@ -1502,7 +1503,7 @@ void (async () => {
store.close();
});
await test("previewBatch resolves titles and gives up after the cap", async () => {
await test("titleBatch resolves titles and gives up after the cap", async () => {
const embed = {
type: "video",
url: "https://www.youtube.com/watch?v=titlepass",
@@ -1534,7 +1535,8 @@ void (async () => {
const started = await engine.start({ channelId: "135", guildId: null, mode: "links", backend });
await engine.processQueue();
await engine.previewBatch(started.id, 5);
const batch = await engine.titleBatch(started.id, 5);
assert.equal(batch.titled, 1);
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "video" })[0];
@@ -1561,7 +1563,8 @@ void (async () => {
const dead = await deadEngine.start({ channelId: "136", guildId: null, mode: "links", backend: deadBackend });
await deadEngine.processQueue();
await deadEngine.previewBatch(dead.id, 5);
const deadBatch = await deadEngine.titleBatch(dead.id, 5);
assert.equal(deadBatch.failed, 1);
const deadStore = new ScrapeStore(join(root, dead.id));
const deadRow = deadStore.queryMedia({ kind: "video" })[0];
@@ -1570,6 +1573,55 @@ void (async () => {
deadStore.close();
});
await test("title lane runs a catch-up across scrapes", async () => {
const laneRoot = join(root, "lane-root");
mkdirSync(laneRoot, { recursive: true });
const embed = (id: string) => ({
type: "video",
url: `https://www.youtube.com/watch?v=${id}`,
title: null,
provider: "YouTube",
videoUrl: `https://www.youtube.com/watch?v=${id}`,
imageUrl: null,
thumbnailUrl: null
});
const backend = new FakeBackend([
{ messages: [message("1500", { embeds: [embed("lane1"), embed("lane2")] })] }
]);
const engine = engineFor(laneRoot, backend, {
download: {
fetchImpl: async (input: any) =>
new Response(JSON.stringify({ title: `T ${String(input).length}` }), {
status: 200,
headers: { "content-type": "application/json" }
}),
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
}
});
await engine.start({ channelId: "137", guildId: null, mode: "links", backend });
await engine.processQueue();
const lane = new TitleLane(engine, {
rootDir: laneRoot,
batchSize: 1,
batchPauseMs: 0,
sleep: async () => {},
log: () => {}
});
lane.start();
await lane.idle();
const status = lane.status();
assert.equal(status.state, "done");
assert.equal(status.pending, 0);
assert.equal(status.resolved, 2);
assert.equal(status.failed, 0);
assert.ok(status.processed >= 2);
});
await test("title search matches word prefixes and exact phrases", () => {
const dir = join(root, "title-search");
mkdirSync(dir, { recursive: true });