zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the Bot-tab backfill control room and media strip rules

Browse Source
This commit is contained in:
Fisto
2026-09-17 09:04:27 +02:00
parent 16ce809270
commit 6ae2ff5ab3
14 changed files with 949 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/src/control-bridge.ts
+8 -7
View File
@@ -42,13 +42,6 @@ let puppet: Puppet | undefined;
let coreConnected = false;
let connecting = false;
const media = new MediaService({
getCore: () => core,
isCoreConnected: () => coreConnected,
getMediaPrefs: () => preferences.get().media,
log: line => console.log(`[media] ${line}`)
});
async function connectLoop(): Promise<void> {
if (connecting) return;
connecting = true;
@@ -185,6 +178,14 @@ const preferences = new PreferencesStore(
);
const stt = new Stt();
// After `preferences`: the media engine and backfill queue read their tunables at construction.
const media = new MediaService({
getCore: () => core,
isCoreConnected: () => coreConnected,
getMediaPrefs: () => preferences.get().media,
log: line => console.log(`[media] ${line}`)
});
const audio = new AudioService({
cacheDir: process.env.VESKSLOP_CACHE_DIR,
transcripts,
poc/src/media.ts
+89 -27
View File
@@ -8,6 +8,7 @@
import { rmSync } from "node:fs";
import { isAbsolute, join } from "node:path";
import {
BackfillQueue,
betterCandidate,
dirSizeBytes,
isNearLimit,
@@ -19,6 +20,8 @@ import {
resolveDbRoot,
type MediaKind,
type MediaRow,
type BackfillSettings,
type BackfillStatus,
type ScrapeBackend,
type ScrapeMeta
} from "../../media-scrape/src/index.js";
@@ -153,11 +156,14 @@ const FILE_CONTENT_TYPES: Record<string, string> = {
export interface MediaStatus {
slopbot: SlopbotStatus;
scrapes: ScrapeMeta[];
/** ISO time while a Discord rate limit holds media traffic. */
limitedUntil: string | null;
}
export class MediaService {
private readonly rootDir = resolveDbRoot();
private readonly engine: ScrapeEngine;
private readonly backfill: BackfillQueue;
private bot: BotClient | null = null;
private botFetcher: SlopbotFetcher | null = null;
@@ -167,50 +173,102 @@ export class MediaService {
private readonly errors: MediaErrorEntry[] = [];
constructor(private readonly deps: MediaServiceDeps) {
const number = (value: string | undefined, fallback: number) => {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
};
const tunables = this.deps.getMediaPrefs().tunables;
this.engine = new ScrapeEngine({
rootDir: this.rootDir,
log: line => this.deps.log?.(line),
onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3),
walkCyclePages: tunables.scrape.walkCyclePages,
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
pageMinMs: tunables.timers.pageMinMs,
pageMaxMs: tunables.timers.pageMaxMs,
maxRetries: tunables.scrape.maxRetries,
previewMaxAttempts: tunables.backfill.maxAttempts,
download: {
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25),
previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000),
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
maxFileBytes: tunables.scrape.maxFileBytes,
previewMaxBytes: tunables.backfill.maxBytes,
previewMinGapMs: tunables.backfill.previewMinGapMs,
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
previewBatchSize: tunables.backfill.batchSize,
previewBatchPauseMs: tunables.backfill.batchPauseMs,
minGapMs: tunables.scrape.downloadMinGapMs,
maxGapMs: tunables.scrape.downloadMaxGapMs
}
});
this.backfill = new BackfillQueue(this.engine, {
rootDir: this.rootDir,
getSettings: () => this.backfillSettings(),
getGaps: () => {
const live = this.deps.getMediaPrefs().tunables;
return { minGapMs: live.backfill.previewMinGapMs, maxGapMs: live.backfill.previewMaxGapMs };
},
log: line => this.deps.log?.(line)
});
const stale = this.engine.reconcileStaleJobs();
if (stale.length) {
this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`);
}
// Preview passes only run when a walk ends; fill in scrapes that paused early.
void this.backfillPreviews();
// Preview passes only run when a walk ends; the queue fills in scrapes that paused early.
this.backfill.autoStart();
}
private async backfillPreviews(): Promise<void> {
for (const meta of this.engine.list()) {
if (meta.state === "running" || meta.state === "queued") continue;
private backfillSettings(): BackfillSettings {
const tunables = this.deps.getMediaPrefs().tunables;
return {
batchSize: tunables.backfill.batchSize,
batchPauseMs: tunables.backfill.batchPauseMs,
auto: tunables.backfill.auto
};
}
try {
const added = await this.engine.backfillPreviews(meta.id);
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
} catch (error) {
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
/** Push preference changes into the live engine and backfill queue. */
applyTunables(): void {
const tunables = this.deps.getMediaPrefs().tunables;
this.engine.updateOptions({
walkCyclePages: tunables.scrape.walkCyclePages,
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
pageMinMs: tunables.timers.pageMinMs,
pageMaxMs: tunables.timers.pageMaxMs,
maxRetries: tunables.scrape.maxRetries,
previewMaxAttempts: tunables.backfill.maxAttempts,
download: {
maxFileBytes: tunables.scrape.maxFileBytes,
previewMaxBytes: tunables.backfill.maxBytes,
previewMinGapMs: tunables.backfill.previewMinGapMs,
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
previewBatchSize: tunables.backfill.batchSize,
previewBatchPauseMs: tunables.backfill.batchPauseMs,
minGapMs: tunables.scrape.downloadMinGapMs,
maxGapMs: tunables.scrape.downloadMaxGapMs
}
}
});
}
/** Backfill jobs, totals, and pacing, plus the engine's live cooldown. */
backfillStatus(): BackfillStatus {
const status = this.backfill.list();
const limit = this.engine.rateLimitStatus();
return {
...status,
limitedUntil: limit.active
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
: status.limitedUntil
};
}
startBackfill(chatId?: string): BackfillStatus {
this.backfill.start(chatId);
return this.backfillStatus();
}
pauseBackfill(chatId?: string): BackfillStatus {
this.backfill.pause(chatId);
return this.backfillStatus();
}
/** Log slopbot in on first use. False when it is disabled or the login failed. */
@@ -425,9 +483,13 @@ export class MediaService {
}
status(): MediaStatus {
const limit = this.engine.rateLimitStatus();
return {
slopbot: slopbotStatus(this.bot),
scrapes: this.engine.list()
scrapes: this.engine.list(),
limitedUntil: limit.active
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
: null
};
}
poc/src/preferences.ts
+117 -2
View File
@@ -76,6 +76,43 @@ export interface AudioPreferences {
duck: boolean;
}
/** Pacing and limit knobs for preview backfilling, scraping, and cooldowns. */
export interface MediaTunables {
backfill: {
/** Preview rows per batch. */
batchSize: number;
/** Cooldown between batches, ms. */
batchPauseMs: number;
/** Transient failures allowed before a row is closed as unreachable. */
maxAttempts: number;
/** Gap between preview fetches, ms. */
previewMinGapMs: number;
previewMaxGapMs: number;
/** Largest preview to cache, bytes. */
maxBytes: number;
/** Start backfill jobs automatically on daemon boot. */
auto: boolean;
};
scrape: {
/** Walk pages between inline preview cycles. 0 defers previews to the end. */
walkCyclePages: number;
/** Largest file to download, bytes. */
maxFileBytes: number;
/** Gap between file downloads, ms. */
downloadMinGapMs: number;
downloadMaxGapMs: number;
/** Page fetch retries before a scrape fails. */
maxRetries: number;
};
timers: {
/** Gap between walk pages, ms. */
pageMinMs: number;
pageMaxMs: number;
/** Cooldown after a Discord rate limit, ms. */
rateLimitPauseMs: number;
};
}
export interface MediaPreferences {
/** Combined size limit for the media DB root: databases, downloads, previews. */
limitBytes: number;
@@ -83,6 +120,7 @@ export interface MediaPreferences {
warnPercent: number;
/** No-delete overrun allowance in percent, 0-100. */
overrunPercent: number;
tunables: MediaTunables;
}
export interface WebPreferences {
@@ -115,13 +153,89 @@ const DEFAULTS: WebPreferences = {
media: {
limitBytes: 30 * 1024 * 1024 * 1024,
warnPercent: 90,
overrunPercent: 5
overrunPercent: 5,
tunables: {
backfill: {
batchSize: 25,
batchPauseMs: 60_000,
maxAttempts: 3,
previewMinGapMs: 200,
previewMaxGapMs: 600,
maxBytes: 5 * 1024 * 1024,
auto: true
},
scrape: {
walkCyclePages: 5,
maxFileBytes: 50 * 1024 * 1024,
downloadMinGapMs: 1000,
downloadMaxGapMs: 3000,
maxRetries: 2
},
timers: {
pageMinMs: 1200,
pageMaxMs: 3500,
rateLimitPauseMs: 60_000
}
}
}
};
const MEDIA_LIMIT_MIN = 256 * 1024 * 1024;
const MEDIA_LIMIT_MAX = 10 * 1024 * 1024 * 1024 * 1024;
function intIn(value: unknown, min: number, max: number, fallback: number): number {
const number = Number(value);
if (!Number.isFinite(number)) return fallback;
return Math.min(max, Math.max(min, Math.round(number)));
}
function normalizeTunables(raw: any, fallback: MediaTunables): MediaTunables {
const backfill = raw?.backfill ?? {};
const scrape = raw?.scrape ?? {};
const timers = raw?.timers ?? {};
const previewMinGapMs = intIn(backfill.previewMinGapMs, 0, 60_000, fallback.backfill.previewMinGapMs);
const previewMaxGapMs = intIn(
backfill.previewMaxGapMs,
previewMinGapMs,
60_000,
fallback.backfill.previewMaxGapMs
);
const downloadMinGapMs = intIn(scrape.downloadMinGapMs, 0, 600_000, fallback.scrape.downloadMinGapMs);
const downloadMaxGapMs = intIn(
scrape.downloadMaxGapMs,
downloadMinGapMs,
600_000,
fallback.scrape.downloadMaxGapMs
);
const pageMinMs = intIn(timers.pageMinMs, 0, 600_000, fallback.timers.pageMinMs);
const pageMaxMs = intIn(timers.pageMaxMs, pageMinMs, 600_000, fallback.timers.pageMaxMs);
return {
backfill: {
batchSize: intIn(backfill.batchSize, 1, 500, fallback.backfill.batchSize),
batchPauseMs: intIn(backfill.batchPauseMs, 0, 3_600_000, fallback.backfill.batchPauseMs),
maxAttempts: intIn(backfill.maxAttempts, 1, 100, fallback.backfill.maxAttempts),
previewMinGapMs,
previewMaxGapMs,
maxBytes: intIn(backfill.maxBytes, 65_536, 2 * 1024 ** 3, fallback.backfill.maxBytes),
auto: typeof backfill.auto === "boolean" ? backfill.auto : fallback.backfill.auto
},
scrape: {
walkCyclePages: intIn(scrape.walkCyclePages, 0, 100, fallback.scrape.walkCyclePages),
maxFileBytes: intIn(scrape.maxFileBytes, 1024 ** 2, 8 * 1024 ** 3, fallback.scrape.maxFileBytes),
downloadMinGapMs,
downloadMaxGapMs,
maxRetries: intIn(scrape.maxRetries, 0, 10, fallback.scrape.maxRetries)
},
timers: {
pageMinMs,
pageMaxMs,
rateLimitPauseMs: intIn(timers.rateLimitPauseMs, 0, 3_600_000, fallback.timers.rateLimitPauseMs)
}
};
}
function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences {
const limit = Number(raw?.limitBytes);
const warn = Number(raw?.warnPercent);
@@ -136,7 +250,8 @@ function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences
overrunPercent:
Number.isFinite(overrun) && overrun >= 0 && overrun <= 100
? Math.round(overrun)
: fallback.overrunPercent
: fallback.overrunPercent,
tunables: normalizeTunables(raw?.tunables, fallback.tunables)
};
}
poc/src/webui.ts
+28 -2
View File
@@ -478,7 +478,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
: current.warnPercent,
overrunPercent: Number.isFinite(Number(mediaPrefs?.overrunPercent))
? Number(mediaPrefs.overrunPercent)
: current.overrunPercent
: current.overrunPercent,
tunables: mediaPrefs?.tunables ?? current.tunables
};
}
@@ -487,7 +488,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
deps.audio.setDuckEnabled(patch.audio.duck);
}
res.json({ ok: true, result: deps.preferences.update(patch) });
const updated = deps.preferences.update(patch);
if (mediaPrefs !== undefined) deps.media.applyTunables();
res.json({ ok: true, result: updated });
})
);
@@ -773,6 +776,29 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
})
);
app.get(
"/api/media/backfill",
wrap(async (_req, res) => {
res.json({ ok: true, result: deps.media.backfillStatus() });
})
);
app.post(
"/api/media/backfill/start",
wrap(async (req, res) => {
const chatId = req.body?.chatId ? String(req.body.chatId) : undefined;
res.json({ ok: true, result: deps.media.startBackfill(chatId) });
})
);
app.post(
"/api/media/backfill/pause",
wrap(async (req, res) => {
const chatId = req.body?.chatId ? String(req.body.chatId) : undefined;
res.json({ ok: true, result: deps.media.pauseBackfill(chatId) });
})
);
app.get(
"/api/media/:scrapeId/:itemId/thumb",
wrap(async (req, res) => {