zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the Bot-tab backfill control room and media strip rules

Browse Source
This commit is contained in:
Fisto
2026-09-17 09:04:27 +02:00
parent 16ce809270
commit 6ae2ff5ab3
14 changed files with 949 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/src/media.ts
+89 -27
View File
@@ -8,6 +8,7 @@
import { rmSync } from "node:fs";
import { isAbsolute, join } from "node:path";
import {
BackfillQueue,
betterCandidate,
dirSizeBytes,
isNearLimit,
@@ -19,6 +20,8 @@ import {
resolveDbRoot,
type MediaKind,
type MediaRow,
type BackfillSettings,
type BackfillStatus,
type ScrapeBackend,
type ScrapeMeta
} from "../../media-scrape/src/index.js";
@@ -153,11 +156,14 @@ const FILE_CONTENT_TYPES: Record<string, string> = {
export interface MediaStatus {
slopbot: SlopbotStatus;
scrapes: ScrapeMeta[];
/** ISO time while a Discord rate limit holds media traffic. */
limitedUntil: string | null;
}
export class MediaService {
private readonly rootDir = resolveDbRoot();
private readonly engine: ScrapeEngine;
private readonly backfill: BackfillQueue;
private bot: BotClient | null = null;
private botFetcher: SlopbotFetcher | null = null;
@@ -167,50 +173,102 @@ export class MediaService {
private readonly errors: MediaErrorEntry[] = [];
constructor(private readonly deps: MediaServiceDeps) {
const number = (value: string | undefined, fallback: number) => {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
};
const tunables = this.deps.getMediaPrefs().tunables;
this.engine = new ScrapeEngine({
rootDir: this.rootDir,
log: line => this.deps.log?.(line),
onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
previewMaxAttempts: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_ATTEMPTS, 3),
walkCyclePages: tunables.scrape.walkCyclePages,
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
pageMinMs: tunables.timers.pageMinMs,
pageMaxMs: tunables.timers.pageMaxMs,
maxRetries: tunables.scrape.maxRetries,
previewMaxAttempts: tunables.backfill.maxAttempts,
download: {
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25),
previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000),
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
maxFileBytes: tunables.scrape.maxFileBytes,
previewMaxBytes: tunables.backfill.maxBytes,
previewMinGapMs: tunables.backfill.previewMinGapMs,
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
previewBatchSize: tunables.backfill.batchSize,
previewBatchPauseMs: tunables.backfill.batchPauseMs,
minGapMs: tunables.scrape.downloadMinGapMs,
maxGapMs: tunables.scrape.downloadMaxGapMs
}
});
this.backfill = new BackfillQueue(this.engine, {
rootDir: this.rootDir,
getSettings: () => this.backfillSettings(),
getGaps: () => {
const live = this.deps.getMediaPrefs().tunables;
return { minGapMs: live.backfill.previewMinGapMs, maxGapMs: live.backfill.previewMaxGapMs };
},
log: line => this.deps.log?.(line)
});
const stale = this.engine.reconcileStaleJobs();
if (stale.length) {
this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`);
}
// Preview passes only run when a walk ends; fill in scrapes that paused early.
void this.backfillPreviews();
// Preview passes only run when a walk ends; the queue fills in scrapes that paused early.
this.backfill.autoStart();
}
private async backfillPreviews(): Promise<void> {
for (const meta of this.engine.list()) {
if (meta.state === "running" || meta.state === "queued") continue;
private backfillSettings(): BackfillSettings {
const tunables = this.deps.getMediaPrefs().tunables;
return {
batchSize: tunables.backfill.batchSize,
batchPauseMs: tunables.backfill.batchPauseMs,
auto: tunables.backfill.auto
};
}
try {
const added = await this.engine.backfillPreviews(meta.id);
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
} catch (error) {
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
/** Push preference changes into the live engine and backfill queue. */
applyTunables(): void {
const tunables = this.deps.getMediaPrefs().tunables;
this.engine.updateOptions({
walkCyclePages: tunables.scrape.walkCyclePages,
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
pageMinMs: tunables.timers.pageMinMs,
pageMaxMs: tunables.timers.pageMaxMs,
maxRetries: tunables.scrape.maxRetries,
previewMaxAttempts: tunables.backfill.maxAttempts,
download: {
maxFileBytes: tunables.scrape.maxFileBytes,
previewMaxBytes: tunables.backfill.maxBytes,
previewMinGapMs: tunables.backfill.previewMinGapMs,
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
previewBatchSize: tunables.backfill.batchSize,
previewBatchPauseMs: tunables.backfill.batchPauseMs,
minGapMs: tunables.scrape.downloadMinGapMs,
maxGapMs: tunables.scrape.downloadMaxGapMs
}
}
});
}
/** Backfill jobs, totals, and pacing, plus the engine's live cooldown. */
backfillStatus(): BackfillStatus {
const status = this.backfill.list();
const limit = this.engine.rateLimitStatus();
return {
...status,
limitedUntil: limit.active
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
: status.limitedUntil
};
}
startBackfill(chatId?: string): BackfillStatus {
this.backfill.start(chatId);
return this.backfillStatus();
}
pauseBackfill(chatId?: string): BackfillStatus {
this.backfill.pause(chatId);
return this.backfillStatus();
}
/** Log slopbot in on first use. False when it is disabled or the login failed. */
@@ -425,9 +483,13 @@ export class MediaService {
}
status(): MediaStatus {
const limit = this.engine.rateLimitStatus();
return {
slopbot: slopbotStatus(this.bot),
scrapes: this.engine.list()
scrapes: this.engine.list(),
limitedUntil: limit.active
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
: null
};
}