381 lines
12 KiB
TypeScript
381 lines
12 KiB
TypeScript
/*
|
|
* Media archive service: the daemon side of media-scrape/. See docs/media.md.
|
|
*
|
|
* Owns the scrape engine, the slopbot client, and the read path the Web UI and
|
|
* (later) MCP tools call.
|
|
*/
|
|
|
|
import { rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import {
|
|
dirSizeBytes,
|
|
isNearLimit,
|
|
planRetention,
|
|
ScrapeEngine,
|
|
ScrapeStore,
|
|
scrapeDir,
|
|
resolveDbRoot,
|
|
type MediaKind,
|
|
type MediaRow,
|
|
type ScrapeBackend,
|
|
type ScrapeMeta
|
|
} from "../../media-scrape/src/index.js";
|
|
import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js";
|
|
import {
|
|
loadSlopbotConfig,
|
|
slopbotFetcher,
|
|
slopbotStatus,
|
|
SlopbotBackend,
|
|
startSlopbot,
|
|
type SlopbotFetcher,
|
|
type SlopbotStatus
|
|
} from "../../media-scrape/slopbot/index.js";
|
|
import type { CoreClient } from "./core-client.js";
|
|
import type { MediaPreferences } from "./preferences.js";
|
|
|
|
type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
|
|
|
|
export interface MediaServiceDeps {
|
|
getCore: () => CoreClient | undefined;
|
|
isCoreConnected: () => boolean;
|
|
getMediaPrefs: () => MediaPreferences;
|
|
log?: (line: string) => void;
|
|
}
|
|
|
|
export interface StartScrapeInput {
|
|
channelId: string;
|
|
guildId?: string | null;
|
|
mode?: "links" | "download";
|
|
backend?: "auto" | "slopbot" | "puppet";
|
|
/** Set by a re-run of the command after the near-limit warning. */
|
|
confirm?: boolean;
|
|
}
|
|
|
|
export interface ScrapeBytes {
|
|
id: string;
|
|
channelId: string;
|
|
state: string;
|
|
createdAt: string;
|
|
bytes: number;
|
|
}
|
|
|
|
export interface MediaStorageStatus {
|
|
usedBytes: number;
|
|
limitBytes: number;
|
|
warnPercent: number;
|
|
overrunPercent: number;
|
|
nearLimit: boolean;
|
|
byKind: Record<MediaKind, number>;
|
|
scrapes: ScrapeBytes[];
|
|
}
|
|
|
|
/** Thrown when a scrape starts near the storage limit without confirmation. */
|
|
export class MediaLimitError extends Error {
|
|
readonly code = "MEDIA_LIMIT";
|
|
|
|
constructor(
|
|
message: string,
|
|
readonly usedBytes: number,
|
|
readonly limitBytes: number
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes: number): string {
|
|
const gib = bytes / 1024 ** 3;
|
|
return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
|
|
}
|
|
|
|
export interface MediaItemResult {
|
|
scrapeId: string;
|
|
itemId: number;
|
|
channelId: string;
|
|
kind: MediaKind;
|
|
url: string;
|
|
localUrl: string | null;
|
|
provider: string | null;
|
|
downloaded: boolean;
|
|
createdAt: string;
|
|
}
|
|
|
|
const FILE_CONTENT_TYPES: Record<string, string> = {
|
|
png: "image/png",
|
|
jpg: "image/jpeg",
|
|
jpeg: "image/jpeg",
|
|
gif: "image/gif",
|
|
webp: "image/webp",
|
|
avif: "image/avif",
|
|
mp4: "video/mp4",
|
|
webm: "video/webm",
|
|
mov: "video/quicktime",
|
|
mkv: "video/x-matroska",
|
|
mp3: "audio/mpeg",
|
|
ogg: "audio/ogg",
|
|
wav: "audio/wav",
|
|
m4a: "audio/mp4",
|
|
flac: "audio/flac"
|
|
};
|
|
|
|
export interface MediaStatus {
|
|
slopbot: SlopbotStatus;
|
|
scrapes: ScrapeMeta[];
|
|
}
|
|
|
|
export class MediaService {
|
|
private readonly rootDir = resolveDbRoot();
|
|
private readonly engine: ScrapeEngine;
|
|
|
|
private bot: BotClient | null = null;
|
|
private botFetcher: SlopbotFetcher | null = null;
|
|
private botStarting: Promise<void> | null = null;
|
|
|
|
constructor(private readonly deps: MediaServiceDeps) {
|
|
const number = (value: string | undefined, fallback: number) => {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
|
};
|
|
|
|
this.engine = new ScrapeEngine({
|
|
rootDir: this.rootDir,
|
|
log: line => this.deps.log?.(line),
|
|
download: {
|
|
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
|
|
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
|
|
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
|
|
}
|
|
});
|
|
}
|
|
|
|
/** Log slopbot in on first use. False when it is disabled or the login failed. */
|
|
private async ensureSlopbot(): Promise<boolean> {
|
|
if (this.bot) return true;
|
|
|
|
const config = loadSlopbotConfig();
|
|
if (!config.enabled) return false;
|
|
|
|
if (!this.botStarting) {
|
|
this.botStarting = startSlopbot(config)
|
|
.then(client => {
|
|
this.bot = client;
|
|
this.botFetcher = slopbotFetcher(client);
|
|
this.deps.log?.("slopbot connected");
|
|
})
|
|
.catch(error => {
|
|
this.deps.log?.(`slopbot login failed: ${String(error)}`);
|
|
});
|
|
}
|
|
|
|
await this.botStarting;
|
|
return this.bot !== null;
|
|
}
|
|
|
|
private puppetBackend(): ScrapeBackend {
|
|
return new PuppetBackend(async (channelId, limit, options) => {
|
|
const core = this.deps.getCore();
|
|
if (!core || !this.deps.isCoreConnected()) {
|
|
throw new Error("the core bridge is not connected");
|
|
}
|
|
return (await core.fetchMessages(channelId, limit, options)) as { messages?: unknown[] };
|
|
});
|
|
}
|
|
|
|
/** Combined size of every scrape directory, with the limit settings applied. */
|
|
storageStatus(): MediaStorageStatus {
|
|
const prefs = this.deps.getMediaPrefs();
|
|
const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
|
|
|
|
const scrapes = this.engine.list().map(meta => {
|
|
const dir = scrapeDir(this.rootDir, meta.id);
|
|
const bytes = dirSizeBytes(dir);
|
|
|
|
try {
|
|
const store = new ScrapeStore(dir);
|
|
try {
|
|
const sums = store.storedBytesByKind();
|
|
for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
|
|
} finally {
|
|
store.close();
|
|
}
|
|
} catch {
|
|
// an unreadable index counts as zero bytes
|
|
}
|
|
|
|
return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
|
|
});
|
|
|
|
const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
|
|
|
|
return {
|
|
usedBytes,
|
|
limitBytes: prefs.limitBytes,
|
|
warnPercent: prefs.warnPercent,
|
|
overrunPercent: prefs.overrunPercent,
|
|
nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
|
|
byKind,
|
|
scrapes
|
|
};
|
|
}
|
|
|
|
/** Mark the oldest completed scrapes when the usage is at or over the limit. */
|
|
refreshRetention(): MediaStorageStatus {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
return this.storageStatus();
|
|
}
|
|
|
|
private applyRetention(): void {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
for (const id of plan.delete) this.deleteScrape(id);
|
|
}
|
|
|
|
deleteScrape(id: string): boolean {
|
|
try {
|
|
rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
|
|
this.engine.forget(id);
|
|
this.deps.log?.(`deleted scrape ${id}`);
|
|
return true;
|
|
} catch (error) {
|
|
this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Delete the given scrapes, or every scrape when no IDs are given. */
|
|
cleanMedia(ids?: string[]): number {
|
|
const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
|
|
let deleted = 0;
|
|
for (const id of targets) {
|
|
if (this.deleteScrape(id)) deleted++;
|
|
}
|
|
return deleted;
|
|
}
|
|
|
|
async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
|
|
this.applyRetention();
|
|
|
|
const status = this.storageStatus();
|
|
if (status.nearLimit && !input.confirm) {
|
|
throw new MediaLimitError(
|
|
`media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
|
|
status.usedBytes,
|
|
status.limitBytes
|
|
);
|
|
}
|
|
|
|
const mode = input.mode ?? "links";
|
|
let backend: ScrapeBackend;
|
|
|
|
if (input.backend === "puppet") {
|
|
backend = this.puppetBackend();
|
|
} else {
|
|
const available = await this.ensureSlopbot();
|
|
const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false);
|
|
|
|
if (input.backend === "slopbot" && (!available || !inGuild)) {
|
|
throw new Error("slopbot is not connected to that guild");
|
|
}
|
|
|
|
const fetcher = this.botFetcher;
|
|
backend =
|
|
available && inGuild && fetcher ? new SlopbotBackend(fetcher) : this.puppetBackend();
|
|
}
|
|
|
|
return this.engine.start({
|
|
channelId: input.channelId,
|
|
guildId: input.guildId ?? null,
|
|
mode,
|
|
backend
|
|
});
|
|
}
|
|
|
|
stopScrape(id: string): void {
|
|
this.engine.stop(id);
|
|
}
|
|
|
|
listScrapes(): ScrapeMeta[] {
|
|
return this.engine.list();
|
|
}
|
|
|
|
getScrape(id: string): ScrapeMeta | null {
|
|
return this.engine.get(id);
|
|
}
|
|
|
|
status(): MediaStatus {
|
|
return {
|
|
slopbot: slopbotStatus(this.bot),
|
|
scrapes: this.engine.list()
|
|
};
|
|
}
|
|
|
|
/** Absolute path and content type for a downloaded file, or null. */
|
|
getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.file_path) return null;
|
|
|
|
const extension = item.file_path.split(".").pop()?.toLowerCase() ?? "";
|
|
return {
|
|
path: join(this.rootDir, scrapeId, item.file_path),
|
|
contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
/** One media row inside a scrape, with its scrape and channel for context. */
|
|
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
|
|
try {
|
|
const row = store.getMedia(itemId);
|
|
return row ? { ...row, scrapeId, channelId: meta.channelId } : null;
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
/** Read media rows across every scrape, newest scrape first. */
|
|
queryMedia(
|
|
options: { kind?: MediaKind; scrapeId?: string; limit?: number; offset?: number } = {}
|
|
): MediaItemResult[] {
|
|
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
|
|
const offset = Math.max(0, options.offset ?? 0);
|
|
const results: MediaItemResult[] = [];
|
|
let skipped = 0;
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted") continue;
|
|
if (options.scrapeId && meta.id !== options.scrapeId) continue;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
const rows = store.queryMedia({ kind: options.kind, limit: limit + offset });
|
|
for (const row of rows) {
|
|
if (skipped++ < offset) continue;
|
|
if (results.length >= limit) break;
|
|
results.push({
|
|
scrapeId: meta.id,
|
|
itemId: row.id,
|
|
channelId: meta.channelId,
|
|
kind: row.kind,
|
|
url: row.url,
|
|
localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null,
|
|
provider: row.provider,
|
|
downloaded: row.downloaded === 1,
|
|
createdAt: row.created_at
|
|
});
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
|
|
if (results.length >= limit) break;
|
|
}
|
|
|
|
return results;
|
|
}
|
|
}
|