Add media storage limits, retention, and the near-limit flow

poc/src/media.ts

@@ -5,7 +5,11 @@
  * (later) MCP tools call.
  */
 
+import { rmSync } from "node:fs";
 import {
+    dirSizeBytes,
+    isNearLimit,
+    planRetention,
     ScrapeEngine,
     ScrapeStore,
     scrapeDir,
@@ -25,12 +29,14 @@ import {
     type SlopbotStatus
 } from "../../media-scrape/slopbot/index.js";
 import type { CoreClient } from "./core-client.js";
+import type { MediaPreferences } from "./preferences.js";
 
 type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
 
 export interface MediaServiceDeps {
     getCore: () => CoreClient | undefined;
     isCoreConnected: () => boolean;
+    getMediaPrefs: () => MediaPreferences;
     log?: (line: string) => void;
 }
 
@@ -39,6 +45,44 @@ export interface StartScrapeInput {
     guildId?: string | null;
     mode?: "links" | "download";
     backend?: "auto" | "slopbot" | "puppet";
+    /** Set by a re-run of the command after the near-limit warning. */
+    confirm?: boolean;
+}
+
+export interface ScrapeBytes {
+    id: string;
+    channelId: string;
+    state: string;
+    createdAt: string;
+    bytes: number;
+}
+
+export interface MediaStorageStatus {
+    usedBytes: number;
+    limitBytes: number;
+    warnPercent: number;
+    overrunPercent: number;
+    nearLimit: boolean;
+    byKind: Record<MediaKind, number>;
+    scrapes: ScrapeBytes[];
+}
+
+/** Thrown when a scrape starts near the storage limit without confirmation. */
+export class MediaLimitError extends Error {
+    readonly code = "MEDIA_LIMIT";
+
+    constructor(
+        message: string,
+        readonly usedBytes: number,
+        readonly limitBytes: number
+    ) {
+        super(message);
+    }
+}
+
+function formatBytes(bytes: number): string {
+    const gib = bytes / 1024 ** 3;
+    return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
 }
 
 export interface MediaItemResult {
@@ -104,7 +148,92 @@ export class MediaService {
         });
     }
 
+    /** Combined size of every scrape directory, with the limit settings applied. */
+    storageStatus(): MediaStorageStatus {
+        const prefs = this.deps.getMediaPrefs();
+        const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
+
+        const scrapes = this.engine.list().map(meta => {
+            const dir = scrapeDir(this.rootDir, meta.id);
+            const bytes = dirSizeBytes(dir);
+
+            try {
+                const store = new ScrapeStore(dir);
+                try {
+                    const sums = store.sumBytesByKind();
+                    for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
+                } finally {
+                    store.close();
+                }
+            } catch {
+                // an unreadable index counts as zero bytes
+            }
+
+            return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
+        });
+
+        const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
+
+        return {
+            usedBytes,
+            limitBytes: prefs.limitBytes,
+            warnPercent: prefs.warnPercent,
+            overrunPercent: prefs.overrunPercent,
+            nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
+            byKind,
+            scrapes
+        };
+    }
+
+    /** Mark the oldest completed scrapes when the usage is at or over the limit. */
+    refreshRetention(): MediaStorageStatus {
+        const status = this.storageStatus();
+        const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
+        for (const id of plan.mark) this.engine.setState(id, "pending-delete");
+        return this.storageStatus();
+    }
+
+    private applyRetention(): void {
+        const status = this.storageStatus();
+        const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
+        for (const id of plan.mark) this.engine.setState(id, "pending-delete");
+        for (const id of plan.delete) this.deleteScrape(id);
+    }
+
+    deleteScrape(id: string): boolean {
+        try {
+            rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
+            this.engine.forget(id);
+            this.deps.log?.(`deleted scrape ${id}`);
+            return true;
+        } catch (error) {
+            this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
+            return false;
+        }
+    }
+
+    /** Delete the given scrapes, or every scrape when no IDs are given. */
+    cleanMedia(ids?: string[]): number {
+        const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
+        let deleted = 0;
+        for (const id of targets) {
+            if (this.deleteScrape(id)) deleted++;
+        }
+        return deleted;
+    }
+
     async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
+        this.applyRetention();
+
+        const status = this.storageStatus();
+        if (status.nearLimit && !input.confirm) {
+            throw new MediaLimitError(
+                `media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
+                status.usedBytes,
+                status.limitBytes
+            );
+        }
+
         const mode = input.mode ?? "links";
         let backend: ScrapeBackend;