Add media storage limits, retention, and the near-limit flow

poc/src/control-bridge.ts

@@ -43,6 +43,7 @@ let connecting = false;
 const media = new MediaService({
     getCore: () => core,
     isCoreConnected: () => coreConnected,
+    getMediaPrefs: () => preferences.get().media,
     log: line => console.log(`[media] ${line}`)
 });
 

poc/src/media.ts

@@ -5,7 +5,11 @@
  * (later) MCP tools call.
  */
 
+import { rmSync } from "node:fs";
 import {
+    dirSizeBytes,
+    isNearLimit,
+    planRetention,
     ScrapeEngine,
     ScrapeStore,
     scrapeDir,
@@ -25,12 +29,14 @@ import {
     type SlopbotStatus
 } from "../../media-scrape/slopbot/index.js";
 import type { CoreClient } from "./core-client.js";
+import type { MediaPreferences } from "./preferences.js";
 
 type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
 
 export interface MediaServiceDeps {
     getCore: () => CoreClient | undefined;
     isCoreConnected: () => boolean;
+    getMediaPrefs: () => MediaPreferences;
     log?: (line: string) => void;
 }
 
@@ -39,6 +45,44 @@ export interface StartScrapeInput {
     guildId?: string | null;
     mode?: "links" | "download";
     backend?: "auto" | "slopbot" | "puppet";
+    /** Set by a re-run of the command after the near-limit warning. */
+    confirm?: boolean;
+}
+
+export interface ScrapeBytes {
+    id: string;
+    channelId: string;
+    state: string;
+    createdAt: string;
+    bytes: number;
+}
+
+export interface MediaStorageStatus {
+    usedBytes: number;
+    limitBytes: number;
+    warnPercent: number;
+    overrunPercent: number;
+    nearLimit: boolean;
+    byKind: Record<MediaKind, number>;
+    scrapes: ScrapeBytes[];
+}
+
+/** Thrown when a scrape starts near the storage limit without confirmation. */
+export class MediaLimitError extends Error {
+    readonly code = "MEDIA_LIMIT";
+
+    constructor(
+        message: string,
+        readonly usedBytes: number,
+        readonly limitBytes: number
+    ) {
+        super(message);
+    }
+}
+
+function formatBytes(bytes: number): string {
+    const gib = bytes / 1024 ** 3;
+    return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
 }
 
 export interface MediaItemResult {
@@ -104,7 +148,92 @@ export class MediaService {
         });
     }
 
+    /** Combined size of every scrape directory, with the limit settings applied. */
+    storageStatus(): MediaStorageStatus {
+        const prefs = this.deps.getMediaPrefs();
+        const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
+
+        const scrapes = this.engine.list().map(meta => {
+            const dir = scrapeDir(this.rootDir, meta.id);
+            const bytes = dirSizeBytes(dir);
+
+            try {
+                const store = new ScrapeStore(dir);
+                try {
+                    const sums = store.sumBytesByKind();
+                    for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
+                } finally {
+                    store.close();
+                }
+            } catch {
+                // an unreadable index counts as zero bytes
+            }
+
+            return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
+        });
+
+        const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
+
+        return {
+            usedBytes,
+            limitBytes: prefs.limitBytes,
+            warnPercent: prefs.warnPercent,
+            overrunPercent: prefs.overrunPercent,
+            nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
+            byKind,
+            scrapes
+        };
+    }
+
+    /** Mark the oldest completed scrapes when the usage is at or over the limit. */
+    refreshRetention(): MediaStorageStatus {
+        const status = this.storageStatus();
+        const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
+        for (const id of plan.mark) this.engine.setState(id, "pending-delete");
+        return this.storageStatus();
+    }
+
+    private applyRetention(): void {
+        const status = this.storageStatus();
+        const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
+        for (const id of plan.mark) this.engine.setState(id, "pending-delete");
+        for (const id of plan.delete) this.deleteScrape(id);
+    }
+
+    deleteScrape(id: string): boolean {
+        try {
+            rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
+            this.engine.forget(id);
+            this.deps.log?.(`deleted scrape ${id}`);
+            return true;
+        } catch (error) {
+            this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
+            return false;
+        }
+    }
+
+    /** Delete the given scrapes, or every scrape when no IDs are given. */
+    cleanMedia(ids?: string[]): number {
+        const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
+        let deleted = 0;
+        for (const id of targets) {
+            if (this.deleteScrape(id)) deleted++;
+        }
+        return deleted;
+    }
+
     async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
+        this.applyRetention();
+
+        const status = this.storageStatus();
+        if (status.nearLimit && !input.confirm) {
+            throw new MediaLimitError(
+                `media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
+                status.usedBytes,
+                status.limitBytes
+            );
+        }
+
         const mode = input.mode ?? "links";
         let backend: ScrapeBackend;
 

poc/src/preferences.ts

@@ -76,12 +76,22 @@ export interface AudioPreferences {
     duck: boolean;
 }
 
+export interface MediaPreferences {
+    /** Combined size limit for the media DB root: databases, downloads, previews. */
+    limitBytes: number;
+    /** Warn threshold in percent, 1-100. */
+    warnPercent: number;
+    /** No-delete overrun allowance in percent, 0-100. */
+    overrunPercent: number;
+}
+
 export interface WebPreferences {
     /** null means every guild is visible. An empty array hides all. */
     visibleGuilds: string[] | null;
     tts: TtsPreferences;
     audio: AudioPreferences;
     favourites: FavouritesPreferences;
+    media: MediaPreferences;
 }
 
 const DEFAULTS: WebPreferences = {
@@ -101,9 +111,35 @@ const DEFAULTS: WebPreferences = {
         wordGapMs: null
     },
     audio: { duck: true },
-    favourites: { users: [], channels: [] }
+    favourites: { users: [], channels: [] },
+    media: {
+        limitBytes: 30 * 1024 * 1024 * 1024,
+        warnPercent: 90,
+        overrunPercent: 5
+    }
 };
 
+const MEDIA_LIMIT_MIN = 256 * 1024 * 1024;
+const MEDIA_LIMIT_MAX = 10 * 1024 * 1024 * 1024 * 1024;
+
+function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences {
+    const limit = Number(raw?.limitBytes);
+    const warn = Number(raw?.warnPercent);
+    const overrun = Number(raw?.overrunPercent);
+
+    return {
+        limitBytes:
+            Number.isFinite(limit) && limit >= MEDIA_LIMIT_MIN && limit <= MEDIA_LIMIT_MAX
+                ? Math.round(limit)
+                : fallback.limitBytes,
+        warnPercent: Number.isFinite(warn) && warn >= 1 && warn <= 100 ? Math.round(warn) : fallback.warnPercent,
+        overrunPercent:
+            Number.isFinite(overrun) && overrun >= 0 && overrun <= 100
+                ? Math.round(overrun)
+                : fallback.overrunPercent
+    };
+}
+
 export interface PreferencesOptions {
     /** The old state file that may still hold preferences. */
     legacyFile?: string;
@@ -199,6 +235,10 @@ export class PreferencesStore {
             };
         }
 
+        if (patch.media !== undefined) {
+            this.preferences.media = normalizeMedia(patch.media, this.preferences.media);
+        }
+
         if (patch.favourites !== undefined) {
             this.preferences.favourites = {
                 users: Array.isArray(patch.favourites?.users)
@@ -255,6 +295,7 @@ export class PreferencesStore {
             audio: {
                 duck: raw?.audio?.duck !== false
             },
+            media: normalizeMedia(raw?.media, DEFAULTS.media),
             favourites: {
                 users: Array.isArray(raw?.favourites?.users)
                     ? [...new Set<string>(raw.favourites.users.map((id: unknown) => String(id)))].slice(0, 100)

poc/src/puppet.ts

@@ -123,6 +123,7 @@ export interface PuppetMedia {
         channelId: string;
         guildId: string | null;
         mode: "links" | "download";
+        confirm?: boolean;
     }): Promise<{ id: string }>;
     stopScrape(id: string): void;
     listScrapes(): PuppetScrape[];
@@ -174,6 +175,8 @@ export class Puppet {
     private readonly lastFetchAt = new Map<string, number>();
     private readonly pending = new Map<string, { entryId: string; expiresAt: number }>();
     private readonly polls = new Map<string, PollState>();
+    /** Near-limit scrape warnings waiting for the same user to re-run the command. */
+    private readonly scrapeConfirm = new Map<string, number>();
 
     private voiceChannelId?: string | null;
     private ownerVoiceChannelId?: string | null;
@@ -753,11 +756,26 @@ export class Puppet {
             return this.respond(authorId, channelId, "scrape", "Download mode needs the admin tier.");
         }
 
+        const key = `${channelId}:${authorId}`;
+        const confirmUntil = this.scrapeConfirm.get(key);
+        const confirm = confirmUntil !== undefined && Date.now() < confirmUntil;
+        if (confirm) this.scrapeConfirm.delete(key);
+
         try {
-            const scrape = await this.media.startScrape({ channelId, guildId: guildId ?? null, mode });
+            const scrape = await this.media.startScrape({ channelId, guildId: guildId ?? null, mode, confirm });
             this.actionLog.append({ type: "media", action: "scrape", channelId, mode, scrapeId: scrape.id });
             return this.respond(authorId, channelId, "scrape", `Media scrape started (${mode}): ${scrape.id}.`);
         } catch (error) {
+            if ((error as { code?: string }).code === "MEDIA_LIMIT" && !confirm) {
+                this.scrapeConfirm.set(key, Date.now() + 120_000);
+                return this.respond(
+                    authorId,
+                    channelId,
+                    "scrape",
+                    `${(error as Error).message}. Run the command again within 2 minutes to accept.`
+                );
+            }
+
             this.actionLog.append({ type: "error", scope: "media", channelId, message: String(error) });
             return this.respond(
                 authorId,

poc/src/webui.ts

@@ -20,7 +20,7 @@ import type { AuthService, AuthStatus } from "./auth-service.js";
 import type { AudioService } from "./audio-service.js";
 import { CoreError, type CoreClient } from "./core-client.js";
 import type { CoreProcess } from "./core-process.js";
-import type { MediaService } from "./media.js";
+import { MediaLimitError, type MediaService } from "./media.js";
 import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js";
 import type { PreferencesStore, WebPreferences } from "./preferences.js";
 import type { Puppet } from "./puppet.js";
@@ -206,6 +206,16 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
                     return;
                 }
 
+                if (error instanceof MediaLimitError) {
+                    res.status(409).json({
+                        ok: false,
+                        code: error.code,
+                        error: error.message,
+                        result: { usedBytes: error.usedBytes, limitBytes: error.limitBytes }
+                    });
+                    return;
+                }
+
                 const status = error instanceof HttpError ? error.status : 500;
                 res.status(status).json({ ok: false, error: String(error?.message ?? error) });
             });
@@ -403,7 +413,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
     app.post(
         "/api/preferences",
         wrap(async (req, res) => {
-            const { visibleGuilds, tts, audio, favourites } = req.body ?? {};
+            const { visibleGuilds, tts, audio, favourites, media: mediaPrefs } = req.body ?? {};
 
             if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
                 throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
@@ -438,6 +448,21 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
                 };
             }
 
+            if (mediaPrefs !== undefined) {
+                const current = deps.preferences.get().media;
+                patch.media = {
+                    limitBytes: Number.isFinite(Number(mediaPrefs?.limitBytes))
+                        ? Number(mediaPrefs.limitBytes)
+                        : current.limitBytes,
+                    warnPercent: Number.isFinite(Number(mediaPrefs?.warnPercent))
+                        ? Number(mediaPrefs.warnPercent)
+                        : current.warnPercent,
+                    overrunPercent: Number.isFinite(Number(mediaPrefs?.overrunPercent))
+                        ? Number(mediaPrefs.overrunPercent)
+                        : current.overrunPercent
+                };
+            }
+
             if (audio !== undefined) {
                 patch.audio = { duck: audio?.duck !== false };
                 deps.audio.setDuckEnabled(patch.audio.duck);
@@ -729,6 +754,23 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         })
     );
 
+    app.get(
+        "/api/media/storage",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: deps.media.refreshRetention() });
+        })
+    );
+
+    app.post(
+        "/api/media/clean",
+        wrap(async (req, res) => {
+            const ids = Array.isArray(req.body?.ids) ? req.body.ids.map(String) : undefined;
+            const deleted = deps.media.cleanMedia(ids);
+            deps.getPuppet()?.appendAction({ type: "media", action: "clean", deleted });
+            res.json({ ok: true, result: { deleted } });
+        })
+    );
+
     app.get(
         "/api/media",
         wrap(async (req, res) => {
@@ -750,8 +792,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
 
             const mode = req.body?.mode === "download" ? "download" : "links";
             const guildId = req.body?.guildId ? String(req.body.guildId) : null;
+            const confirm = req.body?.confirm === true;
 
-            const scrape = await deps.media.startScrape({ channelId, guildId, mode });
+            const scrape = await deps.media.startScrape({ channelId, guildId, mode, confirm });
             deps.getPuppet()?.appendAction({
                 type: "media",
                 action: "scrape",