Add media storage limits, retention, and the near-limit flow

media-scrape/src/engine.ts

@@ -18,7 +18,8 @@ import {
     type BackendName,
     type CoreMessage,
     type ScrapeMeta,
-    type ScrapeMode
+    type ScrapeMode,
+    type ScrapeState
 } from "./types.js";
 
 export interface FetchPage {
@@ -146,6 +147,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Persist a state change made outside a job, such as retention marking. */
+    setState(id: string, state: ScrapeState): ScrapeMeta | null {
+        const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
+        if (!meta) return null;
+
+        meta.state = state;
+        meta.updatedAt = new Date(this.now()).toISOString();
+        writeScrapeMeta(join(this.options.rootDir, id), meta);
+        this.active.set(id, meta);
+        return { ...meta };
+    }
+
+    /** Drop a deleted scrape from the in-memory list. */
+    forget(id: string): void {
+        this.active.delete(id);
+        this.stopRequested.delete(id);
+    }
+
     private async runJob(job: QueuedJob): Promise<void> {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);

media-scrape/src/index.ts

@@ -4,6 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export {
     ScrapeEngine,
     type FetchPage,

media-scrape/src/retention.ts

@@ -0,0 +1,65 @@
+/*
+ * Pure retention planning for the media archive. Rules: docs/media.md.
+ */
+
+export interface RetentionScrape {
+    id: string;
+    state: string;
+    createdAt: string;
+    bytes: number;
+}
+
+export interface RetentionPlan {
+    /** Completed scrapes to mark pending-delete so the usage can fit the limit. */
+    mark: string[];
+    /** Marked scrapes to delete now: the usage is past the overrun allowance. */
+    delete: string[];
+    totalBytes: number;
+}
+
+/** States that may be marked for deletion. A running scrape is never touched. */
+const MARKABLE = new Set(["done", "failed", "paused"]);
+
+export function isNearLimit(usedBytes: number, limitBytes: number, warnPercent: number): boolean {
+    return usedBytes >= limitBytes * (warnPercent / 100);
+}
+
+export function planRetention(
+    scrapes: RetentionScrape[],
+    limitBytes: number,
+    overrunPercent: number
+): RetentionPlan {
+    const totalBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
+    const oldest = [...scrapes].sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
+    const overrunLimit = limitBytes * (1 + overrunPercent / 100);
+
+    // Past the allowance: delete marked scrapes, oldest first, until the usage fits.
+    if (totalBytes > overrunLimit) {
+        const remove: string[] = [];
+        let planned = totalBytes;
+
+        for (const scrape of oldest) {
+            if (planned <= limitBytes) break;
+            if (scrape.state !== "pending-delete") continue;
+            remove.push(scrape.id);
+            planned -= scrape.bytes;
+        }
+
+        return { mark: [], delete: remove, totalBytes };
+    }
+
+    if (totalBytes <= limitBytes) return { mark: [], delete: [], totalBytes };
+
+    // Between the limit and the allowance: mark oldest completed scrapes, no deletion.
+    const mark: string[] = [];
+    let planned = totalBytes;
+
+    for (const scrape of oldest) {
+        if (planned <= limitBytes) break;
+        if (!MARKABLE.has(scrape.state)) continue;
+        mark.push(scrape.id);
+        planned -= scrape.bytes;
+    }
+
+    return { mark, delete: [], totalBytes };
+}

media-scrape/src/store.ts

@@ -216,6 +216,19 @@ export class ScrapeStore {
         return added;
     }
 
+    /** Sum of the recorded source sizes per kind. Used by the storage bar. */
+    sumBytesByKind(): Record<MediaKind, number> {
+        const rows = this.db
+            .prepare("SELECT kind, COALESCE(SUM(size), 0) AS bytes FROM media GROUP BY kind")
+            .all() as unknown as Array<{ kind: string; bytes: number }>;
+
+        const result = emptyMediaCounts();
+        for (const row of rows) {
+            if (row.kind in result) result[row.kind as MediaKind] = row.bytes;
+        }
+        return result;
+    }
+
     counts(): Record<MediaKind, number> {
         const rows = this.db
             .prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")

media-scrape/test/run.ts

@@ -10,6 +10,7 @@ import { join } from "node:path";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { isNearLimit, planRetention } from "../src/retention.js";
 import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
 import type { CoreAttachment, CoreMessage } from "../src/types.js";
 
@@ -173,7 +174,9 @@ void (async () => {
         const backend = new FakeBackend([
             {
                 messages: [
-                    message("100", { attachments: [attachment({ filename: "one.png", contentType: "image/png" })] }),
+                    message("100", {
+                        attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })]
+                    }),
                     message("101")
                 ]
             },
@@ -202,6 +205,7 @@ void (async () => {
         assert.equal(stored.counts().image, 1);
         assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
         assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
+        assert.equal(stored.sumBytesByKind().image, 10);
         stored.close();
 
         assert.equal(listScrapes(root).length, 1);
@@ -272,6 +276,42 @@ void (async () => {
         assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
     });
 
+    await test("retention marks the oldest completed scrapes up to the limit", () => {
+        const scrapes = [
+            { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
+            { id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
+            { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
+        ];
+
+        const plan = planRetention(scrapes, 1000, 100);
+        assert.deepEqual(plan.mark, ["old", "new"]);
+        assert.deepEqual(plan.delete, []);
+    });
+
+    await test("retention deletes marked scrapes past the overrun allowance", () => {
+        const scrapes = [
+            { id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
+            { id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
+            { id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
+        ];
+
+        const plan = planRetention(scrapes, 1000, 5);
+        assert.deepEqual(plan.delete, ["old", "new"]);
+        assert.deepEqual(plan.mark, []);
+    });
+
+    await test("retention leaves usage under the limit alone", () => {
+        const plan = planRetention(
+            [{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }],
+            1000,
+            5
+        );
+
+        assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 });
+        assert.equal(isNearLimit(900, 1000, 90), true);
+        assert.equal(isNearLimit(899, 1000, 90), false);
+    });
+
     rmSync(root, { recursive: true, force: true });
 
     if (failures > 0) {