Add media storage limits, retention, and the near-limit flow

media-scrape/src/engine.ts

@@ -18,7 +18,8 @@ import {
     type BackendName,
     type CoreMessage,
     type ScrapeMeta,
-    type ScrapeMode
+    type ScrapeMode,
+    type ScrapeState
 } from "./types.js";
 
 export interface FetchPage {
@@ -146,6 +147,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Persist a state change made outside a job, such as retention marking. */
+    setState(id: string, state: ScrapeState): ScrapeMeta | null {
+        const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
+        if (!meta) return null;
+
+        meta.state = state;
+        meta.updatedAt = new Date(this.now()).toISOString();
+        writeScrapeMeta(join(this.options.rootDir, id), meta);
+        this.active.set(id, meta);
+        return { ...meta };
+    }
+
+    /** Drop a deleted scrape from the in-memory list. */
+    forget(id: string): void {
+        this.active.delete(id);
+        this.stopRequested.delete(id);
+    }
+
     private async runJob(job: QueuedJob): Promise<void> {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);

media-scrape/src/index.ts

@@ -4,6 +4,7 @@
  */
 
 export { extractMedia, type ExtractedMedia } from "./extract.js";
+export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
 export {
     ScrapeEngine,
     type FetchPage,

media-scrape/src/retention.ts

@@ -0,0 +1,65 @@
+/*
+ * Pure retention planning for the media archive. Rules: docs/media.md.
+ */
+
+export interface RetentionScrape {
+    id: string;
+    state: string;
+    createdAt: string;
+    bytes: number;
+}
+
+export interface RetentionPlan {
+    /** Completed scrapes to mark pending-delete so the usage can fit the limit. */
+    mark: string[];
+    /** Marked scrapes to delete now: the usage is past the overrun allowance. */
+    delete: string[];
+    totalBytes: number;
+}
+
+/** States that may be marked for deletion. A running scrape is never touched. */
+const MARKABLE = new Set(["done", "failed", "paused"]);
+
+export function isNearLimit(usedBytes: number, limitBytes: number, warnPercent: number): boolean {
+    return usedBytes >= limitBytes * (warnPercent / 100);
+}
+
+export function planRetention(
+    scrapes: RetentionScrape[],
+    limitBytes: number,
+    overrunPercent: number
+): RetentionPlan {
+    const totalBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
+    const oldest = [...scrapes].sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
+    const overrunLimit = limitBytes * (1 + overrunPercent / 100);
+
+    // Past the allowance: delete marked scrapes, oldest first, until the usage fits.
+    if (totalBytes > overrunLimit) {
+        const remove: string[] = [];
+        let planned = totalBytes;
+
+        for (const scrape of oldest) {
+            if (planned <= limitBytes) break;
+            if (scrape.state !== "pending-delete") continue;
+            remove.push(scrape.id);
+            planned -= scrape.bytes;
+        }
+
+        return { mark: [], delete: remove, totalBytes };
+    }
+
+    if (totalBytes <= limitBytes) return { mark: [], delete: [], totalBytes };
+
+    // Between the limit and the allowance: mark oldest completed scrapes, no deletion.
+    const mark: string[] = [];
+    let planned = totalBytes;
+
+    for (const scrape of oldest) {
+        if (planned <= limitBytes) break;
+        if (!MARKABLE.has(scrape.state)) continue;
+        mark.push(scrape.id);
+        planned -= scrape.bytes;
+    }
+
+    return { mark, delete: [], totalBytes };
+}

media-scrape/src/store.ts

@@ -216,6 +216,19 @@ export class ScrapeStore {
         return added;
     }
 
+    /** Sum of the recorded source sizes per kind. Used by the storage bar. */
+    sumBytesByKind(): Record<MediaKind, number> {
+        const rows = this.db
+            .prepare("SELECT kind, COALESCE(SUM(size), 0) AS bytes FROM media GROUP BY kind")
+            .all() as unknown as Array<{ kind: string; bytes: number }>;
+
+        const result = emptyMediaCounts();
+        for (const row of rows) {
+            if (row.kind in result) result[row.kind as MediaKind] = row.bytes;
+        }
+        return result;
+    }
+
     counts(): Record<MediaKind, number> {
         const rows = this.db
             .prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")