Add media storage limits, retention, and the near-limit flow

media-scrape/src/engine.ts

@@ -18,7 +18,8 @@ import {
     type BackendName,
     type CoreMessage,
     type ScrapeMeta,
-    type ScrapeMode
+    type ScrapeMode,
+    type ScrapeState
 } from "./types.js";
 
 export interface FetchPage {
@@ -146,6 +147,24 @@ export class ScrapeEngine {
         this.stopRequested.add(id);
     }
 
+    /** Persist a state change made outside a job, such as retention marking. */
+    setState(id: string, state: ScrapeState): ScrapeMeta | null {
+        const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
+        if (!meta) return null;
+
+        meta.state = state;
+        meta.updatedAt = new Date(this.now()).toISOString();
+        writeScrapeMeta(join(this.options.rootDir, id), meta);
+        this.active.set(id, meta);
+        return { ...meta };
+    }
+
+    /** Drop a deleted scrape from the in-memory list. */
+    forget(id: string): void {
+        this.active.delete(id);
+        this.stopRequested.delete(id);
+    }
+
     private async runJob(job: QueuedJob): Promise<void> {
         const { meta, dir, backend } = job;
         const store = new ScrapeStore(dir);