diff --git a/CURRENT.md b/CURRENT.md index a254e58..56e5068 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -32,10 +32,12 @@ mobile draft at 390px. - [x] C6 Media and Bot tabs in the Web UI - [x] C7a storage limits, retention, warning/confirm flow, storage and clean API - [x] C7b Settings group: limit field with info hover, usage bar, clean modal - - [ ] C8 MCP service hooks - - [~] C9 live verification: slopbot login works (Slopbot#5775), but the bot is in 0 guilds, so - the test channel fetch fails with Missing Access. Waiting on the owner to invite the bot to - guild `177931418704281600`. Live check: `pnpm test:slopbot:live`. + - [x] C8 MCP service hooks (`scrape_media`, scrape reads, storage, clean; reads in `run_commands`) + - [~] C9 live verification. Slopbot path verified live on 2026-09-16: login Slopbot#5775, guild + `177931418704281600`, test channel `1549681689222053988`. Scrape `1549681689222053988- + 20260916075150`: state done, 1 page, 17 messages, 4 images + 13 videos, 0 errors, ~20 MB of + indexed source sizes. Still owed: the puppet `message.fetch` field check, a `download` mode + run, a real 429 observation, and pacing tuning. - Flextalk voice profile, end to end. The profile window is a movable panel that recenters on each open; it carries the character sliders with engine defaults and +/-2 snapping, five named presets, radio-kit effects (high-pass 40-400 Hz, echo 0-100, word mode with a gap in ms), and diff --git a/docs/daemon.md b/docs/daemon.md index e0a1d04..8fffe3c 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -78,6 +78,14 @@ The daemon exposes MCP over Streamable HTTP. The default address is `http://127. | `list_dms` | List cached DM and group DM channels with recipients. | | `get_message_log` | Read the trigger log. | | `get_action_log` | Read the persistent action log. | +| `scrape_media` | Start a media scrape of a channel (`channelId`, `guildId?`, `mode?`, `confirm?`). | +| `list_media_scrapes` | Media scrapes with state, pages, messages, and counts. | +| `get_media_scrape` | One scrape by ID. | +| `stop_media_scrape` | Stop a scrape; it pauses after the current page. | +| `get_media_storage` | Usage against the limit, per-kind sizes, and the scrape list. | +| `query_media` | Media items across scrapes (`kind?`, `scrapeId?`, `limit?`, `offset?`). | +| `get_media_item` | One media item by `scrapeId` and `itemId`. | +| `delete_media_scrape` | Delete a scrape and its files. Irreversible. | | `run_commands` | Run several read-only tools in one call, in order. | | `get_context` | Assemble conversation context for a channel. Fetches new messages by default. | | `get_puppet_config` | Read the response policy and context settings. | diff --git a/docs/media-mcp.md b/docs/media-mcp.md index f98960a..cc11340 100644 --- a/docs/media-mcp.md +++ b/docs/media-mcp.md @@ -4,7 +4,9 @@ The media archive is designed for agent access. The daemon exposes one service, are thin wrappers around it. The same service backs the Web UI routes, so the two clients cannot drift. -Status: hooks designed, implementation at checkpoint C8 in [CURRENT.md](../CURRENT.md). +Status: implemented (C8). The tools are registered in `poc/src/control-bridge.ts`; read tools are +also in `run_commands`. Advanced `query_media` filters (author, date, text) are planned; the index +has the columns. ## Service interface @@ -21,8 +23,8 @@ interface MediaService { } ``` -`MediaQuery`: `kind`, `scrapeId`, `channelId`, `guildId`, `authorId`, `after`, `before`, `text`, -`limit`, `cursor`. +`MediaQuery` (implemented): `kind`, `scrapeId`, `limit`, `offset`. Planned: `channelId`, +`guildId`, `authorId`, `after`, `before`, `text`. `StorageStatus`: `usedBytes`, `limitBytes`, `warnPercent`, `overrunPercent`, `byKind`, `scrapes: [{ id, bytes, state, createdAt }]`. diff --git a/docs/media.md b/docs/media.md index f567e73..9b8c937 100644 --- a/docs/media.md +++ b/docs/media.md @@ -1,8 +1,10 @@ # Media archive Status: design approved 2026-09-16. The engine, the slopbot backend, the `scrape media` command, -the `/api/media/*` routes, storage limits with retention, and the Settings group are implemented -(C3-C7b). The Media and Bot tabs are C6. Checkpoints live in [CURRENT.md](../CURRENT.md). +the `/api/media/*` routes, storage limits with retention, the Settings group, and the MCP tools are +implemented (C3-C8). The Media and Bot tabs are C6. The slopbot path is live-verified: the test +channel produced 17 messages and 4 images + 13 videos with no errors. The puppet path and a +`download` run are still owed (C9). Checkpoints live in [CURRENT.md](../CURRENT.md). The media archive saves the media history of a channel. A trusted user runs `scrape media` in the channel. The daemon walks the history through a scrape backend and indexes every image, GIF, diff --git a/media-scrape/slopbot/client.ts b/media-scrape/slopbot/client.ts index 5b356ea..af2a190 100644 --- a/media-scrape/slopbot/client.ts +++ b/media-scrape/slopbot/client.ts @@ -3,7 +3,7 @@ * Setup and security: docs/slopbot.md. */ -import { Client, GatewayIntentBits } from "discord.js"; +import { Client, Events, GatewayIntentBits } from "discord.js"; import type { SlopbotConfig } from "./config.js"; import type { SlopbotFetcher } from "./backend.js"; @@ -23,7 +23,16 @@ export async function startSlopbot(config: SlopbotConfig): Promise { } const client = new Client({ intents: [GatewayIntentBits.Guilds] }); + + // login() resolves before the READY dispatch, so wait for it: the guild cache + // and client.user are only populated then. + const ready = new Promise((resolve, reject) => { + client.once(Events.ClientReady, () => resolve()); + client.once(Events.Error, reject); + }); + await client.login(config.token); + await ready; return client; } diff --git a/media-scrape/src/store.ts b/media-scrape/src/store.ts index df88661..8c53d52 100644 --- a/media-scrape/src/store.ts +++ b/media-scrape/src/store.ts @@ -135,6 +135,8 @@ CREATE TABLE IF NOT EXISTS media ( preview_url TEXT, file_hash TEXT, preview_hash TEXT, + file_bytes INTEGER, + preview_bytes INTEGER, created_at TEXT NOT NULL ); @@ -155,6 +157,16 @@ export class ScrapeStore { constructor(readonly dir: string) { this.db = new DatabaseSync(join(dir, "index.db")); this.db.exec(SCHEMA); + this.migrate(); + } + + /** Add columns introduced after a scrape was created. */ + private migrate(): void { + const columns = this.db.prepare("PRAGMA table_info(media)").all() as unknown as Array<{ name: string }>; + const names = new Set(columns.map(column => column.name)); + + if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER"); + if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER"); } close(): void { @@ -216,10 +228,12 @@ export class ScrapeStore { return added; } - /** Sum of the recorded source sizes per kind. Used by the storage bar. */ - sumBytesByKind(): Record { + /** Bytes actually on disk per kind (downloaded files and cached previews). */ + storedBytesByKind(): Record { const rows = this.db - .prepare("SELECT kind, COALESCE(SUM(size), 0) AS bytes FROM media GROUP BY kind") + .prepare( + "SELECT kind, COALESCE(SUM(COALESCE(file_bytes, 0) + COALESCE(preview_bytes, 0)), 0) AS bytes FROM media GROUP BY kind" + ) .all() as unknown as Array<{ kind: string; bytes: number }>; const result = emptyMediaCounts(); @@ -241,6 +255,13 @@ export class ScrapeStore { return result; } + getMedia(id: number): MediaRow | null { + const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as + | MediaRow + | undefined; + return row ?? null; + } + queryMedia(options: { kind?: MediaKind; limit?: number; offset?: number } = {}): MediaRow[] { const limit = Math.max(1, Math.min(options.limit ?? 50, 500)); const offset = Math.max(0, options.offset ?? 0); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index c552b99..063549a 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -205,7 +205,7 @@ void (async () => { assert.equal(stored.counts().image, 1); assert.equal(stored.queryMedia({ kind: "image" }).length, 1); assert.equal(stored.queryMedia({ kind: "video" }).length, 0); - assert.equal(stored.sumBytesByKind().image, 10); + assert.equal(stored.storedBytesByKind().image, 0); stored.close(); assert.equal(listScrapes(root).length, 1); diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index e715af0..907a94e 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -317,7 +317,16 @@ const READ_ONLY_TOOLS: Record Promise | unknown> get_action_log: args => requirePuppet().readActionLog(args?.limit ?? 100), get_puppet_config: () => requirePuppet().getConfig(), whitelist_list: () => ({ ownerId: requirePuppet().owner, whitelist: requirePuppet().whitelist }), - read_messages: args => paced(() => requireCore().readMessages(args?.channelId, args?.limit ?? 20)) + read_messages: args => paced(() => requireCore().readMessages(args?.channelId, args?.limit ?? 20)), + list_media_scrapes: () => media.listScrapes(), + get_media_storage: () => media.refreshRetention(), + query_media: args => + media.queryMedia({ + kind: args?.kind, + scrapeId: args?.scrapeId, + limit: args?.limit, + offset: args?.offset + }) }; function buildServer(): McpServer { @@ -749,6 +758,97 @@ function buildServer(): McpServer { async ({ userIds }) => jsonText({ whitelist: requirePuppet().removeFromWhitelist(userIds) }) ); + server.registerTool( + "scrape_media", + { + description: + "Start a media scrape of a channel. Uses slopbot for guild channels it is in, otherwise the client. Near the storage limit it fails with MEDIA_LIMIT unless confirm is true.", + inputSchema: { + channelId: z.string().describe("Channel ID to scrape"), + guildId: z.string().optional().describe("Guild ID; slopbot needs it to match"), + mode: z + .enum(["links", "download"]) + .optional() + .describe("links indexes URLs (default); download fetches attachment bytes"), + confirm: z.boolean().optional().describe("Accept the near-limit warning") + } + }, + async ({ channelId, guildId, mode, confirm }) => + jsonText(await media.startScrape({ channelId, guildId: guildId ?? null, mode, confirm })) + ); + + server.registerTool( + "list_media_scrapes", + { description: "List media scrapes with state, pages, messages, and media counts." }, + async () => jsonText(media.listScrapes()) + ); + + server.registerTool( + "get_media_scrape", + { + description: "Get one media scrape by ID.", + inputSchema: { scrapeId: z.string().describe("Scrape ID") } + }, + async ({ scrapeId }) => jsonText(media.getScrape(scrapeId)) + ); + + server.registerTool( + "stop_media_scrape", + { + description: "Stop a media scrape. It pauses after the current page.", + inputSchema: { scrapeId: z.string().describe("Scrape ID") } + }, + async ({ scrapeId }) => { + media.stopScrape(scrapeId); + return jsonText({ stopped: scrapeId }); + } + ); + + server.registerTool( + "get_media_storage", + { + description: + "Media storage usage against the limit, per-kind sizes, and the scrape list. Marks old scrapes when over the limit." + }, + async () => jsonText(media.refreshRetention()) + ); + + server.registerTool( + "query_media", + { + description: "Query media items across scrapes, newest scrape first.", + inputSchema: { + kind: z.enum(["image", "gif", "video", "audio"]).optional(), + scrapeId: z.string().optional().describe("Restrict to one scrape"), + limit: z.number().int().min(1).max(500).optional(), + offset: z.number().int().min(0).optional() + } + }, + async ({ kind, scrapeId, limit, offset }) => + jsonText(media.queryMedia({ kind, scrapeId, limit, offset })) + ); + + server.registerTool( + "get_media_item", + { + description: "Get one media item by scrape ID and item ID.", + inputSchema: { + scrapeId: z.string().describe("Scrape ID"), + itemId: z.number().int().min(1).describe("Media row ID inside the scrape") + } + }, + async ({ scrapeId, itemId }) => jsonText(media.getMediaItem(scrapeId, itemId)) + ); + + server.registerTool( + "delete_media_scrape", + { + description: "Delete a scrape and its files. Irreversible.", + inputSchema: { scrapeId: z.string().describe("Scrape ID") } + }, + async ({ scrapeId }) => jsonText({ deleted: media.cleanMedia([scrapeId]) === 1 }) + ); + return server; } diff --git a/poc/src/media.ts b/poc/src/media.ts index 7e6d9b1..0e1a479 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -15,6 +15,7 @@ import { scrapeDir, resolveDbRoot, type MediaKind, + type MediaRow, type ScrapeBackend, type ScrapeMeta } from "../../media-scrape/src/index.js"; @@ -160,7 +161,7 @@ export class MediaService { try { const store = new ScrapeStore(dir); try { - const sums = store.sumBytesByKind(); + const sums = store.storedBytesByKind(); for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind]; } finally { store.close(); @@ -279,8 +280,24 @@ export class MediaService { }; } + /** One media row inside a scrape, with its scrape and channel for context. */ + getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { + const meta = this.engine.get(scrapeId); + if (!meta) return null; + + const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); + try { + const row = store.getMedia(itemId); + return row ? { ...row, scrapeId, channelId: meta.channelId } : null; + } finally { + store.close(); + } + } + /** Read media rows across every scrape, newest scrape first. */ - queryMedia(options: { kind?: MediaKind; limit?: number; offset?: number } = {}): MediaItemResult[] { + queryMedia( + options: { kind?: MediaKind; scrapeId?: string; limit?: number; offset?: number } = {} + ): MediaItemResult[] { const limit = Math.max(1, Math.min(options.limit ?? 60, 500)); const offset = Math.max(0, options.offset ?? 0); const results: MediaItemResult[] = []; @@ -288,6 +305,7 @@ export class MediaService { for (const meta of this.engine.list()) { if (meta.state === "deleted") continue; + if (options.scrapeId && meta.id !== options.scrapeId) continue; const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try {