From fae754fba0dd1a242dfb9d3d1dd8a975a503a972 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 12:06:34 +0200 Subject: [PATCH] Add message counting and date-bounded scrapes --- CURRENT.md | 6 ++++ docs/daemon.md | 3 +- docs/media-mcp.md | 3 +- docs/media-scraper.md | 7 ++-- docs/media.md | 5 +-- docs/testing.md | 4 +++ docs/webui.md | 3 +- media-scrape/slopbot/client.ts | 27 ++++++++++++++ media-scrape/slopbot/index.ts | 10 +++++- media-scrape/src/engine.ts | 32 +++++++++++++---- media-scrape/src/index.ts | 1 + media-scrape/src/snowflake.ts | 21 +++++++++++ media-scrape/src/types.ts | 10 ++++++ media-scrape/test/live-count.ts | 50 ++++++++++++++++++++++++++ media-scrape/test/run.ts | 45 ++++++++++++++++++++++++ media-scrape/test/slopbot.ts | 30 ++++++++++++++++ poc/package.json | 1 + poc/public/media.js | 62 +++++++++++++++++++++++++++++++-- poc/src/control-bridge.ts | 33 ++++++++++++++++-- poc/src/media.ts | 55 ++++++++++++++++++++++++++++- poc/src/webui.ts | 24 ++++++++++++- 21 files changed, 410 insertions(+), 22 deletions(-) create mode 100644 media-scrape/src/snowflake.ts create mode 100644 media-scrape/test/live-count.ts diff --git a/CURRENT.md b/CURRENT.md index 2270756..e3195c0 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,12 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C16 Message counting and date-bounded scrapes. Discord search (`total_results`, one guild + request, slopbot only) powers `POST /api/media/count` and MCP `count_media_messages`; the + confirm dialog has From/To dates plus a **Check Message Count** button; the walk honors + `afterId`/`beforeId`; `estimatedMessages`, `fromDate`, and `toDate` land in the scrape meta and + History. Live: count 17 all / 17 after 2026-09-16 / 0 before, DM returns a reason, and a + from-dated scrape stored 17 messages with `estimatedMessages: 17`. - [x] C15 Scrape UX: custom download destination (absolute path per scrape; files there, previews local, embed thumbnails copied to both, custom files not served), confirmation popup before Links/Download, New scrape/History right-aligned with New scrape furthest right, modal diff --git a/docs/daemon.md b/docs/daemon.md index f0a1869..40dc3c2 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -78,7 +78,8 @@ The daemon exposes MCP over Streamable HTTP. The default address is `http://127. | `list_dms` | List cached DM and group DM channels with recipients. | | `get_message_log` | Read the trigger log. | | `get_action_log` | Read the persistent action log. | -| `scrape_media` | Start a media scrape of a channel (`channelId`, `guildId?`, `mode?`, `confirm?`). | +| `scrape_media` | Start a media scrape of a channel (`channelId`, `guildId?`, `mode?`, `backend?`, `confirm?`, `downloadDir?`, `after?`, `before?`). | +| `count_media_messages` | Discord search total for a channel or date range (slopbot guild channels only). | | `list_media_scrapes` | Media scrapes with state, pages, messages, and counts. | | `get_media_scrape` | One scrape by ID. | | `stop_media_scrape` | Stop a scrape; it pauses after the current page. | diff --git a/docs/media-mcp.md b/docs/media-mcp.md index 6426669..21721fa 100644 --- a/docs/media-mcp.md +++ b/docs/media-mcp.md @@ -35,7 +35,8 @@ preset keys from `filter-presets.ts`; `after`/`before` are ISO times on `posted_ | Tool | Access | Purpose | | --- | --- | --- | -| `scrape_media` | trusted | Start a scrape (`channelId`, `guildId?`, `mode?`, `backend?`, `downloadDir?`) | +| `scrape_media` | trusted | Start a scrape (`channelId`, `guildId?`, `mode?`, `backend?`, `downloadDir?`, `after?`, `before?`) | +| `count_media_messages` | trusted | Discord search total for a channel or range (slopbot guild channels only) | | `list_media_scrapes` | trusted | Every scrape with state, size, and counts | | `get_media_scrape` | trusted | One scrape, live counters and cursor | | `stop_media_scrape` | trusted | Stop the active scrape | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 0d13861..cb19643 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -28,7 +28,8 @@ backend only. 1. The command resolves the channel and creates a scrape directory. 2. Newest to oldest: fetch `limit` messages before the cursor. The cursor is the oldest message ID - seen. + seen. A to-date sets the starting cursor; a from-date trims each page and ends the walk when the + boundary is crossed (`afterId`/`beforeId` in the meta). 3. Extract media from each message and write the page to the database in one transaction. 4. Persist `scrape.json` (cursor, counters, state) after every page. 5. Stop when a page returns fewer than `limit` messages or the oldest message is reached. @@ -85,7 +86,9 @@ One SQLite file per scrape (`index.db`, `node:sqlite`). | `media_sources` | `media_id`, `message_id` | | `downloads` | `media_id`, `state`, `attempts`, `error` | -Indexes: `kind`, `posted_at`, and `media_sources(message_id)`. The database is an index; the media +`scrape.json` also records `fromDate`/`toDate`, and `estimatedMessages` from Discord search when the +slopbot backend could ask for it (one request; null otherwise). Indexes: `kind`, `posted_at`, and +`media_sources(message_id)`. The database is an index; the media folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time (`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Filter presets live in `media-scrape/src/filter-presets.ts`: aspect buckets (vertical, square, 4:3, 16:9, 21:9, other) and diff --git a/docs/media.md b/docs/media.md index 6d361f4..6e7281c 100644 --- a/docs/media.md +++ b/docs/media.md @@ -139,8 +139,9 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun Grid with lazy loading; previews come from the cache, then the local file, then the CDN; each tile's tooltip shows the poster and posted time. A scrape strip shows live progress with Stop. **New scrape…** (rightmost) opens a popup with Server and Group chat trees (servers expand to - channels; each row has Links and Download). Every scrape button asks for confirmation, and - Download can target a custom absolute destination. **History** lists each scrape with its start + channels; each row has Links and Download). Every scrape button asks for confirmation; the + dialog takes optional From/To dates and a **Check Message Count** button (Discord search total, + slopbot guild channels only), and Download can target a custom absolute destination. **History** lists each scrape with its start time, duration, pages, messages, media counts, bytes, files, and previews. The popups close on backdrop click, X, or Close. Only active scrapes show in the strip; the view fills the window below the header and the call strip. The Bot tab lists bot and scrape errors with a red badge diff --git a/docs/testing.md b/docs/testing.md index 28e767e..2a57063 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -186,6 +186,10 @@ and fetches one history page without printing the token: cd poc VESKSLOP_SLOPBOT_ENABLED=1 VESKSLOP_SLOPBOT_STATE=/path/to/slopbot.json \ pnpm test:slopbot:live [guildId] + +# Search total for a channel or range, without scraping: +VESKSLOP_SLOPBOT_ENABLED=1 VESKSLOP_SLOPBOT_STATE=/path/to/slopbot.json \ + pnpm test:slopbot:count [afterIso] [beforeIso] ``` ## Sanctioned live target diff --git a/docs/webui.md b/docs/webui.md index bdb7204..6d3d1b7 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -264,7 +264,8 @@ replaceable. | GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts | | GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over | -| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?`, `downloadDir?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`; `downloadDir` is an absolute path for downloaded files (previews stay local). Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` | +| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?`, `downloadDir?`, `after?`, `before?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`; `downloadDir` is an absolute path for downloaded files (previews stay local); `after`/`before` are ISO dates bounding the walk. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` | +| POST | `/api/media/count` | `channelId`, `guildId?`, `after?`, `before?` | Discord search total for the channel or range. Returns `{ count }` or `{ count: null, reason }` (slopbot guild channels only) | | GET | `/api/media/errors` | none | Bot and scrape errors, newest first | | POST | `/api/media/errors/clear` | none | Clear the error list | | POST | `/api/media/scrape/stop` | `id` | Stop a media scrape | diff --git a/media-scrape/slopbot/client.ts b/media-scrape/slopbot/client.ts index af2a190..ce8fc99 100644 --- a/media-scrape/slopbot/client.ts +++ b/media-scrape/slopbot/client.ts @@ -54,6 +54,33 @@ export function slopbotStatus(client: Client | null): SlopbotStatus { }; } +/** The REST surface the search count needs. discord.js `client.rest` satisfies it. */ +export interface SlopbotRest { + get(path: string, options?: { query?: URLSearchParams }): Promise; +} + +/** + * Discord search `total_results` for a channel or range. One request. + * Null when the search fails (rate limit, permissions, or no index). + */ +export async function countMessages( + rest: SlopbotRest, + input: { channelId: string; guildId: string; afterId?: string | null; beforeId?: string | null } +): Promise { + const query = new URLSearchParams({ channel_id: input.channelId, limit: "1" }); + if (input.afterId) query.set("min_id", input.afterId); + if (input.beforeId) query.set("max_id", input.beforeId); + + try { + const data = (await rest.get(`/guilds/${input.guildId}/messages/search`, { query })) as { + total_results?: number; + }; + return typeof data?.total_results === "number" ? data.total_results : null; + } catch { + return null; + } +} + /** The ScrapeBackend fetcher over a live client. */ export function slopbotFetcher(client: Client): SlopbotFetcher { return { diff --git a/media-scrape/slopbot/index.ts b/media-scrape/slopbot/index.ts index 32cfa63..b1c24f3 100644 --- a/media-scrape/slopbot/index.ts +++ b/media-scrape/slopbot/index.ts @@ -13,7 +13,15 @@ export type SlopbotClient = Client; export { loadSlopbotConfig, type SlopbotConfig }; export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js"; -export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js"; +export { + countMessages, + slopbotFetcher, + slopbotStatus, + startSlopbot, + stopSlopbot, + type SlopbotRest, + type SlopbotStatus +} from "./client.js"; /** View Channels + Read Message History. The bot reads only; it never sends. */ const PERMISSIONS = String((1 << 10) | (1 << 16)); diff --git a/media-scrape/src/engine.ts b/media-scrape/src/engine.ts index 88f8fbc..c492f66 100644 --- a/media-scrape/src/engine.ts +++ b/media-scrape/src/engine.ts @@ -95,6 +95,11 @@ export class ScrapeEngine { mode: ScrapeMode; backend: ScrapeBackend; downloadDir?: string | null; + afterId?: string | null; + beforeId?: string | null; + fromDate?: string | null; + toDate?: string | null; + estimatedMessages?: number | null; }): Promise { const now = this.now(); const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now)); @@ -108,7 +113,7 @@ export class ScrapeEngine { backend: input.backend.name, mode: input.mode, state: "queued", - cursor: null, + cursor: input.beforeId ?? null, createdAt: timestamp, updatedAt: timestamp, pages: 0, @@ -116,7 +121,12 @@ export class ScrapeEngine { media: emptyMediaCounts(), errors: 0, lastError: null, - downloadDir: input.downloadDir ?? null + downloadDir: input.downloadDir ?? null, + afterId: input.afterId ?? null, + beforeId: input.beforeId ?? null, + fromDate: input.fromDate ?? null, + toDate: input.toDate ?? null, + estimatedMessages: input.estimatedMessages ?? null }; writeScrapeMeta(dir, meta); @@ -217,15 +227,25 @@ export class ScrapeEngine { break; } - const added = store.insertPage(page.messages); + // The from-date is a lower ID boundary: trim anything older and finish. + const kept = meta.afterId + ? page.messages.filter(message => BigInt(message.id) > BigInt(meta.afterId as string)) + : page.messages; + + if (kept.length === 0) { + meta.state = "done"; + break; + } + + const added = store.insertPage(kept); for (const kind of MEDIA_KINDS) meta.media[kind] += added[kind]; meta.pages++; - meta.messages += page.messages.length; - meta.cursor = oldestId(page.messages) ?? meta.cursor; + meta.messages += kept.length; + meta.cursor = oldestId(kept) ?? meta.cursor; this.pageTimes.push(this.now()); this.persist(dir, meta); - if (page.messages.length < pageSize) { + if (kept.length < page.messages.length || page.messages.length < pageSize) { meta.state = "done"; break; } diff --git a/media-scrape/src/index.ts b/media-scrape/src/index.ts index 2e8cf8e..854eb70 100644 --- a/media-scrape/src/index.ts +++ b/media-scrape/src/index.ts @@ -5,6 +5,7 @@ export { extractMedia, type ExtractedMedia } from "./extract.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; +export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js"; export { ASPECT_PRESETS, RESOLUTION_PRESETS, diff --git a/media-scrape/src/snowflake.ts b/media-scrape/src/snowflake.ts new file mode 100644 index 0000000..27bfc8b --- /dev/null +++ b/media-scrape/src/snowflake.ts @@ -0,0 +1,21 @@ +/* + * Discord snowflake helpers for date-bounded scrapes. + */ + +const DISCORD_EPOCH = 1420070400000; + +/** The snowflake at an instant. Null for an unparsable date; "0" before the epoch. */ +export function snowflakeFromIso(iso: string): string | null { + const time = Date.parse(iso); + if (!Number.isFinite(time)) return null; + + const value = BigInt(time - DISCORD_EPOCH) << 22n; + return value > 0n ? value.toString() : "0"; +} + +/** The instant a snowflake encodes, as an ISO string. Null when it is not a snowflake. */ +export function isoFromSnowflake(id: string | null | undefined): string | null { + if (!id || !/^\d{17,20}$/.test(id)) return null; + const time = Number(BigInt(id) >> 22n) + DISCORD_EPOCH; + return Number.isFinite(time) ? new Date(time).toISOString() : null; +} diff --git a/media-scrape/src/types.ts b/media-scrape/src/types.ts index eb2af74..34c37ac 100644 --- a/media-scrape/src/types.ts +++ b/media-scrape/src/types.ts @@ -96,6 +96,16 @@ export interface ScrapeMeta { finishedAt?: string | null; /** Absolute destination for downloaded files. Null means the local `media/` folder. */ downloadDir?: string | null; + /** Lower message-ID boundary (exclusive), derived from the from-date. */ + afterId?: string | null; + /** Upper message-ID boundary; the walk starts just below it. */ + beforeId?: string | null; + /** From-date as ISO, for display in history. */ + fromDate?: string | null; + /** To-date as ISO, for display in history. */ + toDate?: string | null; + /** Discord search total for the range, when known (slopbot only). */ + estimatedMessages?: number | null; } export function emptyMediaCounts(): Record { diff --git a/media-scrape/test/live-count.ts b/media-scrape/test/live-count.ts new file mode 100644 index 0000000..997f9d5 --- /dev/null +++ b/media-scrape/test/live-count.ts @@ -0,0 +1,50 @@ +/* + * Live check: Discord search total_results for a channel or a date range. + * + * Usage (from poc/): + * VESKSLOP_SLOPBOT_ENABLED=1 VESKSLOP_SLOPBOT_STATE=/path/to/slopbot.json \ + * tsx ../media-scrape/test/live-count.ts [afterIso] [beforeIso] + */ + +import { loadSlopbotConfig, startSlopbot, stopSlopbot } from "../slopbot/index.js"; + +function snowflakeFromIso(iso: string): string { + return String((BigInt(Date.parse(iso) - 1420070400000) << 22n)); +} + +async function main() { + const [channelId, guildId, after, before] = process.argv.slice(2); + if (!channelId || !guildId) { + console.error("usage: live-count.ts [afterIso] [beforeIso]"); + process.exit(1); + } + + const config = loadSlopbotConfig(); + if (!config.enabled) { + console.error("slopbot disabled: set VESKSLOP_SLOPBOT_ENABLED=1 and a token"); + process.exit(1); + } + + const client = await startSlopbot(config); + try { + const query = new URLSearchParams({ channel_id: channelId, limit: "1" }); + if (after) query.set("min_id", snowflakeFromIso(after)); + if (before) query.set("max_id", snowflakeFromIso(before)); + + for (const [label, path] of [ + ["guild", `/guilds/${guildId}/messages/search`], + ["channel", `/channels/${channelId}/messages/search`] + ] as const) { + try { + const data = (await client.rest.get(path, { query })) as Record; + console.log(`${label}: total_results=${String(data?.total_results)} keys=${Object.keys(data ?? {}).join(",")}`); + } catch (error) { + console.log(`${label}: error ${(error as Error).message}`); + } + } + } finally { + await stopSlopbot(client); + } +} + +void main(); diff --git a/media-scrape/test/run.ts b/media-scrape/test/run.ts index 2609ed4..bd8ca96 100644 --- a/media-scrape/test/run.ts +++ b/media-scrape/test/run.ts @@ -14,6 +14,7 @@ import { extractMedia } from "../src/extract.js"; import { PuppetBackend } from "../src/puppet-backend.js"; import { aspectCondition, resolutionCondition } from "../src/filter-presets.js"; import { isNearLimit, planRetention } from "../src/retention.js"; +import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js"; import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js"; import type { CoreAttachment, CoreMessage } from "../src/types.js"; @@ -710,6 +711,50 @@ void (async () => { store.close(); }); + await test("snowflake helpers round-trip dates", () => { + const id = snowflakeFromIso("2026-09-16T00:00:00.000Z"); + assert.ok(id); + assert.equal(isoFromSnowflake(id), "2026-09-16T00:00:00.000Z"); + assert.equal(snowflakeFromIso("not a date"), null); + assert.equal(isoFromSnowflake("nope"), null); + }); + + await test("engine honors a from-date boundary", async () => { + const backend = new FakeBackend([ + { messages: [message("1000"), message("900"), message("800")] } + ]); + + const engine = new ScrapeEngine({ + rootDir: root, + pageSize: 10, + pageMinMs: 0, + pageMaxMs: 0, + sleep: async () => {}, + random: () => 0, + log: () => {}, + previews: false + }); + + const started = await engine.start({ + channelId: "131", + guildId: null, + mode: "links", + backend, + afterId: "900" + }); + await engine.processQueue(); + + const meta = engine.get(started.id); + assert.equal(meta?.state, "done"); + assert.equal(meta?.messages, 1); + assert.equal(meta?.afterId, "900"); + assert.equal(backend.calls.length, 1); + + const store = new ScrapeStore(join(root, started.id)); + assert.equal(store.queryMedia({}).length, 0); + store.close(); + }); + await test("retention marks the oldest completed scrapes up to the limit", () => { const scrapes = [ { id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 }, diff --git a/media-scrape/test/slopbot.ts b/media-scrape/test/slopbot.ts index ff6ecda..7410094 100644 --- a/media-scrape/test/slopbot.ts +++ b/media-scrape/test/slopbot.ts @@ -6,6 +6,7 @@ import assert from "node:assert/strict"; import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js"; import { mapSlopbotMessage } from "../slopbot/backend.js"; +import { countMessages } from "../slopbot/client.js"; let failures = 0; @@ -138,6 +139,35 @@ void (async () => { assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm"); }); + await test("countMessages reads guild search totals", async () => { + const calls: string[] = []; + const rest = { + async get(path: string, options?: { query?: URLSearchParams }) { + calls.push(`${path}?${options?.query?.toString() ?? ""}`); + return { total_results: 7 }; + } + }; + + const count = await countMessages(rest, { + channelId: "1", + guildId: "2", + afterId: "100", + beforeId: "200" + }); + assert.equal(count, 7); + assert.equal(calls[0], "/guilds/2/messages/search?channel_id=1&limit=1&min_id=100&max_id=200"); + + const failed = await countMessages( + { + async get() { + throw new Error("nope"); + } + }, + { channelId: "1", guildId: "2" } + ); + assert.equal(failed, null); + }); + await test("backend pages oldest first and passes before and limit", async () => { const fetcher = new FakeFetcher([ [discordMessage({ id: "1002" }), discordMessage({ id: "1001" })], diff --git a/poc/package.json b/poc/package.json index 3f92ab4..b27b745 100644 --- a/poc/package.json +++ b/poc/package.json @@ -18,6 +18,7 @@ "test:audio": "tsx src/test-audio.ts", "test:media": "tsx ../media-scrape/test/run.ts && tsx ../media-scrape/test/slopbot.ts", "test:slopbot:live": "tsx ../media-scrape/test/live-slopbot.ts", + "test:slopbot:count": "tsx ../media-scrape/test/live-count.ts", "audio:doctor": "bash scripts/audio-doctor.sh" }, "dependencies": { diff --git a/poc/public/media.js b/poc/public/media.js index 68f8404..1373301 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -199,7 +199,7 @@ renderGrid(); } - async function startScrape(channelId, guildId, mode, downloadDir) { + async function startScrape(channelId, guildId, mode, downloadDir, after, before) { if (!channelId) { setStatus("no channel selected"); return; @@ -211,7 +211,9 @@ guildId: guildId ?? null, mode, backend: guildId ? "auto" : "puppet", - downloadDir: downloadDir ?? null + downloadDir: downloadDir ?? null, + after: after ?? null, + before: before ?? null }); setStatus(`started ${scrape.id} (${mode})`); scrapeModal.hidden = true; @@ -670,6 +672,16 @@ placeholder="/path/to/download/" hidden /> +
+ From + + To + + + +
@@ -700,8 +712,40 @@ destInput.hidden = !checkbox.checked; }; + const afterInput = confirmModal.querySelector("#confirm-after"); + const beforeInput = confirmModal.querySelector("#confirm-before"); + const countResult = confirmModal.querySelector("#confirm-count-result"); + afterInput.value = ""; + beforeInput.value = ""; + countResult.textContent = ""; + + confirmModal.querySelector("#confirm-count").onclick = async () => { + countResult.textContent = "checking..."; + try { + const response = await window.VK.api.post("/api/media/count", { + channelId, + guildId: guildId ?? null, + after: afterInput.value ? `${afterInput.value}T00:00:00.000Z` : null, + before: beforeInput.value ? `${beforeInput.value}T23:59:59.999Z` : null + }); + countResult.textContent = + response.count === null + ? response.reason ?? "unavailable" + : `${response.count} messages`; + } catch (error) { + countResult.textContent = `failed: ${error.message}`; + } + }; + confirmAction = () => - startScrape(channelId, guildId, mode, isDownload && checkbox.checked ? destInput.value.trim() : null); + startScrape( + channelId, + guildId, + mode, + isDownload && checkbox.checked ? destInput.value.trim() : null, + afterInput.value ? `${afterInput.value}T00:00:00.000Z` : null, + beforeInput.value ? `${beforeInput.value}T23:59:59.999Z` : null + ); confirmModal.hidden = false; } @@ -776,6 +820,18 @@ null, `${scrape.state} · started ${new Date(scrape.createdAt).toLocaleString()} · ` + `took ${durationLabel(scrape)} · ${scrape.pages}p/${scrape.messages}m · ` + + (scrape.estimatedMessages != null + ? `~${scrape.estimatedMessages} total · ` + : "") + + (scrape.fromDate || scrape.toDate + ? `range ${ + scrape.fromDate + ? new Date(scrape.fromDate).toLocaleDateString() + : "..." + } to ${ + scrape.toDate ? new Date(scrape.toDate).toLocaleDateString() : "..." + } · ` + : "") + `img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0} · ` + `${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}` ) diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index bbf6d46..9885da2 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -778,10 +778,12 @@ function buildServer(): McpServer { downloadDir: z .string() .optional() - .describe("Absolute destination for downloads; previews stay local") + .describe("Absolute destination for downloads; previews stay local"), + after: z.string().optional().describe("From-date ISO; the walk stops below it"), + before: z.string().optional().describe("To-date ISO; the walk starts just below it") } }, - async ({ channelId, guildId, mode, backend, confirm, downloadDir }) => + async ({ channelId, guildId, mode, backend, confirm, downloadDir, after, before }) => jsonText( await media.startScrape({ channelId, @@ -789,7 +791,32 @@ function buildServer(): McpServer { mode, backend, confirm, - downloadDir: downloadDir ?? null + downloadDir: downloadDir ?? null, + after: after ?? null, + before: before ?? null + }) + ) + ); + + server.registerTool( + "count_media_messages", + { + description: + "Discord search total for a channel or date range. Guild channels with slopbot only; DMs and group chats are not available.", + inputSchema: { + channelId: z.string(), + guildId: z.string().optional(), + after: z.string().optional().describe("From-date ISO"), + before: z.string().optional().describe("To-date ISO") + } + }, + async ({ channelId, guildId, after, before }) => + jsonText( + await media.countMessages({ + channelId, + guildId: guildId ?? null, + after: after ?? null, + before: before ?? null }) ) ); diff --git a/poc/src/media.ts b/poc/src/media.ts index f29a7ef..2c235bf 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -14,6 +14,7 @@ import { ScrapeEngine, ScrapeStore, scrapeDir, + snowflakeFromIso, resolveDbRoot, type MediaKind, type MediaRow, @@ -22,12 +23,14 @@ import { } from "../../media-scrape/src/index.js"; import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js"; import { + countMessages, loadSlopbotConfig, slopbotFetcher, slopbotStatus, SlopbotBackend, startSlopbot, type SlopbotFetcher, + type SlopbotRest, type SlopbotStatus } from "../../media-scrape/slopbot/index.js"; import type { CoreClient } from "./core-client.js"; @@ -51,6 +54,10 @@ export interface StartScrapeInput { confirm?: boolean; /** Absolute destination for downloaded files. Null keeps them under the scrape dir. */ downloadDir?: string | null; + /** From-date as ISO; the walk stops below it. */ + after?: string | null; + /** To-date as ISO; the walk starts just below it. */ + before?: string | null; } export interface MediaErrorEntry { @@ -295,6 +302,8 @@ export class MediaService { try { const mode = input.mode ?? "links"; + const afterId = input.after ? snowflakeFromIso(input.after) : null; + const beforeId = input.before ? snowflakeFromIso(input.before) : null; let backend: ScrapeBackend; if (input.backend === "puppet") { @@ -314,12 +323,28 @@ export class MediaService { : this.puppetBackend(); } + // Best effort: one search request gives the total for the range. + let estimatedMessages: number | null = null; + if (backend.name === "slopbot" && input.guildId && this.bot) { + estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, { + channelId: input.channelId, + guildId: input.guildId, + afterId, + beforeId + }); + } + return this.engine.start({ channelId: input.channelId, guildId: input.guildId ?? null, mode, backend, - downloadDir: input.downloadDir ?? null + downloadDir: input.downloadDir ?? null, + afterId, + beforeId, + fromDate: input.after ?? null, + toDate: input.before ?? null, + estimatedMessages }); } catch (error) { this.recordError("scrape", error instanceof Error ? error.message : String(error)); @@ -373,6 +398,34 @@ export class MediaService { }; } + /** Discord search total for a channel or date range. Slopbot (guild) channels only. */ + async countMessages(input: { + channelId: string; + guildId?: string | null; + after?: string | null; + before?: string | null; + }): Promise<{ count: number | null; reason?: string }> { + if (!input.guildId) { + return { count: null, reason: "not available for DMs or group chats" }; + } + + const available = await this.ensureSlopbot(); + if (!available || !this.bot) { + return { count: null, reason: "slopbot is not connected" }; + } + + const count = await countMessages(this.bot.rest as SlopbotRest, { + channelId: input.channelId, + guildId: input.guildId, + afterId: input.after ? snowflakeFromIso(input.after) : null, + beforeId: input.before ? snowflakeFromIso(input.before) : null + }); + + return count === null + ? { count: null, reason: "search unavailable or rate limited" } + : { count }; + } + private recordError(source: string, message: string): void { this.errors.unshift({ at: new Date().toISOString(), source, message }); if (this.errors.length > 50) this.errors.length = 50; diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 7e2ec55..b57db0d 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -832,6 +832,24 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.post( + "/api/media/count", + wrap(async (req, res) => { + const channelId = String(req.body?.channelId ?? ""); + if (!channelId) throw new HttpError(400, "channelId is required"); + + res.json({ + ok: true, + result: await deps.media.countMessages({ + channelId, + guildId: req.body?.guildId ? String(req.body.guildId) : null, + after: req.body?.after ? String(req.body.after) : null, + before: req.body?.before ? String(req.body.before) : null + }) + }); + }) + ); + app.get( "/api/media/errors", wrap(async (_req, res) => { @@ -859,6 +877,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { const backend = requested === "slopbot" || requested === "puppet" ? requested : "auto"; const downloadDir = req.body?.downloadDir ? String(req.body.downloadDir) : null; + const after = req.body?.after ? String(req.body.after) : null; + const before = req.body?.before ? String(req.body.before) : null; const scrape = await deps.media.startScrape({ channelId, @@ -866,7 +886,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { mode, confirm, backend, - downloadDir + downloadDir, + after, + before }); deps.getPuppet()?.appendAction({ type: "media",