Add message counting and date-bounded scrapes
This commit is contained in:
@@ -778,10 +778,12 @@ function buildServer(): McpServer {
|
||||
downloadDir: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe("Absolute destination for downloads; previews stay local")
|
||||
.describe("Absolute destination for downloads; previews stay local"),
|
||||
after: z.string().optional().describe("From-date ISO; the walk stops below it"),
|
||||
before: z.string().optional().describe("To-date ISO; the walk starts just below it")
|
||||
}
|
||||
},
|
||||
async ({ channelId, guildId, mode, backend, confirm, downloadDir }) =>
|
||||
async ({ channelId, guildId, mode, backend, confirm, downloadDir, after, before }) =>
|
||||
jsonText(
|
||||
await media.startScrape({
|
||||
channelId,
|
||||
@@ -789,7 +791,32 @@ function buildServer(): McpServer {
|
||||
mode,
|
||||
backend,
|
||||
confirm,
|
||||
downloadDir: downloadDir ?? null
|
||||
downloadDir: downloadDir ?? null,
|
||||
after: after ?? null,
|
||||
before: before ?? null
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"count_media_messages",
|
||||
{
|
||||
description:
|
||||
"Discord search total for a channel or date range. Guild channels with slopbot only; DMs and group chats are not available.",
|
||||
inputSchema: {
|
||||
channelId: z.string(),
|
||||
guildId: z.string().optional(),
|
||||
after: z.string().optional().describe("From-date ISO"),
|
||||
before: z.string().optional().describe("To-date ISO")
|
||||
}
|
||||
},
|
||||
async ({ channelId, guildId, after, before }) =>
|
||||
jsonText(
|
||||
await media.countMessages({
|
||||
channelId,
|
||||
guildId: guildId ?? null,
|
||||
after: after ?? null,
|
||||
before: before ?? null
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
+54
-1
@@ -14,6 +14,7 @@ import {
|
||||
ScrapeEngine,
|
||||
ScrapeStore,
|
||||
scrapeDir,
|
||||
snowflakeFromIso,
|
||||
resolveDbRoot,
|
||||
type MediaKind,
|
||||
type MediaRow,
|
||||
@@ -22,12 +23,14 @@ import {
|
||||
} from "../../media-scrape/src/index.js";
|
||||
import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js";
|
||||
import {
|
||||
countMessages,
|
||||
loadSlopbotConfig,
|
||||
slopbotFetcher,
|
||||
slopbotStatus,
|
||||
SlopbotBackend,
|
||||
startSlopbot,
|
||||
type SlopbotFetcher,
|
||||
type SlopbotRest,
|
||||
type SlopbotStatus
|
||||
} from "../../media-scrape/slopbot/index.js";
|
||||
import type { CoreClient } from "./core-client.js";
|
||||
@@ -51,6 +54,10 @@ export interface StartScrapeInput {
|
||||
confirm?: boolean;
|
||||
/** Absolute destination for downloaded files. Null keeps them under the scrape dir. */
|
||||
downloadDir?: string | null;
|
||||
/** From-date as ISO; the walk stops below it. */
|
||||
after?: string | null;
|
||||
/** To-date as ISO; the walk starts just below it. */
|
||||
before?: string | null;
|
||||
}
|
||||
|
||||
export interface MediaErrorEntry {
|
||||
@@ -295,6 +302,8 @@ export class MediaService {
|
||||
|
||||
try {
|
||||
const mode = input.mode ?? "links";
|
||||
const afterId = input.after ? snowflakeFromIso(input.after) : null;
|
||||
const beforeId = input.before ? snowflakeFromIso(input.before) : null;
|
||||
let backend: ScrapeBackend;
|
||||
|
||||
if (input.backend === "puppet") {
|
||||
@@ -314,12 +323,28 @@ export class MediaService {
|
||||
: this.puppetBackend();
|
||||
}
|
||||
|
||||
// Best effort: one search request gives the total for the range.
|
||||
let estimatedMessages: number | null = null;
|
||||
if (backend.name === "slopbot" && input.guildId && this.bot) {
|
||||
estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, {
|
||||
channelId: input.channelId,
|
||||
guildId: input.guildId,
|
||||
afterId,
|
||||
beforeId
|
||||
});
|
||||
}
|
||||
|
||||
return this.engine.start({
|
||||
channelId: input.channelId,
|
||||
guildId: input.guildId ?? null,
|
||||
mode,
|
||||
backend,
|
||||
downloadDir: input.downloadDir ?? null
|
||||
downloadDir: input.downloadDir ?? null,
|
||||
afterId,
|
||||
beforeId,
|
||||
fromDate: input.after ?? null,
|
||||
toDate: input.before ?? null,
|
||||
estimatedMessages
|
||||
});
|
||||
} catch (error) {
|
||||
this.recordError("scrape", error instanceof Error ? error.message : String(error));
|
||||
@@ -373,6 +398,34 @@ export class MediaService {
|
||||
};
|
||||
}
|
||||
|
||||
/** Discord search total for a channel or date range. Slopbot (guild) channels only. */
|
||||
async countMessages(input: {
|
||||
channelId: string;
|
||||
guildId?: string | null;
|
||||
after?: string | null;
|
||||
before?: string | null;
|
||||
}): Promise<{ count: number | null; reason?: string }> {
|
||||
if (!input.guildId) {
|
||||
return { count: null, reason: "not available for DMs or group chats" };
|
||||
}
|
||||
|
||||
const available = await this.ensureSlopbot();
|
||||
if (!available || !this.bot) {
|
||||
return { count: null, reason: "slopbot is not connected" };
|
||||
}
|
||||
|
||||
const count = await countMessages(this.bot.rest as SlopbotRest, {
|
||||
channelId: input.channelId,
|
||||
guildId: input.guildId,
|
||||
afterId: input.after ? snowflakeFromIso(input.after) : null,
|
||||
beforeId: input.before ? snowflakeFromIso(input.before) : null
|
||||
});
|
||||
|
||||
return count === null
|
||||
? { count: null, reason: "search unavailable or rate limited" }
|
||||
: { count };
|
||||
}
|
||||
|
||||
private recordError(source: string, message: string): void {
|
||||
this.errors.unshift({ at: new Date().toISOString(), source, message });
|
||||
if (this.errors.length > 50) this.errors.length = 50;
|
||||
|
||||
+23
-1
@@ -832,6 +832,24 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/media/count",
|
||||
wrap(async (req, res) => {
|
||||
const channelId = String(req.body?.channelId ?? "");
|
||||
if (!channelId) throw new HttpError(400, "channelId is required");
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
result: await deps.media.countMessages({
|
||||
channelId,
|
||||
guildId: req.body?.guildId ? String(req.body.guildId) : null,
|
||||
after: req.body?.after ? String(req.body.after) : null,
|
||||
before: req.body?.before ? String(req.body.before) : null
|
||||
})
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/media/errors",
|
||||
wrap(async (_req, res) => {
|
||||
@@ -859,6 +877,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
const backend =
|
||||
requested === "slopbot" || requested === "puppet" ? requested : "auto";
|
||||
const downloadDir = req.body?.downloadDir ? String(req.body.downloadDir) : null;
|
||||
const after = req.body?.after ? String(req.body.after) : null;
|
||||
const before = req.body?.before ? String(req.body.before) : null;
|
||||
|
||||
const scrape = await deps.media.startScrape({
|
||||
channelId,
|
||||
@@ -866,7 +886,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
mode,
|
||||
confirm,
|
||||
backend,
|
||||
downloadDir
|
||||
downloadDir,
|
||||
after,
|
||||
before
|
||||
});
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "media",
|
||||
|
||||
Reference in New Issue
Block a user