zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add message counting and date-bounded scrapes

Browse Source
This commit is contained in:
Fisto
2026-09-16 12:06:34 +02:00
parent f14f067413
commit fae754fba0
21 changed files with 410 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/package.json
+1
View File
@@ -18,6 +18,7 @@
"test:audio": "tsx src/test-audio.ts",
"test:media": "tsx ../media-scrape/test/run.ts && tsx ../media-scrape/test/slopbot.ts",
"test:slopbot:live": "tsx ../media-scrape/test/live-slopbot.ts",
"test:slopbot:count": "tsx ../media-scrape/test/live-count.ts",
"audio:doctor": "bash scripts/audio-doctor.sh"
},
"dependencies": {
poc/public/media.js
+59 -3
View File
@@ -199,7 +199,7 @@
renderGrid();
}
async function startScrape(channelId, guildId, mode, downloadDir) {
async function startScrape(channelId, guildId, mode, downloadDir, after, before) {
if (!channelId) {
setStatus("no channel selected");
return;
@@ -211,7 +211,9 @@
guildId: guildId ?? null,
mode,
backend: guildId ? "auto" : "puppet",
downloadDir: downloadDir ?? null
downloadDir: downloadDir ?? null,
after: after ?? null,
before: before ?? null
});
setStatus(`started ${scrape.id} (${mode})`);
scrapeModal.hidden = true;
@@ -670,6 +672,16 @@
placeholder="/path/to/download/"
hidden
/>
<div class="vk-media-toolbar">
<span class="vk-media-status">From</span>
<input id="confirm-after" class="vk-media-input" type="date" />
<span class="vk-media-status">To</span>
<input id="confirm-before" class="vk-media-input" type="date" />
<button id="confirm-count" class="btn btn-secondary vk-small-btn" type="button">
Check Message Count
</button>
<span id="confirm-count-result" class="vk-media-status"></span>
</div>
<div class="vk-media-toolbar">
<button id="confirm-cancel" class="btn btn-secondary vk-small-btn" type="button">Cancel</button>
<button id="confirm-start" class="btn btn-primary vk-small-btn" type="button">Start</button>
@@ -700,8 +712,40 @@
destInput.hidden = !checkbox.checked;
};
const afterInput = confirmModal.querySelector("#confirm-after");
const beforeInput = confirmModal.querySelector("#confirm-before");
const countResult = confirmModal.querySelector("#confirm-count-result");
afterInput.value = "";
beforeInput.value = "";
countResult.textContent = "";
confirmModal.querySelector("#confirm-count").onclick = async () => {
countResult.textContent = "checking...";
try {
const response = await window.VK.api.post("/api/media/count", {
channelId,
guildId: guildId ?? null,
after: afterInput.value ? `${afterInput.value}T00:00:00.000Z` : null,
before: beforeInput.value ? `${beforeInput.value}T23:59:59.999Z` : null
});
countResult.textContent =
response.count === null
? response.reason ?? "unavailable"
: `${response.count} messages`;
} catch (error) {
countResult.textContent = `failed: ${error.message}`;
}
};
confirmAction = () =>
startScrape(channelId, guildId, mode, isDownload && checkbox.checked ? destInput.value.trim() : null);
startScrape(
channelId,
guildId,
mode,
isDownload && checkbox.checked ? destInput.value.trim() : null,
afterInput.value ? `${afterInput.value}T00:00:00.000Z` : null,
beforeInput.value ? `${beforeInput.value}T23:59:59.999Z` : null
);
confirmModal.hidden = false;
}
@@ -776,6 +820,18 @@
null,
`${scrape.state} · started ${new Date(scrape.createdAt).toLocaleString()} · ` +
`took ${durationLabel(scrape)} · ${scrape.pages}p/${scrape.messages}m · ` +
(scrape.estimatedMessages != null
? `~${scrape.estimatedMessages} total · `
: "") +
(scrape.fromDate || scrape.toDate
? `range ${
scrape.fromDate
? new Date(scrape.fromDate).toLocaleDateString()
: "..."
} to ${
scrape.toDate ? new Date(scrape.toDate).toLocaleDateString() : "..."
} · `
: "") +
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0} · ` +
`${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}`
)
poc/src/control-bridge.ts
+30 -3
View File
@@ -778,10 +778,12 @@ function buildServer(): McpServer {
downloadDir: z
.string()
.optional()
.describe("Absolute destination for downloads; previews stay local")
.describe("Absolute destination for downloads; previews stay local"),
after: z.string().optional().describe("From-date ISO; the walk stops below it"),
before: z.string().optional().describe("To-date ISO; the walk starts just below it")
}
},
async ({ channelId, guildId, mode, backend, confirm, downloadDir }) =>
async ({ channelId, guildId, mode, backend, confirm, downloadDir, after, before }) =>
jsonText(
await media.startScrape({
channelId,
@@ -789,7 +791,32 @@ function buildServer(): McpServer {
mode,
backend,
confirm,
downloadDir: downloadDir ?? null
downloadDir: downloadDir ?? null,
after: after ?? null,
before: before ?? null
})
)
);
server.registerTool(
"count_media_messages",
{
description:
"Discord search total for a channel or date range. Guild channels with slopbot only; DMs and group chats are not available.",
inputSchema: {
channelId: z.string(),
guildId: z.string().optional(),
after: z.string().optional().describe("From-date ISO"),
before: z.string().optional().describe("To-date ISO")
}
},
async ({ channelId, guildId, after, before }) =>
jsonText(
await media.countMessages({
channelId,
guildId: guildId ?? null,
after: after ?? null,
before: before ?? null
})
)
);
poc/src/media.ts
+54 -1
View File
@@ -14,6 +14,7 @@ import {
ScrapeEngine,
ScrapeStore,
scrapeDir,
snowflakeFromIso,
resolveDbRoot,
type MediaKind,
type MediaRow,
@@ -22,12 +23,14 @@ import {
} from "../../media-scrape/src/index.js";
import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js";
import {
countMessages,
loadSlopbotConfig,
slopbotFetcher,
slopbotStatus,
SlopbotBackend,
startSlopbot,
type SlopbotFetcher,
type SlopbotRest,
type SlopbotStatus
} from "../../media-scrape/slopbot/index.js";
import type { CoreClient } from "./core-client.js";
@@ -51,6 +54,10 @@ export interface StartScrapeInput {
confirm?: boolean;
/** Absolute destination for downloaded files. Null keeps them under the scrape dir. */
downloadDir?: string | null;
/** From-date as ISO; the walk stops below it. */
after?: string | null;
/** To-date as ISO; the walk starts just below it. */
before?: string | null;
}
export interface MediaErrorEntry {
@@ -295,6 +302,8 @@ export class MediaService {
try {
const mode = input.mode ?? "links";
const afterId = input.after ? snowflakeFromIso(input.after) : null;
const beforeId = input.before ? snowflakeFromIso(input.before) : null;
let backend: ScrapeBackend;
if (input.backend === "puppet") {
@@ -314,12 +323,28 @@ export class MediaService {
: this.puppetBackend();
}
// Best effort: one search request gives the total for the range.
let estimatedMessages: number | null = null;
if (backend.name === "slopbot" && input.guildId && this.bot) {
estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, {
channelId: input.channelId,
guildId: input.guildId,
afterId,
beforeId
});
}
return this.engine.start({
channelId: input.channelId,
guildId: input.guildId ?? null,
mode,
backend,
downloadDir: input.downloadDir ?? null
downloadDir: input.downloadDir ?? null,
afterId,
beforeId,
fromDate: input.after ?? null,
toDate: input.before ?? null,
estimatedMessages
});
} catch (error) {
this.recordError("scrape", error instanceof Error ? error.message : String(error));
@@ -373,6 +398,34 @@ export class MediaService {
};
}
/** Discord search total for a channel or date range. Slopbot (guild) channels only. */
async countMessages(input: {
channelId: string;
guildId?: string | null;
after?: string | null;
before?: string | null;
}): Promise<{ count: number | null; reason?: string }> {
if (!input.guildId) {
return { count: null, reason: "not available for DMs or group chats" };
}
const available = await this.ensureSlopbot();
if (!available || !this.bot) {
return { count: null, reason: "slopbot is not connected" };
}
const count = await countMessages(this.bot.rest as SlopbotRest, {
channelId: input.channelId,
guildId: input.guildId,
afterId: input.after ? snowflakeFromIso(input.after) : null,
beforeId: input.before ? snowflakeFromIso(input.before) : null
});
return count === null
? { count: null, reason: "search unavailable or rate limited" }
: { count };
}
private recordError(source: string, message: string): void {
this.errors.unshift({ at: new Date().toISOString(), source, message });
if (this.errors.length > 50) this.errors.length = 50;
poc/src/webui.ts
+23 -1
View File
@@ -832,6 +832,24 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
})
);
app.post(
"/api/media/count",
wrap(async (req, res) => {
const channelId = String(req.body?.channelId ?? "");
if (!channelId) throw new HttpError(400, "channelId is required");
res.json({
ok: true,
result: await deps.media.countMessages({
channelId,
guildId: req.body?.guildId ? String(req.body.guildId) : null,
after: req.body?.after ? String(req.body.after) : null,
before: req.body?.before ? String(req.body.before) : null
})
});
})
);
app.get(
"/api/media/errors",
wrap(async (_req, res) => {
@@ -859,6 +877,8 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
const backend =
requested === "slopbot" || requested === "puppet" ? requested : "auto";
const downloadDir = req.body?.downloadDir ? String(req.body.downloadDir) : null;
const after = req.body?.after ? String(req.body.after) : null;
const before = req.body?.before ? String(req.body.before) : null;
const scrape = await deps.media.startScrape({
channelId,
@@ -866,7 +886,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
mode,
confirm,
backend,
downloadDir
downloadDir,
after,
before
});
deps.getPuppet()?.appendAction({
type: "media",