diff --git a/CURRENT.md b/CURRENT.md index 7fd9a5d..0f5dc7d 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -27,7 +27,7 @@ mobile draft at 390px. - [x] C1 design docs, AGENTS/INDEX pointers, `.gitignore` - [x] C2 core message shape: embeds, forwards, attachment metadata - [x] C3 media-scrape store and puppet walker, tests - - [ ] C4 slopbot backend, tests + - [x] C4 slopbot backend, tests - [ ] C5 Web UI Media and Bot tabs, API routes - [ ] C6 storage limit, warning flow, clean modal, settings - [ ] C7 MCP service hooks diff --git a/docs/slopbot.md b/docs/slopbot.md index 0e1681c..cdbc1c3 100644 --- a/docs/slopbot.md +++ b/docs/slopbot.md @@ -47,10 +47,16 @@ channels. ## Code -- `media-scrape/slopbot/` holds the bot core: client lifecycle, the `ScrapeBackend` implementation, - and the invite helper. -- discord.js is a dependency of the daemon package (`poc/package.json`), the same way the MCP SDK - is. The code stays in `media-scrape/`; the daemon only mounts it. +| File | Role | +| --- | --- | +| `media-scrape/slopbot/config.ts` | Environment and state-file settings | +| `media-scrape/slopbot/client.ts` | Login, status, and the live fetcher | +| `media-scrape/slopbot/backend.ts` | discord.js messages to the core message shape | +| `media-scrape/slopbot/index.ts` | Module surface | + +- discord.js lives in `media-scrape/package.json`; the folder is its own pnpm workspace root. +- The mapping is duck-typed, so the tests run without a token. The live Discord shapes were checked + against the discord.js 14 typings, but a real login is still owed at C8. - The Bot tab shows connection state, the guilds slopbot is in, the active scrape, and the queue. It also renders the invite URL from the client ID. @@ -69,6 +75,6 @@ Slopbot runs with the faster bot budget; the puppet backend uses the slower user ## Status -Design approved 2026-09-16. Scaffold added 2026-09-16: `media-scrape/slopbot/` (config loader and -invite helper) and the discord.js dependency in `media-scrape/package.json`. Unused until -checkpoint C4 in [CURRENT.md](../CURRENT.md). +Backend implemented 2026-09-16 (C4): mapping, paging, and client lifecycle, tested with a fake +fetcher. Not wired into the daemon yet; that is C5. A live login with a real token is owed at C8. +Checkpoints: [CURRENT.md](../CURRENT.md). diff --git a/docs/testing.md b/docs/testing.md index f963865..aac6605 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -17,8 +17,9 @@ pnpm test:audio pnpm test:media ``` -The files are `poc/src/test-communication.ts`, `poc/src/test-cache.ts`, and -`media-scrape/test/run.ts` (media archive). The tests do not need Discord or Vesktop. +The files are `poc/src/test-communication.ts`, `poc/src/test-cache.ts`, and the media archive +tests `media-scrape/test/run.ts` (engine) and `media-scrape/test/slopbot.ts` (bot mapping). The +tests do not need Discord or Vesktop. ## Type checks diff --git a/media-scrape/slopbot/backend.ts b/media-scrape/slopbot/backend.ts new file mode 100644 index 0000000..0161435 --- /dev/null +++ b/media-scrape/slopbot/backend.ts @@ -0,0 +1,131 @@ +/* + * Slopbot scrape backend: discord.js messages to the core message shape. + * + * The mapping is duck-typed so tests run without a bot token. The live shape was + * checked against the discord.js 14 typings; a live check is still owed at C8. + */ + +import type { FetchPage, ScrapeBackend } from "../src/engine.js"; +import type { CoreAttachment, CoreEmbed, CoreMessage, CoreSticker, MediaMessage } from "../src/types.js"; + +/** Discord message flag for a voice message. */ +const VOICE_MESSAGE_FLAG = 1 << 13; + +/** The minimal fetcher the backend needs. The real one wraps a discord.js client. */ +export interface SlopbotFetcher { + fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise; +} + +/** discord.js Collections are iterable and have `.values()`; tests use Maps and arrays. */ +function collectionValues(value: any): any[] { + if (!value) return []; + if (Array.isArray(value)) return value; + if (typeof value.values === "function") return [...value.values()]; + return []; +} + +function mapAttachments(message: any): CoreAttachment[] { + return collectionValues(message?.attachments).map((attachment: any) => ({ + id: attachment?.id ?? null, + filename: attachment?.name ?? attachment?.filename ?? "", + url: attachment?.url ?? null, + contentType: attachment?.contentType ?? attachment?.content_type ?? null, + size: attachment?.size ?? null, + width: attachment?.width ?? null, + height: attachment?.height ?? null, + durationSecs: attachment?.duration ?? attachment?.durationSecs ?? attachment?.duration_secs ?? null + })); +} + +function mapEmbeds(message: any): CoreEmbed[] { + const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null); + + return collectionValues(message?.embeds).map((embed: any) => { + const data = embed?.data ?? embed ?? {}; + return { + type: data.type ?? null, + url: data.url ?? null, + title: data.title ?? null, + provider: data.provider?.name ?? null, + videoUrl: assetUrl(data.video), + imageUrl: assetUrl(data.image), + thumbnailUrl: assetUrl(data.thumbnail) + }; + }); +} + +function mapStickers(message: any): CoreSticker[] { + return collectionValues(message?.stickers).map((sticker: any) => ({ + id: sticker?.id ?? null, + name: sticker?.name ?? null, + formatType: sticker?.format ?? null + })); +} + +function isVoiceMessage(message: any): boolean { + const flags = message?.flags; + if (typeof flags?.has === "function") return flags.has(VOICE_MESSAGE_FLAG); + return Boolean(Number(flags ?? 0) & VOICE_MESSAGE_FLAG); +} + +function createdAt(message: any): string | null { + if (message?.createdAt instanceof Date) return message.createdAt.toISOString(); + if (typeof message?.timestamp === "string") return message.timestamp; + return null; +} + +function mapSnapshot(snapshot: any): MediaMessage & { content: string; timestamp: string | null } { + const source = snapshot?.message ?? snapshot; + + return { + content: source?.content ?? "", + timestamp: createdAt(source), + attachments: mapAttachments(source), + embeds: mapEmbeds(source), + stickerItems: mapStickers(source), + voiceMessage: isVoiceMessage(source) + }; +} + +/** Map one discord.js message to the core message shape. */ +export function mapSlopbotMessage(message: any): CoreMessage { + const reference = message?.reference; + const edited = message?.editedAt ?? message?.editedTimestamp ?? null; + + return { + id: String(message?.id ?? ""), + author: { + id: String(message?.author?.id ?? ""), + username: message?.author?.username ?? "", + avatar: message?.author?.avatar ?? null + }, + content: message?.content ?? "", + timestamp: createdAt(message) ?? "", + editedAt: edited instanceof Date ? edited.toISOString() : null, + replyTo: reference?.messageId + ? { id: String(reference.messageId), authorId: null, authorName: null } + : null, + attachments: mapAttachments(message), + embeds: mapEmbeds(message), + stickerItems: mapStickers(message), + forwarded: collectionValues(message?.messageSnapshots).map(mapSnapshot), + voiceMessage: isVoiceMessage(message) + }; +} + +export class SlopbotBackend implements ScrapeBackend { + readonly name = "slopbot" as const; + + constructor(private readonly fetcher: SlopbotFetcher) {} + + async fetchPage(channelId: string, before: string | null, limit: number): Promise { + const messages = await this.fetcher.fetchMessages(channelId, { + limit, + ...(before ? { before } : {}) + }); + + const mapped = messages.map(mapSlopbotMessage).filter(message => message.id.length > 0); + mapped.sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1)); + return { messages: mapped }; + } +} diff --git a/media-scrape/slopbot/client.ts b/media-scrape/slopbot/client.ts new file mode 100644 index 0000000..5b356ea --- /dev/null +++ b/media-scrape/slopbot/client.ts @@ -0,0 +1,65 @@ +/* + * Slopbot client lifecycle. Separate from the Vesktop core by design. + * Setup and security: docs/slopbot.md. + */ + +import { Client, GatewayIntentBits } from "discord.js"; +import type { SlopbotConfig } from "./config.js"; +import type { SlopbotFetcher } from "./backend.js"; + +export interface SlopbotStatus { + connected: boolean; + user: string | null; + guilds: string[]; +} + +/** + * Log in and return a ready client. The bot needs no message events, so it + * declares only the Guilds intent; REST history does not use the gateway. + */ +export async function startSlopbot(config: SlopbotConfig): Promise { + if (!config.enabled || !config.token) { + throw new Error("slopbot is disabled or has no token"); + } + + const client = new Client({ intents: [GatewayIntentBits.Guilds] }); + await client.login(config.token); + return client; +} + +export async function stopSlopbot(client: Client | null): Promise { + if (!client) return; + await client.destroy(); +} + +/** The Bot tab reads this. Safe with a null client. */ +export function slopbotStatus(client: Client | null): SlopbotStatus { + if (!client || !client.isReady()) { + return { connected: false, user: null, guilds: [] }; + } + + return { + connected: true, + user: client.user?.tag ?? null, + guilds: [...client.guilds.cache.values()].map(guild => guild.name) + }; +} + +/** The ScrapeBackend fetcher over a live client. */ +export function slopbotFetcher(client: Client): SlopbotFetcher { + return { + async fetchMessages(channelId, options) { + const channel = await client.channels.fetch(channelId); + if (!channel || !channel.isTextBased()) { + throw new Error(`channel ${channelId} is not text based`); + } + + const messages = await channel.messages.fetch({ + limit: options.limit, + ...(options.before ? { before: options.before } : {}) + }); + + return [...messages.values()]; + } + }; +} diff --git a/media-scrape/slopbot/index.ts b/media-scrape/slopbot/index.ts index 46e60e1..32cfa63 100644 --- a/media-scrape/slopbot/index.ts +++ b/media-scrape/slopbot/index.ts @@ -8,10 +8,12 @@ import type { Client } from "discord.js"; import { loadSlopbotConfig, type SlopbotConfig } from "./config.js"; -/** Kept as a type so the discord.js dependency is type-checked before C4 uses it. */ +/** Kept as a type so the discord.js dependency stays type-checked. */ export type SlopbotClient = Client; export { loadSlopbotConfig, type SlopbotConfig }; +export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js"; +export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js"; /** View Channels + Read Message History. The bot reads only; it never sends. */ const PERMISSIONS = String((1 << 10) | (1 << 16)); diff --git a/media-scrape/test/slopbot.ts b/media-scrape/test/slopbot.ts new file mode 100644 index 0000000..ff6ecda --- /dev/null +++ b/media-scrape/test/slopbot.ts @@ -0,0 +1,169 @@ +/* + * Slopbot backend tests: discord.js message mapping and paging. Offline, fake fetcher. + * Run with: pnpm test:media (from poc/). + */ + +import assert from "node:assert/strict"; +import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js"; +import { mapSlopbotMessage } from "../slopbot/backend.js"; + +let failures = 0; + +async function test(name: string, fn: () => void | Promise): Promise { + try { + await fn(); + console.log(`ok - ${name}`); + } catch (error) { + failures++; + console.error(`not ok - ${name}`); + console.error(error); + } +} + +const noFlags = { has: () => false }; +const voiceFlags = { has: (bit: number) => (bit & (1 << 13)) !== 0 }; + +function discordMessage(extra: Record = {}) { + return { + id: "1000", + author: { id: "7", username: "user", avatar: "abc" }, + content: "hello", + createdAt: new Date("2026-09-16T12:00:00.000Z"), + editedAt: null, + reference: null, + flags: noFlags, + attachments: new Map(), + embeds: [], + stickers: new Map(), + messageSnapshots: new Map(), + ...extra + }; +} + +class FakeFetcher implements SlopbotFetcher { + readonly calls: Array<{ channelId: string; limit: number; before?: string }> = []; + + constructor(private readonly pages: unknown[][]) {} + + async fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise { + this.calls.push({ channelId, ...options }); + return this.pages.shift() ?? []; + } +} + +void (async () => { + await test("maps attachments, embeds, stickers, and replies", () => { + const mapped = mapSlopbotMessage( + discordMessage({ + reference: { messageId: "900" }, + attachments: new Map([ + [ + "a1", + { + id: "a1", + name: "pic.png", + url: "https://cdn.discordapp.com/attachments/1/pic.png", + contentType: "image/png", + size: 10, + width: 5, + height: 6, + duration: null + } + ] + ]), + embeds: [ + { + data: { + type: "video", + url: "https://www.youtube.com/watch?v=x", + provider: { name: "YouTube" }, + video: { url: "https://www.youtube.com/embed/x" }, + image: null, + thumbnail: { url: "https://i.ytimg.com/vi/x/hqdefault.jpg" } + } + } + ], + stickers: new Map([["s1", { id: "s1", name: "wave", format: 4 }]]) + }) + ); + + assert.equal(mapped.attachments[0].filename, "pic.png"); + assert.equal(mapped.attachments[0].contentType, "image/png"); + assert.equal(mapped.attachments[0].size, 10); + assert.equal(mapped.embeds[0].provider, "YouTube"); + assert.equal(mapped.embeds[0].videoUrl, "https://www.youtube.com/embed/x"); + assert.equal(mapped.stickerItems[0].formatType, 4); + assert.equal(mapped.replyTo?.id, "900"); + assert.equal(mapped.voiceMessage, false); + assert.equal(mapped.timestamp, "2026-09-16T12:00:00.000Z"); + }); + + await test("maps forwarded snapshots and voice flags", () => { + const mapped = mapSlopbotMessage( + discordMessage({ + flags: voiceFlags, + messageSnapshots: new Map([ + [ + "s1", + { + content: "forwarded", + createdAt: new Date("2026-09-15T00:00:00.000Z"), + flags: noFlags, + attachments: new Map([ + [ + "a2", + { + id: "a2", + name: "clip.webm", + url: "https://cdn.discordapp.com/attachments/1/clip.webm", + contentType: "video/webm", + size: null, + width: null, + height: null, + duration: null + } + ] + ]), + embeds: [], + stickers: new Map() + } + ] + ]) + }) + ); + + assert.equal(mapped.voiceMessage, true); + assert.equal(mapped.forwarded.length, 1); + assert.equal(mapped.forwarded[0].content, "forwarded"); + assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm"); + }); + + await test("backend pages oldest first and passes before and limit", async () => { + const fetcher = new FakeFetcher([ + [discordMessage({ id: "1002" }), discordMessage({ id: "1001" })], + [discordMessage({ id: "999" })] + ]); + const backend = new SlopbotBackend(fetcher); + + const page = await backend.fetchPage("123", null, 2); + assert.deepEqual( + page.messages.map(message => message.id), + ["1001", "1002"] + ); + assert.deepEqual(fetcher.calls[0], { channelId: "123", limit: 2 }); + + const next = await backend.fetchPage("123", "1001", 2); + assert.deepEqual( + next.messages.map(message => message.id), + ["999"] + ); + assert.deepEqual(fetcher.calls[1], { channelId: "123", limit: 2, before: "1001" }); + }); + + if (failures > 0) { + console.error(`${failures} test(s) failed`); + process.exit(1); + } + + console.log("all slopbot tests passed"); +})(); diff --git a/poc/package.json b/poc/package.json index 38fd96d..2b4e132 100644 --- a/poc/package.json +++ b/poc/package.json @@ -16,7 +16,7 @@ "cdp:dispatch": "tsx src/cdp-dispatch.ts", "test:cache": "tsx src/test-cache.ts", "test:audio": "tsx src/test-audio.ts", - "test:media": "tsx ../media-scrape/test/run.ts", + "test:media": "tsx ../media-scrape/test/run.ts && tsx ../media-scrape/test/slopbot.ts", "audio:doctor": "bash scripts/audio-doctor.sh" }, "dependencies": {