Add the slopbot scrape backend and client lifecycle

CURRENT.md

@@ -27,7 +27,7 @@ mobile draft at 390px.
   - [x] C1 design docs, AGENTS/INDEX pointers, `.gitignore`
   - [x] C2 core message shape: embeds, forwards, attachment metadata
   - [x] C3 media-scrape store and puppet walker, tests
-  - [ ] C4 slopbot backend, tests
+  - [x] C4 slopbot backend, tests
   - [ ] C5 Web UI Media and Bot tabs, API routes
   - [ ] C6 storage limit, warning flow, clean modal, settings
   - [ ] C7 MCP service hooks

docs/slopbot.md

@@ -47,10 +47,16 @@ channels.
 
 ## Code
 
-- `media-scrape/slopbot/` holds the bot core: client lifecycle, the `ScrapeBackend` implementation,
-  and the invite helper.
-- discord.js is a dependency of the daemon package (`poc/package.json`), the same way the MCP SDK
-  is. The code stays in `media-scrape/`; the daemon only mounts it.
+| File | Role |
+| --- | --- |
+| `media-scrape/slopbot/config.ts` | Environment and state-file settings |
+| `media-scrape/slopbot/client.ts` | Login, status, and the live fetcher |
+| `media-scrape/slopbot/backend.ts` | discord.js messages to the core message shape |
+| `media-scrape/slopbot/index.ts` | Module surface |
+
+- discord.js lives in `media-scrape/package.json`; the folder is its own pnpm workspace root.
+- The mapping is duck-typed, so the tests run without a token. The live Discord shapes were checked
+  against the discord.js 14 typings, but a real login is still owed at C8.
 - The Bot tab shows connection state, the guilds slopbot is in, the active scrape, and the queue.
   It also renders the invite URL from the client ID.
 
@@ -69,6 +75,6 @@ Slopbot runs with the faster bot budget; the puppet backend uses the slower user
 
 ## Status
 
-Design approved 2026-09-16. Scaffold added 2026-09-16: `media-scrape/slopbot/` (config loader and
-invite helper) and the discord.js dependency in `media-scrape/package.json`. Unused until
-checkpoint C4 in [CURRENT.md](../CURRENT.md).
+Backend implemented 2026-09-16 (C4): mapping, paging, and client lifecycle, tested with a fake
+fetcher. Not wired into the daemon yet; that is C5. A live login with a real token is owed at C8.
+Checkpoints: [CURRENT.md](../CURRENT.md).

docs/testing.md

@@ -17,8 +17,9 @@ pnpm test:audio
 pnpm test:media
 ```
 
-The files are `poc/src/test-communication.ts`, `poc/src/test-cache.ts`, and
-`media-scrape/test/run.ts` (media archive). The tests do not need Discord or Vesktop.
+The files are `poc/src/test-communication.ts`, `poc/src/test-cache.ts`, and the media archive
+tests `media-scrape/test/run.ts` (engine) and `media-scrape/test/slopbot.ts` (bot mapping). The
+tests do not need Discord or Vesktop.
 
 ## Type checks
 

media-scrape/slopbot/backend.ts

@@ -0,0 +1,131 @@
+/*
+ * Slopbot scrape backend: discord.js messages to the core message shape.
+ *
+ * The mapping is duck-typed so tests run without a bot token. The live shape was
+ * checked against the discord.js 14 typings; a live check is still owed at C8.
+ */
+
+import type { FetchPage, ScrapeBackend } from "../src/engine.js";
+import type { CoreAttachment, CoreEmbed, CoreMessage, CoreSticker, MediaMessage } from "../src/types.js";
+
+/** Discord message flag for a voice message. */
+const VOICE_MESSAGE_FLAG = 1 << 13;
+
+/** The minimal fetcher the backend needs. The real one wraps a discord.js client. */
+export interface SlopbotFetcher {
+    fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]>;
+}
+
+/** discord.js Collections are iterable and have `.values()`; tests use Maps and arrays. */
+function collectionValues(value: any): any[] {
+    if (!value) return [];
+    if (Array.isArray(value)) return value;
+    if (typeof value.values === "function") return [...value.values()];
+    return [];
+}
+
+function mapAttachments(message: any): CoreAttachment[] {
+    return collectionValues(message?.attachments).map((attachment: any) => ({
+        id: attachment?.id ?? null,
+        filename: attachment?.name ?? attachment?.filename ?? "",
+        url: attachment?.url ?? null,
+        contentType: attachment?.contentType ?? attachment?.content_type ?? null,
+        size: attachment?.size ?? null,
+        width: attachment?.width ?? null,
+        height: attachment?.height ?? null,
+        durationSecs: attachment?.duration ?? attachment?.durationSecs ?? attachment?.duration_secs ?? null
+    }));
+}
+
+function mapEmbeds(message: any): CoreEmbed[] {
+    const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
+
+    return collectionValues(message?.embeds).map((embed: any) => {
+        const data = embed?.data ?? embed ?? {};
+        return {
+            type: data.type ?? null,
+            url: data.url ?? null,
+            title: data.title ?? null,
+            provider: data.provider?.name ?? null,
+            videoUrl: assetUrl(data.video),
+            imageUrl: assetUrl(data.image),
+            thumbnailUrl: assetUrl(data.thumbnail)
+        };
+    });
+}
+
+function mapStickers(message: any): CoreSticker[] {
+    return collectionValues(message?.stickers).map((sticker: any) => ({
+        id: sticker?.id ?? null,
+        name: sticker?.name ?? null,
+        formatType: sticker?.format ?? null
+    }));
+}
+
+function isVoiceMessage(message: any): boolean {
+    const flags = message?.flags;
+    if (typeof flags?.has === "function") return flags.has(VOICE_MESSAGE_FLAG);
+    return Boolean(Number(flags ?? 0) & VOICE_MESSAGE_FLAG);
+}
+
+function createdAt(message: any): string | null {
+    if (message?.createdAt instanceof Date) return message.createdAt.toISOString();
+    if (typeof message?.timestamp === "string") return message.timestamp;
+    return null;
+}
+
+function mapSnapshot(snapshot: any): MediaMessage & { content: string; timestamp: string | null } {
+    const source = snapshot?.message ?? snapshot;
+
+    return {
+        content: source?.content ?? "",
+        timestamp: createdAt(source),
+        attachments: mapAttachments(source),
+        embeds: mapEmbeds(source),
+        stickerItems: mapStickers(source),
+        voiceMessage: isVoiceMessage(source)
+    };
+}
+
+/** Map one discord.js message to the core message shape. */
+export function mapSlopbotMessage(message: any): CoreMessage {
+    const reference = message?.reference;
+    const edited = message?.editedAt ?? message?.editedTimestamp ?? null;
+
+    return {
+        id: String(message?.id ?? ""),
+        author: {
+            id: String(message?.author?.id ?? ""),
+            username: message?.author?.username ?? "",
+            avatar: message?.author?.avatar ?? null
+        },
+        content: message?.content ?? "",
+        timestamp: createdAt(message) ?? "",
+        editedAt: edited instanceof Date ? edited.toISOString() : null,
+        replyTo: reference?.messageId
+            ? { id: String(reference.messageId), authorId: null, authorName: null }
+            : null,
+        attachments: mapAttachments(message),
+        embeds: mapEmbeds(message),
+        stickerItems: mapStickers(message),
+        forwarded: collectionValues(message?.messageSnapshots).map(mapSnapshot),
+        voiceMessage: isVoiceMessage(message)
+    };
+}
+
+export class SlopbotBackend implements ScrapeBackend {
+    readonly name = "slopbot" as const;
+
+    constructor(private readonly fetcher: SlopbotFetcher) {}
+
+    async fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage> {
+        const messages = await this.fetcher.fetchMessages(channelId, {
+            limit,
+            ...(before ? { before } : {})
+        });
+
+        const mapped = messages.map(mapSlopbotMessage).filter(message => message.id.length > 0);
+        mapped.sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
+        return { messages: mapped };
+    }
+}

media-scrape/slopbot/client.ts

@@ -0,0 +1,65 @@
+/*
+ * Slopbot client lifecycle. Separate from the Vesktop core by design.
+ * Setup and security: docs/slopbot.md.
+ */
+
+import { Client, GatewayIntentBits } from "discord.js";
+import type { SlopbotConfig } from "./config.js";
+import type { SlopbotFetcher } from "./backend.js";
+
+export interface SlopbotStatus {
+    connected: boolean;
+    user: string | null;
+    guilds: string[];
+}
+
+/**
+ * Log in and return a ready client. The bot needs no message events, so it
+ * declares only the Guilds intent; REST history does not use the gateway.
+ */
+export async function startSlopbot(config: SlopbotConfig): Promise<Client> {
+    if (!config.enabled || !config.token) {
+        throw new Error("slopbot is disabled or has no token");
+    }
+
+    const client = new Client({ intents: [GatewayIntentBits.Guilds] });
+    await client.login(config.token);
+    return client;
+}
+
+export async function stopSlopbot(client: Client | null): Promise<void> {
+    if (!client) return;
+    await client.destroy();
+}
+
+/** The Bot tab reads this. Safe with a null client. */
+export function slopbotStatus(client: Client | null): SlopbotStatus {
+    if (!client || !client.isReady()) {
+        return { connected: false, user: null, guilds: [] };
+    }
+
+    return {
+        connected: true,
+        user: client.user?.tag ?? null,
+        guilds: [...client.guilds.cache.values()].map(guild => guild.name)
+    };
+}
+
+/** The ScrapeBackend fetcher over a live client. */
+export function slopbotFetcher(client: Client): SlopbotFetcher {
+    return {
+        async fetchMessages(channelId, options) {
+            const channel = await client.channels.fetch(channelId);
+            if (!channel || !channel.isTextBased()) {
+                throw new Error(`channel ${channelId} is not text based`);
+            }
+
+            const messages = await channel.messages.fetch({
+                limit: options.limit,
+                ...(options.before ? { before: options.before } : {})
+            });
+
+            return [...messages.values()];
+        }
+    };
+}

media-scrape/slopbot/index.ts

@@ -8,10 +8,12 @@
 import type { Client } from "discord.js";
 import { loadSlopbotConfig, type SlopbotConfig } from "./config.js";
 
-/** Kept as a type so the discord.js dependency is type-checked before C4 uses it. */
+/** Kept as a type so the discord.js dependency stays type-checked. */
 export type SlopbotClient = Client;
 
 export { loadSlopbotConfig, type SlopbotConfig };
+export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js";
+export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js";
 
 /** View Channels + Read Message History. The bot reads only; it never sends. */
 const PERMISSIONS = String((1 << 10) | (1 << 16));

media-scrape/test/slopbot.ts

@@ -0,0 +1,169 @@
+/*
+ * Slopbot backend tests: discord.js message mapping and paging. Offline, fake fetcher.
+ * Run with: pnpm test:media (from poc/).
+ */
+
+import assert from "node:assert/strict";
+import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js";
+import { mapSlopbotMessage } from "../slopbot/backend.js";
+
+let failures = 0;
+
+async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
+    try {
+        await fn();
+        console.log(`ok - ${name}`);
+    } catch (error) {
+        failures++;
+        console.error(`not ok - ${name}`);
+        console.error(error);
+    }
+}
+
+const noFlags = { has: () => false };
+const voiceFlags = { has: (bit: number) => (bit & (1 << 13)) !== 0 };
+
+function discordMessage(extra: Record<string, unknown> = {}) {
+    return {
+        id: "1000",
+        author: { id: "7", username: "user", avatar: "abc" },
+        content: "hello",
+        createdAt: new Date("2026-09-16T12:00:00.000Z"),
+        editedAt: null,
+        reference: null,
+        flags: noFlags,
+        attachments: new Map(),
+        embeds: [],
+        stickers: new Map(),
+        messageSnapshots: new Map(),
+        ...extra
+    };
+}
+
+class FakeFetcher implements SlopbotFetcher {
+    readonly calls: Array<{ channelId: string; limit: number; before?: string }> = [];
+
+    constructor(private readonly pages: unknown[][]) {}
+
+    async fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]> {
+        this.calls.push({ channelId, ...options });
+        return this.pages.shift() ?? [];
+    }
+}
+
+void (async () => {
+    await test("maps attachments, embeds, stickers, and replies", () => {
+        const mapped = mapSlopbotMessage(
+            discordMessage({
+                reference: { messageId: "900" },
+                attachments: new Map([
+                    [
+                        "a1",
+                        {
+                            id: "a1",
+                            name: "pic.png",
+                            url: "https://cdn.discordapp.com/attachments/1/pic.png",
+                            contentType: "image/png",
+                            size: 10,
+                            width: 5,
+                            height: 6,
+                            duration: null
+                        }
+                    ]
+                ]),
+                embeds: [
+                    {
+                        data: {
+                            type: "video",
+                            url: "https://www.youtube.com/watch?v=x",
+                            provider: { name: "YouTube" },
+                            video: { url: "https://www.youtube.com/embed/x" },
+                            image: null,
+                            thumbnail: { url: "https://i.ytimg.com/vi/x/hqdefault.jpg" }
+                        }
+                    }
+                ],
+                stickers: new Map([["s1", { id: "s1", name: "wave", format: 4 }]])
+            })
+        );
+
+        assert.equal(mapped.attachments[0].filename, "pic.png");
+        assert.equal(mapped.attachments[0].contentType, "image/png");
+        assert.equal(mapped.attachments[0].size, 10);
+        assert.equal(mapped.embeds[0].provider, "YouTube");
+        assert.equal(mapped.embeds[0].videoUrl, "https://www.youtube.com/embed/x");
+        assert.equal(mapped.stickerItems[0].formatType, 4);
+        assert.equal(mapped.replyTo?.id, "900");
+        assert.equal(mapped.voiceMessage, false);
+        assert.equal(mapped.timestamp, "2026-09-16T12:00:00.000Z");
+    });
+
+    await test("maps forwarded snapshots and voice flags", () => {
+        const mapped = mapSlopbotMessage(
+            discordMessage({
+                flags: voiceFlags,
+                messageSnapshots: new Map([
+                    [
+                        "s1",
+                        {
+                            content: "forwarded",
+                            createdAt: new Date("2026-09-15T00:00:00.000Z"),
+                            flags: noFlags,
+                            attachments: new Map([
+                                [
+                                    "a2",
+                                    {
+                                        id: "a2",
+                                        name: "clip.webm",
+                                        url: "https://cdn.discordapp.com/attachments/1/clip.webm",
+                                        contentType: "video/webm",
+                                        size: null,
+                                        width: null,
+                                        height: null,
+                                        duration: null
+                                    }
+                                ]
+                            ]),
+                            embeds: [],
+                            stickers: new Map()
+                        }
+                    ]
+                ])
+            })
+        );
+
+        assert.equal(mapped.voiceMessage, true);
+        assert.equal(mapped.forwarded.length, 1);
+        assert.equal(mapped.forwarded[0].content, "forwarded");
+        assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm");
+    });
+
+    await test("backend pages oldest first and passes before and limit", async () => {
+        const fetcher = new FakeFetcher([
+            [discordMessage({ id: "1002" }), discordMessage({ id: "1001" })],
+            [discordMessage({ id: "999" })]
+        ]);
+        const backend = new SlopbotBackend(fetcher);
+
+        const page = await backend.fetchPage("123", null, 2);
+        assert.deepEqual(
+            page.messages.map(message => message.id),
+            ["1001", "1002"]
+        );
+        assert.deepEqual(fetcher.calls[0], { channelId: "123", limit: 2 });
+
+        const next = await backend.fetchPage("123", "1001", 2);
+        assert.deepEqual(
+            next.messages.map(message => message.id),
+            ["999"]
+        );
+        assert.deepEqual(fetcher.calls[1], { channelId: "123", limit: 2, before: "1001" });
+    });
+
+    if (failures > 0) {
+        console.error(`${failures} test(s) failed`);
+        process.exit(1);
+    }
+
+    console.log("all slopbot tests passed");
+})();

poc/package.json

@@ -16,7 +16,7 @@
         "cdp:dispatch": "tsx src/cdp-dispatch.ts",
         "test:cache": "tsx src/test-cache.ts",
         "test:audio": "tsx src/test-audio.ts",
-        "test:media": "tsx ../media-scrape/test/run.ts",
+        "test:media": "tsx ../media-scrape/test/run.ts && tsx ../media-scrape/test/slopbot.ts",
         "audio:doctor": "bash scripts/audio-doctor.sh"
     },
     "dependencies": {