Add the slopbot scrape backend and client lifecycle

media-scrape/slopbot/backend.ts

@@ -0,0 +1,131 @@
+/*
+ * Slopbot scrape backend: discord.js messages to the core message shape.
+ *
+ * The mapping is duck-typed so tests run without a bot token. The live shape was
+ * checked against the discord.js 14 typings; a live check is still owed at C8.
+ */
+
+import type { FetchPage, ScrapeBackend } from "../src/engine.js";
+import type { CoreAttachment, CoreEmbed, CoreMessage, CoreSticker, MediaMessage } from "../src/types.js";
+
+/** Discord message flag for a voice message. */
+const VOICE_MESSAGE_FLAG = 1 << 13;
+
+/** The minimal fetcher the backend needs. The real one wraps a discord.js client. */
+export interface SlopbotFetcher {
+    fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]>;
+}
+
+/** discord.js Collections are iterable and have `.values()`; tests use Maps and arrays. */
+function collectionValues(value: any): any[] {
+    if (!value) return [];
+    if (Array.isArray(value)) return value;
+    if (typeof value.values === "function") return [...value.values()];
+    return [];
+}
+
+function mapAttachments(message: any): CoreAttachment[] {
+    return collectionValues(message?.attachments).map((attachment: any) => ({
+        id: attachment?.id ?? null,
+        filename: attachment?.name ?? attachment?.filename ?? "",
+        url: attachment?.url ?? null,
+        contentType: attachment?.contentType ?? attachment?.content_type ?? null,
+        size: attachment?.size ?? null,
+        width: attachment?.width ?? null,
+        height: attachment?.height ?? null,
+        durationSecs: attachment?.duration ?? attachment?.durationSecs ?? attachment?.duration_secs ?? null
+    }));
+}
+
+function mapEmbeds(message: any): CoreEmbed[] {
+    const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
+
+    return collectionValues(message?.embeds).map((embed: any) => {
+        const data = embed?.data ?? embed ?? {};
+        return {
+            type: data.type ?? null,
+            url: data.url ?? null,
+            title: data.title ?? null,
+            provider: data.provider?.name ?? null,
+            videoUrl: assetUrl(data.video),
+            imageUrl: assetUrl(data.image),
+            thumbnailUrl: assetUrl(data.thumbnail)
+        };
+    });
+}
+
+function mapStickers(message: any): CoreSticker[] {
+    return collectionValues(message?.stickers).map((sticker: any) => ({
+        id: sticker?.id ?? null,
+        name: sticker?.name ?? null,
+        formatType: sticker?.format ?? null
+    }));
+}
+
+function isVoiceMessage(message: any): boolean {
+    const flags = message?.flags;
+    if (typeof flags?.has === "function") return flags.has(VOICE_MESSAGE_FLAG);
+    return Boolean(Number(flags ?? 0) & VOICE_MESSAGE_FLAG);
+}
+
+function createdAt(message: any): string | null {
+    if (message?.createdAt instanceof Date) return message.createdAt.toISOString();
+    if (typeof message?.timestamp === "string") return message.timestamp;
+    return null;
+}
+
+function mapSnapshot(snapshot: any): MediaMessage & { content: string; timestamp: string | null } {
+    const source = snapshot?.message ?? snapshot;
+
+    return {
+        content: source?.content ?? "",
+        timestamp: createdAt(source),
+        attachments: mapAttachments(source),
+        embeds: mapEmbeds(source),
+        stickerItems: mapStickers(source),
+        voiceMessage: isVoiceMessage(source)
+    };
+}
+
+/** Map one discord.js message to the core message shape. */
+export function mapSlopbotMessage(message: any): CoreMessage {
+    const reference = message?.reference;
+    const edited = message?.editedAt ?? message?.editedTimestamp ?? null;
+
+    return {
+        id: String(message?.id ?? ""),
+        author: {
+            id: String(message?.author?.id ?? ""),
+            username: message?.author?.username ?? "",
+            avatar: message?.author?.avatar ?? null
+        },
+        content: message?.content ?? "",
+        timestamp: createdAt(message) ?? "",
+        editedAt: edited instanceof Date ? edited.toISOString() : null,
+        replyTo: reference?.messageId
+            ? { id: String(reference.messageId), authorId: null, authorName: null }
+            : null,
+        attachments: mapAttachments(message),
+        embeds: mapEmbeds(message),
+        stickerItems: mapStickers(message),
+        forwarded: collectionValues(message?.messageSnapshots).map(mapSnapshot),
+        voiceMessage: isVoiceMessage(message)
+    };
+}
+
+export class SlopbotBackend implements ScrapeBackend {
+    readonly name = "slopbot" as const;
+
+    constructor(private readonly fetcher: SlopbotFetcher) {}
+
+    async fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage> {
+        const messages = await this.fetcher.fetchMessages(channelId, {
+            limit,
+            ...(before ? { before } : {})
+        });
+
+        const mapped = messages.map(mapSlopbotMessage).filter(message => message.id.length > 0);
+        mapped.sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
+        return { messages: mapped };
+    }
+}

media-scrape/slopbot/client.ts

@@ -0,0 +1,65 @@
+/*
+ * Slopbot client lifecycle. Separate from the Vesktop core by design.
+ * Setup and security: docs/slopbot.md.
+ */
+
+import { Client, GatewayIntentBits } from "discord.js";
+import type { SlopbotConfig } from "./config.js";
+import type { SlopbotFetcher } from "./backend.js";
+
+export interface SlopbotStatus {
+    connected: boolean;
+    user: string | null;
+    guilds: string[];
+}
+
+/**
+ * Log in and return a ready client. The bot needs no message events, so it
+ * declares only the Guilds intent; REST history does not use the gateway.
+ */
+export async function startSlopbot(config: SlopbotConfig): Promise<Client> {
+    if (!config.enabled || !config.token) {
+        throw new Error("slopbot is disabled or has no token");
+    }
+
+    const client = new Client({ intents: [GatewayIntentBits.Guilds] });
+    await client.login(config.token);
+    return client;
+}
+
+export async function stopSlopbot(client: Client | null): Promise<void> {
+    if (!client) return;
+    await client.destroy();
+}
+
+/** The Bot tab reads this. Safe with a null client. */
+export function slopbotStatus(client: Client | null): SlopbotStatus {
+    if (!client || !client.isReady()) {
+        return { connected: false, user: null, guilds: [] };
+    }
+
+    return {
+        connected: true,
+        user: client.user?.tag ?? null,
+        guilds: [...client.guilds.cache.values()].map(guild => guild.name)
+    };
+}
+
+/** The ScrapeBackend fetcher over a live client. */
+export function slopbotFetcher(client: Client): SlopbotFetcher {
+    return {
+        async fetchMessages(channelId, options) {
+            const channel = await client.channels.fetch(channelId);
+            if (!channel || !channel.isTextBased()) {
+                throw new Error(`channel ${channelId} is not text based`);
+            }
+
+            const messages = await channel.messages.fetch({
+                limit: options.limit,
+                ...(options.before ? { before: options.before } : {})
+            });
+
+            return [...messages.values()];
+        }
+    };
+}

media-scrape/slopbot/index.ts

@@ -8,10 +8,12 @@
 import type { Client } from "discord.js";
 import { loadSlopbotConfig, type SlopbotConfig } from "./config.js";
 
-/** Kept as a type so the discord.js dependency is type-checked before C4 uses it. */
+/** Kept as a type so the discord.js dependency stays type-checked. */
 export type SlopbotClient = Client;
 
 export { loadSlopbotConfig, type SlopbotConfig };
+export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js";
+export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js";
 
 /** View Channels + Read Message History. The bot reads only; it never sends. */
 const PERMISSIONS = String((1 << 10) | (1 << 16));