Add the slopbot scrape backend and client lifecycle

2026-09-16 08:04:48 +02:00
parent 4f953cefdc
commit 91782f0b80
8 changed files with 386 additions and 12 deletions
@@ -0,0 +1,131 @@
+/*
+ * Slopbot scrape backend: discord.js messages to the core message shape.
+ *
+ * The mapping is duck-typed so tests run without a bot token. The live shape was
+ * checked against the discord.js 14 typings; a live check is still owed at C8.
+ */
+
+import type { FetchPage, ScrapeBackend } from "../src/engine.js";
+import type { CoreAttachment, CoreEmbed, CoreMessage, CoreSticker, MediaMessage } from "../src/types.js";
+
+/** Discord message flag for a voice message. */
+const VOICE_MESSAGE_FLAG = 1 << 13;
+
+/** The minimal fetcher the backend needs. The real one wraps a discord.js client. */
+export interface SlopbotFetcher {
+    fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]>;
+}
+
+/** discord.js Collections are iterable and have `.values()`; tests use Maps and arrays. */
+function collectionValues(value: any): any[] {
+    if (!value) return [];
+    if (Array.isArray(value)) return value;
+    if (typeof value.values === "function") return [...value.values()];
+    return [];
+}
+
+function mapAttachments(message: any): CoreAttachment[] {
+    return collectionValues(message?.attachments).map((attachment: any) => ({
+        id: attachment?.id ?? null,
+        filename: attachment?.name ?? attachment?.filename ?? "",
+        url: attachment?.url ?? null,
+        contentType: attachment?.contentType ?? attachment?.content_type ?? null,
+        size: attachment?.size ?? null,
+        width: attachment?.width ?? null,
+        height: attachment?.height ?? null,
+        durationSecs: attachment?.duration ?? attachment?.durationSecs ?? attachment?.duration_secs ?? null
+    }));
+}
+
+function mapEmbeds(message: any): CoreEmbed[] {
+    const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
+
+    return collectionValues(message?.embeds).map((embed: any) => {
+        const data = embed?.data ?? embed ?? {};
+        return {
+            type: data.type ?? null,
+            url: data.url ?? null,
+            title: data.title ?? null,
+            provider: data.provider?.name ?? null,
+            videoUrl: assetUrl(data.video),
+            imageUrl: assetUrl(data.image),
+            thumbnailUrl: assetUrl(data.thumbnail)
+        };
+    });
+}
+
+function mapStickers(message: any): CoreSticker[] {
+    return collectionValues(message?.stickers).map((sticker: any) => ({
+        id: sticker?.id ?? null,
+        name: sticker?.name ?? null,
+        formatType: sticker?.format ?? null
+    }));
+}
+
+function isVoiceMessage(message: any): boolean {
+    const flags = message?.flags;
+    if (typeof flags?.has === "function") return flags.has(VOICE_MESSAGE_FLAG);
+    return Boolean(Number(flags ?? 0) & VOICE_MESSAGE_FLAG);
+}
+
+function createdAt(message: any): string | null {
+    if (message?.createdAt instanceof Date) return message.createdAt.toISOString();
+    if (typeof message?.timestamp === "string") return message.timestamp;
+    return null;
+}
+
+function mapSnapshot(snapshot: any): MediaMessage & { content: string; timestamp: string | null } {
+    const source = snapshot?.message ?? snapshot;
+
+    return {
+        content: source?.content ?? "",
+        timestamp: createdAt(source),
+        attachments: mapAttachments(source),
+        embeds: mapEmbeds(source),
+        stickerItems: mapStickers(source),
+        voiceMessage: isVoiceMessage(source)
+    };
+}
+
+/** Map one discord.js message to the core message shape. */
+export function mapSlopbotMessage(message: any): CoreMessage {
+    const reference = message?.reference;
+    const edited = message?.editedAt ?? message?.editedTimestamp ?? null;
+
+    return {
+        id: String(message?.id ?? ""),
+        author: {
+            id: String(message?.author?.id ?? ""),
+            username: message?.author?.username ?? "",
+            avatar: message?.author?.avatar ?? null
+        },
+        content: message?.content ?? "",
+        timestamp: createdAt(message) ?? "",
+        editedAt: edited instanceof Date ? edited.toISOString() : null,
+        replyTo: reference?.messageId
+            ? { id: String(reference.messageId), authorId: null, authorName: null }
+            : null,
+        attachments: mapAttachments(message),
+        embeds: mapEmbeds(message),
+        stickerItems: mapStickers(message),
+        forwarded: collectionValues(message?.messageSnapshots).map(mapSnapshot),
+        voiceMessage: isVoiceMessage(message)
+    };
+}
+
+export class SlopbotBackend implements ScrapeBackend {
+    readonly name = "slopbot" as const;
+
+    constructor(private readonly fetcher: SlopbotFetcher) {}
+
+    async fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage> {
+        const messages = await this.fetcher.fetchMessages(channelId, {
+            limit,
+            ...(before ? { before } : {})
+        });
+
+        const mapped = messages.map(mapSlopbotMessage).filter(message => message.id.length > 0);
+        mapped.sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
+        return { messages: mapped };
+    }
+}