Add the slopbot scrape backend and client lifecycle
This commit is contained in:
Fisto
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Slopbot scrape backend: discord.js messages to the core message shape.
|
||||
*
|
||||
* The mapping is duck-typed so tests run without a bot token. The live shape was
|
||||
* checked against the discord.js 14 typings; a live check is still owed at C8.
|
||||
*/
|
||||
|
||||
import type { FetchPage, ScrapeBackend } from "../src/engine.js";
|
||||
import type { CoreAttachment, CoreEmbed, CoreMessage, CoreSticker, MediaMessage } from "../src/types.js";
|
||||
|
||||
/** Discord message flag for a voice message. */
|
||||
const VOICE_MESSAGE_FLAG = 1 << 13;
|
||||
|
||||
/** The minimal fetcher the backend needs. The real one wraps a discord.js client. */
|
||||
export interface SlopbotFetcher {
|
||||
fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]>;
|
||||
}
|
||||
|
||||
/** discord.js Collections are iterable and have `.values()`; tests use Maps and arrays. */
|
||||
function collectionValues(value: any): any[] {
|
||||
if (!value) return [];
|
||||
if (Array.isArray(value)) return value;
|
||||
if (typeof value.values === "function") return [...value.values()];
|
||||
return [];
|
||||
}
|
||||
|
||||
function mapAttachments(message: any): CoreAttachment[] {
|
||||
return collectionValues(message?.attachments).map((attachment: any) => ({
|
||||
id: attachment?.id ?? null,
|
||||
filename: attachment?.name ?? attachment?.filename ?? "",
|
||||
url: attachment?.url ?? null,
|
||||
contentType: attachment?.contentType ?? attachment?.content_type ?? null,
|
||||
size: attachment?.size ?? null,
|
||||
width: attachment?.width ?? null,
|
||||
height: attachment?.height ?? null,
|
||||
durationSecs: attachment?.duration ?? attachment?.durationSecs ?? attachment?.duration_secs ?? null
|
||||
}));
|
||||
}
|
||||
|
||||
function mapEmbeds(message: any): CoreEmbed[] {
|
||||
const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
|
||||
|
||||
return collectionValues(message?.embeds).map((embed: any) => {
|
||||
const data = embed?.data ?? embed ?? {};
|
||||
return {
|
||||
type: data.type ?? null,
|
||||
url: data.url ?? null,
|
||||
title: data.title ?? null,
|
||||
provider: data.provider?.name ?? null,
|
||||
videoUrl: assetUrl(data.video),
|
||||
imageUrl: assetUrl(data.image),
|
||||
thumbnailUrl: assetUrl(data.thumbnail)
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function mapStickers(message: any): CoreSticker[] {
|
||||
return collectionValues(message?.stickers).map((sticker: any) => ({
|
||||
id: sticker?.id ?? null,
|
||||
name: sticker?.name ?? null,
|
||||
formatType: sticker?.format ?? null
|
||||
}));
|
||||
}
|
||||
|
||||
function isVoiceMessage(message: any): boolean {
|
||||
const flags = message?.flags;
|
||||
if (typeof flags?.has === "function") return flags.has(VOICE_MESSAGE_FLAG);
|
||||
return Boolean(Number(flags ?? 0) & VOICE_MESSAGE_FLAG);
|
||||
}
|
||||
|
||||
function createdAt(message: any): string | null {
|
||||
if (message?.createdAt instanceof Date) return message.createdAt.toISOString();
|
||||
if (typeof message?.timestamp === "string") return message.timestamp;
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapSnapshot(snapshot: any): MediaMessage & { content: string; timestamp: string | null } {
|
||||
const source = snapshot?.message ?? snapshot;
|
||||
|
||||
return {
|
||||
content: source?.content ?? "",
|
||||
timestamp: createdAt(source),
|
||||
attachments: mapAttachments(source),
|
||||
embeds: mapEmbeds(source),
|
||||
stickerItems: mapStickers(source),
|
||||
voiceMessage: isVoiceMessage(source)
|
||||
};
|
||||
}
|
||||
|
||||
/** Map one discord.js message to the core message shape. */
|
||||
export function mapSlopbotMessage(message: any): CoreMessage {
|
||||
const reference = message?.reference;
|
||||
const edited = message?.editedAt ?? message?.editedTimestamp ?? null;
|
||||
|
||||
return {
|
||||
id: String(message?.id ?? ""),
|
||||
author: {
|
||||
id: String(message?.author?.id ?? ""),
|
||||
username: message?.author?.username ?? "",
|
||||
avatar: message?.author?.avatar ?? null
|
||||
},
|
||||
content: message?.content ?? "",
|
||||
timestamp: createdAt(message) ?? "",
|
||||
editedAt: edited instanceof Date ? edited.toISOString() : null,
|
||||
replyTo: reference?.messageId
|
||||
? { id: String(reference.messageId), authorId: null, authorName: null }
|
||||
: null,
|
||||
attachments: mapAttachments(message),
|
||||
embeds: mapEmbeds(message),
|
||||
stickerItems: mapStickers(message),
|
||||
forwarded: collectionValues(message?.messageSnapshots).map(mapSnapshot),
|
||||
voiceMessage: isVoiceMessage(message)
|
||||
};
|
||||
}
|
||||
|
||||
export class SlopbotBackend implements ScrapeBackend {
|
||||
readonly name = "slopbot" as const;
|
||||
|
||||
constructor(private readonly fetcher: SlopbotFetcher) {}
|
||||
|
||||
async fetchPage(channelId: string, before: string | null, limit: number): Promise<FetchPage> {
|
||||
const messages = await this.fetcher.fetchMessages(channelId, {
|
||||
limit,
|
||||
...(before ? { before } : {})
|
||||
});
|
||||
|
||||
const mapped = messages.map(mapSlopbotMessage).filter(message => message.id.length > 0);
|
||||
mapped.sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
|
||||
return { messages: mapped };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Slopbot client lifecycle. Separate from the Vesktop core by design.
|
||||
* Setup and security: docs/slopbot.md.
|
||||
*/
|
||||
|
||||
import { Client, GatewayIntentBits } from "discord.js";
|
||||
import type { SlopbotConfig } from "./config.js";
|
||||
import type { SlopbotFetcher } from "./backend.js";
|
||||
|
||||
export interface SlopbotStatus {
|
||||
connected: boolean;
|
||||
user: string | null;
|
||||
guilds: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Log in and return a ready client. The bot needs no message events, so it
|
||||
* declares only the Guilds intent; REST history does not use the gateway.
|
||||
*/
|
||||
export async function startSlopbot(config: SlopbotConfig): Promise<Client> {
|
||||
if (!config.enabled || !config.token) {
|
||||
throw new Error("slopbot is disabled or has no token");
|
||||
}
|
||||
|
||||
const client = new Client({ intents: [GatewayIntentBits.Guilds] });
|
||||
await client.login(config.token);
|
||||
return client;
|
||||
}
|
||||
|
||||
export async function stopSlopbot(client: Client | null): Promise<void> {
|
||||
if (!client) return;
|
||||
await client.destroy();
|
||||
}
|
||||
|
||||
/** The Bot tab reads this. Safe with a null client. */
|
||||
export function slopbotStatus(client: Client | null): SlopbotStatus {
|
||||
if (!client || !client.isReady()) {
|
||||
return { connected: false, user: null, guilds: [] };
|
||||
}
|
||||
|
||||
return {
|
||||
connected: true,
|
||||
user: client.user?.tag ?? null,
|
||||
guilds: [...client.guilds.cache.values()].map(guild => guild.name)
|
||||
};
|
||||
}
|
||||
|
||||
/** The ScrapeBackend fetcher over a live client. */
|
||||
export function slopbotFetcher(client: Client): SlopbotFetcher {
|
||||
return {
|
||||
async fetchMessages(channelId, options) {
|
||||
const channel = await client.channels.fetch(channelId);
|
||||
if (!channel || !channel.isTextBased()) {
|
||||
throw new Error(`channel ${channelId} is not text based`);
|
||||
}
|
||||
|
||||
const messages = await channel.messages.fetch({
|
||||
limit: options.limit,
|
||||
...(options.before ? { before: options.before } : {})
|
||||
});
|
||||
|
||||
return [...messages.values()];
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -8,10 +8,12 @@
|
||||
import type { Client } from "discord.js";
|
||||
import { loadSlopbotConfig, type SlopbotConfig } from "./config.js";
|
||||
|
||||
/** Kept as a type so the discord.js dependency is type-checked before C4 uses it. */
|
||||
/** Kept as a type so the discord.js dependency stays type-checked. */
|
||||
export type SlopbotClient = Client;
|
||||
|
||||
export { loadSlopbotConfig, type SlopbotConfig };
|
||||
export { mapSlopbotMessage, SlopbotBackend, type SlopbotFetcher } from "./backend.js";
|
||||
export { slopbotFetcher, slopbotStatus, startSlopbot, stopSlopbot, type SlopbotStatus } from "./client.js";
|
||||
|
||||
/** View Channels + Read Message History. The bot reads only; it never sends. */
|
||||
const PERMISSIONS = String((1 << 10) | (1 << 16));
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
* Slopbot backend tests: discord.js message mapping and paging. Offline, fake fetcher.
|
||||
* Run with: pnpm test:media (from poc/).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { SlopbotBackend, type SlopbotFetcher } from "../slopbot/backend.js";
|
||||
import { mapSlopbotMessage } from "../slopbot/backend.js";
|
||||
|
||||
let failures = 0;
|
||||
|
||||
async function test(name: string, fn: () => void | Promise<void>): Promise<void> {
|
||||
try {
|
||||
await fn();
|
||||
console.log(`ok - ${name}`);
|
||||
} catch (error) {
|
||||
failures++;
|
||||
console.error(`not ok - ${name}`);
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
const noFlags = { has: () => false };
|
||||
const voiceFlags = { has: (bit: number) => (bit & (1 << 13)) !== 0 };
|
||||
|
||||
function discordMessage(extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: "1000",
|
||||
author: { id: "7", username: "user", avatar: "abc" },
|
||||
content: "hello",
|
||||
createdAt: new Date("2026-09-16T12:00:00.000Z"),
|
||||
editedAt: null,
|
||||
reference: null,
|
||||
flags: noFlags,
|
||||
attachments: new Map(),
|
||||
embeds: [],
|
||||
stickers: new Map(),
|
||||
messageSnapshots: new Map(),
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
class FakeFetcher implements SlopbotFetcher {
|
||||
readonly calls: Array<{ channelId: string; limit: number; before?: string }> = [];
|
||||
|
||||
constructor(private readonly pages: unknown[][]) {}
|
||||
|
||||
async fetchMessages(channelId: string, options: { limit: number; before?: string }): Promise<unknown[]> {
|
||||
this.calls.push({ channelId, ...options });
|
||||
return this.pages.shift() ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
await test("maps attachments, embeds, stickers, and replies", () => {
|
||||
const mapped = mapSlopbotMessage(
|
||||
discordMessage({
|
||||
reference: { messageId: "900" },
|
||||
attachments: new Map([
|
||||
[
|
||||
"a1",
|
||||
{
|
||||
id: "a1",
|
||||
name: "pic.png",
|
||||
url: "https://cdn.discordapp.com/attachments/1/pic.png",
|
||||
contentType: "image/png",
|
||||
size: 10,
|
||||
width: 5,
|
||||
height: 6,
|
||||
duration: null
|
||||
}
|
||||
]
|
||||
]),
|
||||
embeds: [
|
||||
{
|
||||
data: {
|
||||
type: "video",
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
provider: { name: "YouTube" },
|
||||
video: { url: "https://www.youtube.com/embed/x" },
|
||||
image: null,
|
||||
thumbnail: { url: "https://i.ytimg.com/vi/x/hqdefault.jpg" }
|
||||
}
|
||||
}
|
||||
],
|
||||
stickers: new Map([["s1", { id: "s1", name: "wave", format: 4 }]])
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(mapped.attachments[0].filename, "pic.png");
|
||||
assert.equal(mapped.attachments[0].contentType, "image/png");
|
||||
assert.equal(mapped.attachments[0].size, 10);
|
||||
assert.equal(mapped.embeds[0].provider, "YouTube");
|
||||
assert.equal(mapped.embeds[0].videoUrl, "https://www.youtube.com/embed/x");
|
||||
assert.equal(mapped.stickerItems[0].formatType, 4);
|
||||
assert.equal(mapped.replyTo?.id, "900");
|
||||
assert.equal(mapped.voiceMessage, false);
|
||||
assert.equal(mapped.timestamp, "2026-09-16T12:00:00.000Z");
|
||||
});
|
||||
|
||||
await test("maps forwarded snapshots and voice flags", () => {
|
||||
const mapped = mapSlopbotMessage(
|
||||
discordMessage({
|
||||
flags: voiceFlags,
|
||||
messageSnapshots: new Map([
|
||||
[
|
||||
"s1",
|
||||
{
|
||||
content: "forwarded",
|
||||
createdAt: new Date("2026-09-15T00:00:00.000Z"),
|
||||
flags: noFlags,
|
||||
attachments: new Map([
|
||||
[
|
||||
"a2",
|
||||
{
|
||||
id: "a2",
|
||||
name: "clip.webm",
|
||||
url: "https://cdn.discordapp.com/attachments/1/clip.webm",
|
||||
contentType: "video/webm",
|
||||
size: null,
|
||||
width: null,
|
||||
height: null,
|
||||
duration: null
|
||||
}
|
||||
]
|
||||
]),
|
||||
embeds: [],
|
||||
stickers: new Map()
|
||||
}
|
||||
]
|
||||
])
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(mapped.voiceMessage, true);
|
||||
assert.equal(mapped.forwarded.length, 1);
|
||||
assert.equal(mapped.forwarded[0].content, "forwarded");
|
||||
assert.equal(mapped.forwarded[0].attachments[0].filename, "clip.webm");
|
||||
});
|
||||
|
||||
await test("backend pages oldest first and passes before and limit", async () => {
|
||||
const fetcher = new FakeFetcher([
|
||||
[discordMessage({ id: "1002" }), discordMessage({ id: "1001" })],
|
||||
[discordMessage({ id: "999" })]
|
||||
]);
|
||||
const backend = new SlopbotBackend(fetcher);
|
||||
|
||||
const page = await backend.fetchPage("123", null, 2);
|
||||
assert.deepEqual(
|
||||
page.messages.map(message => message.id),
|
||||
["1001", "1002"]
|
||||
);
|
||||
assert.deepEqual(fetcher.calls[0], { channelId: "123", limit: 2 });
|
||||
|
||||
const next = await backend.fetchPage("123", "1001", 2);
|
||||
assert.deepEqual(
|
||||
next.messages.map(message => message.id),
|
||||
["999"]
|
||||
);
|
||||
assert.deepEqual(fetcher.calls[1], { channelId: "123", limit: 2, before: "1001" });
|
||||
});
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`${failures} test(s) failed`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("all slopbot tests passed");
|
||||
})();
|
||||
Reference in New Issue
Block a user