Add the agent scaffold, mobile web UI, voice auto-leave, and text notifications

2026-09-14 16:38:14 +02:00
parent c7aad7547f
commit eab94132bb
27 changed files with 1521 additions and 73 deletions
@@ -51,6 +51,7 @@ export interface PuppetConfig {
    fetchCooldownSeconds: number;
    continuationSeconds: number;
    pollSeconds: number;
+    voiceLeaveDelaySeconds: number;
    responsePolicy: ResponsePolicy;
 }

@@ -60,6 +61,7 @@ export interface PuppetConfigPatch {
    fetchCooldownSeconds?: number;
    continuationSeconds?: number;
    pollSeconds?: number;
+    voiceLeaveDelaySeconds?: number;
    responsePolicy?: {
        defaultMode?: "channel" | "dm";
        sensitiveCommands?: string[];
@@ -73,6 +75,7 @@ const DEFAULT_CONFIG: PuppetConfig = {
    fetchCooldownSeconds: 5,
    continuationSeconds: 10,
    pollSeconds: 120,
+    voiceLeaveDelaySeconds: clamp(Number(process.env.VESKSLOP_VOICE_LEAVE_DELAY ?? 30), 0, 600),
    responsePolicy: {
        defaultMode: "channel",
        sensitiveCommands: ["whitelist", "config"],
@@ -97,6 +100,8 @@ export interface PuppetTransport {
        options?: { before?: string; after?: string }
    ): Promise<{ messages?: unknown[] }>;
    reactionAdd(channelId: string, messageId: string, emoji: string): Promise<unknown>;
+    voiceGet(userId?: string, guildId?: string): Promise<{ channelId?: string | null } | void>;
+    voiceLeave(): Promise<unknown>;
 }

 /** The audio actions a chat command can trigger. */
@@ -143,6 +148,10 @@ export class Puppet {
    private readonly pending = new Map<string, { entryId: string; expiresAt: number }>();
    private readonly polls = new Map<string, PollState>();

+    private voiceChannelId?: string | null;
+    private ownerVoiceChannelId?: string | null;
+    private voiceLeaveTimer?: ReturnType<typeof setTimeout>;
+
    private ownerId = DEFAULT_OWNER_ID;
    private config: PuppetConfig = structuredClone(DEFAULT_CONFIG);
    private selfId?: string;
@@ -180,6 +189,11 @@ export class Puppet {
                        300
                    ),
                    pollSeconds: clamp(state.config.pollSeconds ?? DEFAULT_CONFIG.pollSeconds, 10, 3600),
+                    voiceLeaveDelaySeconds: clamp(
+                        state.config.voiceLeaveDelaySeconds ?? DEFAULT_CONFIG.voiceLeaveDelaySeconds,
+                        0,
+                        600
+                    ),
                    responsePolicy: {
                        defaultMode: state.config.responsePolicy?.defaultMode ?? DEFAULT_CONFIG.responsePolicy.defaultMode,
                        sensitiveCommands:
@@ -228,6 +242,11 @@ export class Puppet {
            ),
            continuationSeconds: clamp(patch.continuationSeconds ?? this.config.continuationSeconds, 0, 300),
            pollSeconds: clamp(patch.pollSeconds ?? this.config.pollSeconds, 10, 3600),
+            voiceLeaveDelaySeconds: clamp(
+                patch.voiceLeaveDelaySeconds ?? this.config.voiceLeaveDelaySeconds,
+                0,
+                600
+            ),
            responsePolicy: {
                defaultMode: policy.defaultMode ?? this.config.responsePolicy.defaultMode,
                sensitiveCommands: policy.sensitiveCommands ?? this.config.responsePolicy.sensitiveCommands,
@@ -360,6 +379,28 @@ export class Puppet {
            return;
        }

+        if (event?.type === "MESSAGE_UPDATE") {
+            this.handleMessageUpdate(event);
+            return;
+        }
+
+        if (event?.type === "MESSAGE_DELETE") {
+            this.handleMessageDelete(event);
+            return;
+        }
+
+        if (event?.type === "MESSAGE_DELETE_BULK") {
+            for (const id of event.ids ?? []) {
+                this.handleMessageDelete({ id, channelId: event.channelId, guildId: event.guildId });
+            }
+            return;
+        }
+
+        if (event?.type === "VOICE_STATE_UPDATE") {
+            await this.handleVoiceStateUpdate(event);
+            return;
+        }
+
        if (event?.type !== "MESSAGE_CREATE") return;

        const { author, channelId, guildId, channelType, content, mentions } = event;
@@ -381,7 +422,15 @@ export class Puppet {
        const isGroupDm = channelType === GROUP_DM_CHANNEL;
        const isDirectDm = !guildId && (channelType === DM_CHANNEL || (!isGroupDm && channelType === null));
        const mentioned = new Set<string>(mentions ?? []).has(this.selfId);
-        const isTrigger = isDirectDm || mentioned;
+
+        // A reply to a message the puppet sent counts as a trigger too.
+        const replyTo = event.replyTo as
+            | { id?: string; authorId?: string | null; authorName?: string | null; content?: string }
+            | null
+            | undefined;
+        const repliesToPuppet = !!replyTo?.authorId && replyTo.authorId === this.selfId;
+
+        const isTrigger = isDirectDm || mentioned || repliesToPuppet;

        const key = `${channelId}:${author.id}`;

@@ -437,6 +486,21 @@ export class Puppet {
            }
        }

+        // Keep the referenced message in the request context.
+        if (replyTo?.id && replyTo.authorId) {
+            const referenced: CachedMessage = {
+                id: replyTo.id,
+                authorId: replyTo.authorId,
+                authorName: replyTo.authorName ?? replyTo.authorId,
+                content: replyTo.content ?? "",
+                timestamp: ""
+            };
+
+            if (!entry.context?.some(message => message.id === referenced.id)) {
+                entry.context = [...(entry.context ?? []), referenced];
+            }
+        }
+
        this.log.add(entry);

        if (this.config.continuationSeconds > 0) {
@@ -454,6 +518,7 @@ export class Puppet {
            authorId: author.id,
            tier,
            command: entry.command,
+            replyTo: replyTo?.id ?? null,
            contextCount: entry.context?.length ?? 0
        });

@@ -675,13 +740,161 @@ export class Puppet {
        ].join("\n");
    }

-    private async handleReactionEvent(event: any) {
-        const poll = this.polls.get(event.messageId);
-        if (!poll) return;
+    private handleMessageUpdate(event: any) {
+        const cached = this.cache.edit(event.channelId, event.id, event.content ?? "");
+
+        this.actionLog.append({
+            type: "message-edit",
+            messageId: event.id,
+            channelId: event.channelId,
+            guildId: event.guildId ?? null,
+            authorId: event.author?.id ?? null,
+            contentLength: String(event.content ?? "").length,
+            cached
+        });
+    }
+
+    private handleMessageDelete(event: any) {
+        const cached = this.cache.remove(event.channelId, event.id);
+
+        this.actionLog.append({
+            type: "message-delete",
+            messageId: event.id,
+            channelId: event.channelId,
+            guildId: event.guildId ?? null,
+            cached
+        });
+    }
+
+    /* Voice auto-leave: the bot follows the owner (zzib) for now. */
+
+    private async handleVoiceStateUpdate(event: any) {
+        const userId = event?.userId;
+        if (!userId) return;

+        this.selfId ??= await this.core.getSelfId();
+        if (!this.selfId) return;
+
+        if (userId === this.selfId) {
+            const previous = this.voiceChannelId;
+            this.voiceChannelId = event.channelId ?? null;
+
+            if (!this.voiceChannelId) {
+                this.cancelAutoLeave("the bot left the call");
+            } else if (previous !== this.voiceChannelId) {
+                await this.checkOwnerPresence();
+            }
+            return;
+        }
+
+        if (userId !== this.ownerId) return;
+
+        this.ownerVoiceChannelId = event.channelId ?? null;
+
+        if (this.ownerVoiceChannelId && this.ownerVoiceChannelId === this.voiceChannelId) {
+            this.cancelAutoLeave("the owner is in the call");
+        } else if (this.voiceChannelId) {
+            this.scheduleAutoLeave();
+        }
+    }
+
+    private async checkOwnerPresence() {
+        try {
+            const owner = (await this.core.voiceGet(this.ownerId)) as { channelId?: string | null } | void;
+            this.ownerVoiceChannelId = owner?.channelId ?? null;
+        } catch (error) {
+            this.actionLog.append({ type: "error", scope: "voice-presence", message: String(error) });
+            return;
+        }
+
+        if (this.ownerVoiceChannelId && this.ownerVoiceChannelId === this.voiceChannelId) {
+            this.cancelAutoLeave("the owner is in the call");
+        } else if (this.voiceChannelId) {
+            this.scheduleAutoLeave();
+        }
+    }
+
+    private scheduleAutoLeave() {
+        if (this.voiceLeaveTimer) return;
+
+        const delaySeconds = this.config.voiceLeaveDelaySeconds;
+        if (delaySeconds <= 0) {
+            void this.leaveVoice("the owner left");
+            return;
+        }
+
+        this.actionLog.append({
+            type: "voice",
+            action: "auto-leave-scheduled",
+            channelId: this.voiceChannelId ?? null,
+            delaySeconds
+        });
+
+        this.voiceLeaveTimer = setTimeout(() => {
+            this.voiceLeaveTimer = undefined;
+            void this.leaveIfOwnerAway();
+        }, delaySeconds * 1000);
+    }
+
+    private cancelAutoLeave(reason: string) {
+        if (!this.voiceLeaveTimer) return;
+
+        clearTimeout(this.voiceLeaveTimer);
+        this.voiceLeaveTimer = undefined;
+        this.actionLog.append({ type: "voice", action: "auto-leave-cancelled", reason });
+    }
+
+    private async leaveIfOwnerAway() {
+        if (!this.voiceChannelId) return;
+
+        if (!this.ownerVoiceChannelId) {
+            try {
+                const owner = (await this.core.voiceGet(this.ownerId)) as { channelId?: string | null } | void;
+                this.ownerVoiceChannelId = owner?.channelId ?? null;
+            } catch (error) {
+                this.actionLog.append({ type: "error", scope: "voice-presence", message: String(error) });
+            }
+        }
+
+        if (this.ownerVoiceChannelId === this.voiceChannelId) {
+            this.cancelAutoLeave("the owner rejoined");
+            return;
+        }
+
+        await this.leaveVoice("the owner is not in the call");
+    }
+
+    private async leaveVoice(reason: string) {
+        try {
+            await this.core.voiceLeave();
+            this.actionLog.append({ type: "voice", action: "auto-leave", reason, channelId: this.voiceChannelId ?? null });
+        } catch (error) {
+            this.actionLog.append({ type: "voice", action: "auto-leave-failed", reason, error: String(error) });
+        }
+
+        this.voiceChannelId = null;
+    }
+
+    private async handleReactionEvent(event: any) {
        this.selfId ??= await this.core.getSelfId();
        if (!this.selfId || event.userId === this.selfId) return;

+        // Trusted reactions are notifications; poll votes do not need a tier.
+        if (this.isTrusted(event.userId)) {
+            this.actionLog.append({
+                type: "reaction",
+                action: event.type === "MESSAGE_REACTION_ADD" ? "add" : "remove",
+                channelId: event.channelId,
+                guildId: event.guildId ?? null,
+                messageId: event.messageId,
+                userId: event.userId,
+                emoji: event.emoji?.name ?? event.emoji?.id ?? null
+            });
+        }
+
+        const poll = this.polls.get(event.messageId);
+        if (!poll) return;
+
        const emoji = event.emoji?.id ?? event.emoji?.name;
        const index = POLL_EMOJIS.indexOf(emoji);
        if (index < 0 || index >= poll.options.length) return;