Add voice audio, capture and transcript, browser monitor, and DM UI

poc/src/webui.ts

@@ -8,14 +8,57 @@
  */
 
 import express, { type Request, type Response } from "express";
+import { readFileSync } from "node:fs";
+import http from "node:http";
+import https from "node:https";
+import { dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import QRCode from "qrcode";
+import { WebSocketServer } from "ws";
 
 import type { AuthService, AuthStatus } from "./auth-service.js";
+import type { AudioService } from "./audio-service.js";
 import { CoreError, type CoreClient } from "./core-client.js";
 import type { CoreProcess } from "./core-process.js";
-import { dedupe, getQueryCount, paced, pacedSend } from "./pacing.js";
+import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js";
+import type { PreferencesStore, WebPreferences } from "./preferences.js";
 import type { Puppet } from "./puppet.js";
+import { ensureTls } from "./tls.js";
+import type { TranscriptStore } from "./transcript.js";
+import type { TtsVoice } from "./tts.js";
+
+const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"];
+const LANGUAGE_NAMES: Record<string, string> = {
+    en: "English",
+    sv: "Swedish",
+    de: "German",
+    da: "Danish",
+    nb: "Norwegian",
+    no: "Norwegian",
+    ru: "Russian",
+    chr: "Cherokee"
+};
+
+function groupVoices(voices: TtsVoice[]) {
+    const groups = new Map<string, { id: string; name: string; voices: TtsVoice[] }>();
+
+    for (const voice of voices) {
+        const base = voice.language.split("-")[0];
+        const group = groups.get(base) ?? { id: base, name: LANGUAGE_NAMES[base] ?? voice.language, voices: [] };
+        group.voices.push(voice);
+        groups.set(base, group);
+    }
+
+    const all = [...groups.values()];
+    const pinned = PINNED_LANGUAGES.map(id => all.find(group => group.id === id)).filter(
+        (group): group is { id: string; name: string; voices: TtsVoice[] } => !!group
+    );
+    const rest = all
+        .filter(group => !PINNED_LANGUAGES.includes(group.id))
+        .sort((a, b) => a.name.localeCompare(b.name));
+
+    return [...pinned, ...rest];
+}
 
 const HOST = process.env.WEBUI_HOST ?? "0.0.0.0";
 const PORT = Number(process.env.WEBUI_PORT ?? 8789);
@@ -26,6 +69,10 @@ export interface WebUiDeps {
     isCoreConnected: () => boolean;
     coreProcess: CoreProcess;
     auth: AuthService;
+    audio: AudioService;
+    transcripts: TranscriptStore;
+    preferences: PreferencesStore;
+    connectAudioDevices: () => Promise<{ connected: boolean; message: string }>;
 }
 
 export interface WebUiHandle {
@@ -54,9 +101,84 @@ async function qrDataUrl(status: AuthStatus): Promise<string | undefined> {
     return QRCode.toDataURL(session.url, { margin: 1, width: 280 });
 }
 
+function stateDir(): string {
+    const stateFile =
+        process.env.VESKSLOP_STATE_FILE ?? fileURLToPath(new URL("../.veskslop-state.json", import.meta.url));
+    return dirname(stateFile);
+}
+
+/** Live browser audio: the call output streamed as raw PCM over a WebSocket. */
+function startAudioSocket(server: http.Server | https.Server, deps: WebUiDeps): void {
+    const wss = new WebSocketServer({ server, path: "/audio" });
+
+    wss.on("connection", socket => {
+        let stop: (() => void) | undefined;
+
+        const send = (message: unknown) => {
+            if (socket.readyState === socket.OPEN) socket.send(JSON.stringify(message));
+        };
+
+        socket.on("message", raw => {
+            let message: any;
+            try {
+                message = JSON.parse(raw.toString());
+            } catch {
+                return;
+            }
+
+            if (message.type !== "listen") return;
+
+            if (!message.on) {
+                stop?.();
+                stop = undefined;
+                send({ type: "status", listening: false });
+                return;
+            }
+
+            if (stop) return;
+
+            try {
+                const status = deps.audio.status();
+                if (!status.available) {
+                    throw new Error(status.hint ?? status.error ?? "audio is not available");
+                }
+                if (!status.ffmpeg) {
+                    throw new Error("ffmpeg is missing; install it to hear the call");
+                }
+
+                stop = deps.audio.startMonitor(
+                    chunk => {
+                        if (socket.readyState === socket.OPEN) socket.send(chunk);
+                    },
+                    error => {
+                        send({ type: "error", error: error.message });
+                        stop?.();
+                        stop = undefined;
+                    }
+                );
+                send({ type: "status", listening: true });
+            } catch (error) {
+                send({ type: "error", error: (error as Error).message });
+            }
+        });
+
+        const cleanup = () => {
+            stop?.();
+            stop = undefined;
+        };
+
+        socket.on("close", cleanup);
+        socket.on("error", cleanup);
+    });
+}
+
 export function startWebUi(deps: WebUiDeps): WebUiHandle {
     const app = express();
     app.use(express.json({ limit: "64kb" }));
+    app.use("/api", (_req, res, next) => {
+        res.setHeader("Cache-Control", "no-store");
+        next();
+    });
     app.use(express.static(fileURLToPath(new URL("../public", import.meta.url))));
 
     const clients = new Set<Response>();
@@ -269,6 +391,84 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         })
     );
 
+    app.get(
+        "/api/preferences",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: deps.preferences.get() });
+        })
+    );
+
+    app.post(
+        "/api/preferences",
+        wrap(async (req, res) => {
+            const { visibleGuilds, tts, favourites } = req.body ?? {};
+
+            if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
+                throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
+            }
+
+            const patch: Partial<WebPreferences> = {};
+            if (visibleGuilds !== undefined) patch.visibleGuilds = visibleGuilds;
+
+            if (tts !== undefined) {
+                patch.tts = {
+                    provider: tts?.provider ?? null,
+                    language: tts?.language ?? null,
+                    voice: tts?.voice ?? null
+                };
+            }
+
+            if (favourites !== undefined) {
+                patch.favourites = {
+                    users: Array.isArray(favourites?.users) ? favourites.users : [],
+                    channels: Array.isArray(favourites?.channels) ? favourites.channels : []
+                };
+            }
+
+            res.json({ ok: true, result: deps.preferences.update(patch) });
+        })
+    );
+
+    app.get(
+        "/api/favourites",
+        wrap(async (_req, res) => {
+            const prefs = deps.preferences.get();
+            const userIds = prefs.favourites.users;
+
+            if (!userIds.length) {
+                res.json({ ok: true, result: { users: [], channels: prefs.favourites.channels } });
+                return;
+            }
+
+            const [statuses, voices] = await Promise.all([
+                requireCore().userStatus(userIds) as Promise<{ users?: Array<{ id: string; status: string }> }>,
+                requireCore().voiceUsers(userIds) as Promise<{ users?: any[] }>
+            ]);
+
+            const statusMap = new Map((statuses.users ?? []).map(user => [user.id, user.status]));
+            const voiceMap = new Map((voices.users ?? []).map(user => [user.id, user]));
+
+            res.json({
+                ok: true,
+                result: {
+                    users: userIds.map(id => {
+                        const voice = voiceMap.get(id);
+                        return {
+                            id,
+                            username: voice?.username ?? id,
+                            avatar: voice?.avatar ?? null,
+                            status: statusMap.get(id) ?? "offline",
+                            channelId: voice?.channelId ?? null,
+                            channelName: voice?.channelName ?? null,
+                            guildId: voice?.guildId ?? null
+                        };
+                    }),
+                    channels: prefs.favourites.channels
+                }
+            });
+        })
+    );
+
     app.get(
         "/api/voice",
         wrap(async (req, res) => {
@@ -286,13 +486,137 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         })
     );
 
-    app.post("/api/voice/join", (_req, res) => {
-        res.status(501).json({ ok: false, error: "voice.join is not implemented yet" });
-    });
+    app.post(
+        "/api/voice/join",
+        wrap(async (req, res) => {
+            const channelId = String(req.body?.channelId ?? "");
+            const mute = req.body?.mute !== false;
 
-    app.post("/api/voice/leave", (_req, res) => {
-        res.status(501).json({ ok: false, error: "voice.leave is not implemented yet" });
-    });
+            const result = await pacedVoice(() => requireCore().voiceJoin(channelId, mute));
+            deps.getPuppet()?.appendAction({ type: "voice", action: "join", channelId, mute });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.post(
+        "/api/voice/leave",
+        wrap(async (_req, res) => {
+            const result = await pacedVoice(() => requireCore().voiceLeave());
+            deps.getPuppet()?.appendAction({ type: "voice", action: "leave" });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.post(
+        "/api/voice/move",
+        wrap(async (req, res) => {
+            const channelId = String(req.body?.channelId ?? "");
+            const result = await pacedVoice(() => requireCore().voiceMove(channelId));
+            deps.getPuppet()?.appendAction({ type: "voice", action: "move", channelId });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.get(
+        "/api/audio",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: deps.audio.status() });
+        })
+    );
+
+    app.get(
+        "/api/audio/tts/voices",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) });
+        })
+    );
+
+    app.get(
+        "/api/audio/devices",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: await requireCore().voiceDevices() });
+        })
+    );
+
+    app.post(
+        "/api/audio/connect",
+        wrap(async (_req, res) => {
+            res.json({ ok: true, result: await deps.connectAudioDevices() });
+        })
+    );
+
+    app.post(
+        "/api/audio/devices",
+        wrap(async (req, res) => {
+            const inputDeviceId = req.body?.inputDeviceId ? String(req.body.inputDeviceId) : undefined;
+            const outputDeviceId = req.body?.outputDeviceId ? String(req.body.outputDeviceId) : undefined;
+
+            const result = await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId);
+            deps.getPuppet()?.appendAction({
+                type: "audio",
+                action: "devices",
+                inputDeviceId: inputDeviceId ?? null,
+                outputDeviceId: outputDeviceId ?? null
+            });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.post(
+        "/api/audio/speak",
+        wrap(async (req, res) => {
+            const text = String(req.body?.text ?? "");
+            const prefs = deps.preferences.get();
+
+            const provider =
+                req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider;
+            const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
+
+            const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice });
+            deps.getPuppet()?.appendAction({
+                type: "audio",
+                action: "speak",
+                provider: provider ?? deps.audio.status().tts.defaultProvider,
+                chars: text.trim().length
+            });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.post(
+        "/api/audio/stop",
+        wrap(async (_req, res) => {
+            deps.audio.stop();
+            deps.getPuppet()?.appendAction({ type: "audio", action: "stop" });
+            res.json({ ok: true, result: deps.audio.status() });
+        })
+    );
+
+    app.post(
+        "/api/audio/capture",
+        wrap(async (req, res) => {
+            const start = req.body?.start !== false;
+            const result = start ? await deps.audio.startCapture() : (deps.audio.stopCapture(), { stopped: true });
+
+            deps.getPuppet()?.appendAction({ type: "audio", action: start ? "capture-start" : "capture-stop" });
+            res.json({ ok: true, result });
+        })
+    );
+
+    app.get(
+        "/api/transcript",
+        wrap(async (req, res) => {
+            const limit = clampLimit(req.query.limit, 100, 500);
+            const sessionId = req.query.sessionId ? String(req.query.sessionId) : undefined;
+            res.json({
+                ok: true,
+                result: {
+                    sessions: deps.transcripts.readSessions(20),
+                    segments: deps.transcripts.readSegments(limit, sessionId)
+                }
+            });
+        })
+    );
 
     app.get("/api/logs", (req, res) => {
         const puppet = requirePuppet();
@@ -325,8 +649,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
         });
     });
 
-    const server = app.listen(PORT, HOST, () => {
-        console.log(`[webui] listening on http://${HOST}:${PORT} (no auth; LAN or Tailscale only)`);
+    const tls = ensureTls(stateDir());
+    const server = tls
+        ? https.createServer({ key: readFileSync(tls.key), cert: readFileSync(tls.cert) }, app)
+        : http.createServer(app);
+
+    startAudioSocket(server, deps);
+
+    server.listen(PORT, HOST, () => {
+        console.log(`[webui] listening on ${tls ? "https" : "http"}://${HOST}:${PORT} (no auth; LAN or Tailscale only)`);
+        if (!tls && process.env.WEBUI_TLS !== "0") {
+            console.warn("[webui] TLS is unavailable; browser media APIs need HTTPS or localhost");
+        }
     });
 
     server.on("error", error => {