Add voice audio, capture and transcript, browser monitor, and DM UI

poc/src/audio.ts

@@ -0,0 +1,326 @@
+/*
+ * Veskslop daemon — audio devices, playback, ducking, and capture.
+ *
+ * The daemon plays audio into a virtual sink. A remap source of that sink's
+ * monitor becomes the client microphone. Client output can be routed to a
+ * second sink that the daemon records. Every system command is optional: when
+ * the host has no PulseAudio/PipeWire, the manager reports unavailable.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+
+const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts";
+const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic";
+const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture";
+const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30);
+
+export class AudioUnavailableError extends Error {}
+
+export interface AudioManagerStatus {
+    available: boolean;
+    backend: "pulse" | null;
+    error?: string;
+    hint?: string;
+    sink: string;
+    source: string;
+    captureSink: string;
+    ducked: boolean;
+    duckPercent: number;
+    playing: boolean;
+    recording: boolean;
+    player: boolean;
+    ffmpeg: boolean;
+}
+
+interface CommandResult {
+    code: number | null;
+    stdout: string;
+    stderr: string;
+    errorCode?: string;
+}
+
+function run(command: string, args: string[]): Promise<CommandResult> {
+    return new Promise(resolve => {
+        const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+        let stdout = "";
+        let stderr = "";
+
+        child.stdout?.on("data", chunk => (stdout += chunk));
+        child.stderr?.on("data", chunk => (stderr += chunk));
+        child.on("error", error =>
+            resolve({ code: null, stdout, stderr: String(error), errorCode: (error as NodeJS.ErrnoException).code })
+        );
+        child.on("close", code => resolve({ code, stdout, stderr }));
+    });
+}
+
+function sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+export class AudioManager {
+    private modules: number[] = [];
+    private player?: ChildProcess;
+    private recorder?: ChildProcess;
+    private ducked = false;
+    private available = false;
+    private error?: string;
+    private hint?: string;
+    private backend: "pulse" | null = null;
+    private playerAvailable = false;
+    private ffmpegAvailable = false;
+
+    async detect(): Promise<void> {
+        const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]);
+        const ffmpeg = await run(process.env.VESKSLOP_FFMPEG ?? "ffmpeg", ["-version"]);
+        this.playerAvailable = ffplay.code === 0;
+        this.ffmpegAvailable = ffmpeg.code === 0;
+
+        let info = await run("pactl", ["info"]);
+        if (info.code !== 0 && info.errorCode !== "ENOENT" && process.env.VESKSLOP_AUDIO_MANAGE_SERVER !== "0") {
+            await this.tryStartServer();
+            info = await run("pactl", ["info"]);
+        }
+
+        this.available = info.code === 0;
+        this.backend = this.available ? "pulse" : null;
+
+        if (this.available) {
+            this.error = undefined;
+            this.hint = this.playerAvailable ? undefined : "ffplay is missing; install ffmpeg to play audio.";
+            return;
+        }
+
+        if (info.errorCode === "ENOENT") {
+            this.error = "pactl is not installed";
+            this.hint =
+                "Install a sound server and its tools, for example: apt install pulseaudio pulseaudio-utils ffmpeg";
+        } else {
+            this.error = info.stderr.trim() || "the audio server is not reachable";
+            this.hint =
+                "The audio server is not running. Start PulseAudio or PipeWire-Pulse. The daemon starts pulseaudio automatically when it is installed.";
+        }
+    }
+
+    /** Start a private PulseAudio daemon when the server is missing. */
+    private async tryStartServer(): Promise<void> {
+        const start = await run("pulseaudio", ["--start", "--exit-idle-time=-1"]);
+        if (start.errorCode === "ENOENT") {
+            this.hint = "pulseaudio is not installed; install it or start PipeWire-Pulse.";
+            return;
+        }
+
+        for (let attempt = 0; attempt < 6; attempt++) {
+            await sleep(500);
+            const info = await run("pactl", ["info"]);
+            if (info.code === 0) return;
+        }
+    }
+
+    status(): AudioManagerStatus {
+        return {
+            available: this.available,
+            backend: this.backend,
+            error: this.error,
+            hint: this.hint,
+            sink: SINK,
+            source: SOURCE,
+            captureSink: CAPTURE_SINK,
+            ducked: this.ducked,
+            duckPercent: DUCK_PERCENT,
+            playing: !!this.player,
+            recording: !!this.recorder,
+            player: this.playerAvailable,
+            ffmpeg: this.ffmpegAvailable
+        };
+    }
+
+    /** Create the virtual devices when they are missing. Safe to call twice. */
+    async ensureDevices(): Promise<void> {
+        if (!this.available) await this.detect();
+        if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
+
+        const sinks = (await this.pactl(["list", "short", "sinks"])).split("\n");
+        const sources = (await this.pactl(["list", "short", "sources"])).split("\n");
+
+        if (!sinks.some(line => line.includes(SINK))) {
+            const id = await this.pactl([
+                "load-module",
+                "module-null-sink",
+                `sink_name=${SINK}`,
+                "sink_properties=device.description=VeskslopTTS"
+            ]);
+            this.modules.push(Number(id));
+        }
+
+        if (!sources.some(line => line.includes(SOURCE))) {
+            const id = await this.pactl([
+                "load-module",
+                "module-remap-source",
+                `master=${SINK}.monitor`,
+                `source_name=${SOURCE}`,
+                "source_properties=device.description=VeskslopMic"
+            ]);
+            this.modules.push(Number(id));
+        }
+
+        if (!sinks.some(line => line.includes(CAPTURE_SINK))) {
+            const id = await this.pactl([
+                "load-module",
+                "module-null-sink",
+                `sink_name=${CAPTURE_SINK}`,
+                "sink_properties=device.description=VeskslopCapture"
+            ]);
+            this.modules.push(Number(id));
+        }
+    }
+
+    /** Stop children and unload the modules this daemon created. */
+    async dispose(): Promise<void> {
+        this.stopPlayback();
+        this.stopCapture();
+        if (this.ducked) await this.setDuck(false).catch(() => undefined);
+
+        for (const id of [...this.modules].reverse()) {
+            await run("pactl", ["unload-module", String(id)]);
+        }
+        this.modules = [];
+    }
+
+    async play(file: string): Promise<void> {
+        if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
+
+        const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay";
+        await new Promise<void>((resolve, reject) => {
+            const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], {
+                env: { ...process.env, PULSE_SINK: SINK },
+                stdio: ["ignore", "ignore", "pipe"]
+            });
+
+            this.player = child;
+            let stderr = "";
+
+            child.stderr?.on("data", chunk => (stderr += chunk));
+            child.on("error", error => {
+                this.player = undefined;
+                reject(new AudioUnavailableError(String(error)));
+            });
+            child.on("close", code => {
+                this.player = undefined;
+                if (code === 0 || code === null) resolve();
+                else reject(new AudioUnavailableError(stderr.trim() || `the player exited with ${code}`));
+            });
+        });
+    }
+
+    stopPlayback(): void {
+        try {
+            this.player?.kill("SIGTERM");
+        } catch {
+            // already gone
+        }
+        this.player = undefined;
+    }
+
+    /** Lower the virtual sink volume while other people speak. */
+    async setDuck(on: boolean): Promise<void> {
+        if (this.ducked === on) return;
+        await this.pactl(["set-sink-volume", SINK, on ? `${DUCK_PERCENT}%` : "100%"]);
+        this.ducked = on;
+    }
+
+    async startCapture(file: string): Promise<void> {
+        if (this.recorder) throw new Error("capture is already running");
+        if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
+
+        const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
+        const child = spawn(
+            binary,
+            [
+                "-hide_banner",
+                "-loglevel",
+                "error",
+                "-f",
+                "pulse",
+                "-i",
+                `${CAPTURE_SINK}.monitor`,
+                "-ac",
+                "1",
+                "-ar",
+                "16000",
+                "-y",
+                file
+            ],
+            { stdio: ["ignore", "ignore", "pipe"] }
+        );
+
+        this.recorder = child;
+        child.on("close", () => {
+            if (this.recorder === child) this.recorder = undefined;
+        });
+    }
+
+    stopCapture(): void {
+        const child = this.recorder;
+        this.recorder = undefined;
+        try {
+            child?.kill("SIGINT");
+        } catch {
+            // already gone
+        }
+    }
+
+    /** Stream the capture sink as raw 48 kHz mono PCM. Returns a stop function. */
+    startMonitor(onData: (chunk: Buffer) => void, onError?: (error: Error) => void): () => void {
+        if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
+        if (!this.ffmpegAvailable) {
+            throw new AudioUnavailableError("ffmpeg is missing; install it to hear the call");
+        }
+
+        const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
+        const child = spawn(
+            binary,
+            [
+                "-hide_banner",
+                "-loglevel",
+                "error",
+                "-f",
+                "pulse",
+                "-i",
+                `${CAPTURE_SINK}.monitor`,
+                "-ac",
+                "1",
+                "-ar",
+                "48000",
+                "-f",
+                "s16le",
+                "-"
+            ],
+            { stdio: ["ignore", "pipe", "pipe"] }
+        );
+
+        child.stdout?.on("data", onData);
+        let stderr = "";
+        child.stderr?.on("data", chunk => (stderr += chunk));
+        child.on("error", error => onError?.(new AudioUnavailableError(String(error))));
+        child.on("close", code => {
+            if (code) onError?.(new AudioUnavailableError(stderr.trim() || `ffmpeg exited with ${code}`));
+        });
+
+        return () => {
+            try {
+                child.kill("SIGTERM");
+            } catch {
+                // already gone
+            }
+        };
+    }
+
+    private async pactl(args: string[]): Promise<string> {
+        const result = await run("pactl", args);
+        if (result.code !== 0) {
+            throw new AudioUnavailableError(result.stderr.trim() || `pactl ${args[0]} failed`);
+        }
+        return result.stdout.trim();
+    }
+}