Add voice processing settings and a speak-time bypass

2026-09-21 18:23:40 +02:00
parent 17d4339ea3
commit 63083c54ec
10 changed files with 230 additions and 9 deletions
@@ -1788,6 +1788,37 @@ async function refreshAudio() {
    await refreshSpeech();
 }

+/* Discord voice processing gates (Settings -> TTS panel). */
+function fillVoiceGates(voice) {
+    const gates = voice ?? {};
+    for (const [id, key] of [
+        ["#vp-echo", "echoCancellation"],
+        ["#vp-agc", "automaticGainControl"],
+        ["#vp-nc", "noiseCancellation"],
+        ["#vp-bypass", "bypassWhileSpeaking"]
+    ]) {
+        const box = $(id);
+        if (box) box.checked = gates[key] === true;
+    }
+}
+
+async function saveVoiceGates() {
+    const voice = {
+        noiseSuppression: false,
+        echoCancellation: $("#vp-echo").checked,
+        automaticGainControl: $("#vp-agc").checked,
+        noiseCancellation: $("#vp-nc").checked,
+        bypassWhileSpeaking: $("#vp-bypass").checked
+    };
+
+    try {
+        const prefs = await VK.api.post("/api/preferences", { voice });
+        fillVoiceGates(prefs.voice ?? voice);
+    } catch (error) {
+        toast(`voice processing save failed: ${error.message}`, "error");
+    }
+}
+
 /* Speech settings (server-side preferences). */

 async function refreshSpeech() {
@@ -1815,6 +1846,7 @@ async function refreshSpeech() {
    try {
        const prefs = await VK.api.get("/api/preferences");
        state.speech = prefs.tts ?? { provider: null, language: null, voice: null, speed: null, pitch: null };
+        fillVoiceGates(prefs.voice);
    } catch {
        // preferences are optional
    }
@@ -2728,6 +2760,10 @@ function init() {
    $("#transcript-refresh").addEventListener("click", () => void refreshTranscript());
    $("#transcript-start").addEventListener("click", () => void setCapture(true));
    $("#transcript-stop").addEventListener("click", () => void setCapture(false));
+    for (const id of ["#vp-echo", "#vp-agc", "#vp-nc", "#vp-bypass"]) {
+        $(id)?.addEventListener("change", () => void saveVoiceGates());
+    }
+
    $("#tts-duck").addEventListener("click", async () => {
        const on = !$("#tts-duck").classList.contains("vk-chip-on");
        try {
@@ -139,6 +139,15 @@
                                    duck on voice
                                </button>
                                <span class="vk-chip vk-chip-on">queue lines</span>
+                                <div class="vk-voice-gates" title="Discord's microphone processing. Leave off so TTS is not gated or ramped in.">
+                                    <span class="vk-micro-label">voice processing</span>
+                                    <label class="vk-tts-check"><input id="vp-echo" type="checkbox" /> echo cancellation</label>
+                                    <label class="vk-tts-check"><input id="vp-agc" type="checkbox" /> auto gain</label>
+                                    <label class="vk-tts-check"><input id="vp-nc" type="checkbox" /> noise cancellation</label>
+                                    <label class="vk-tts-check" title="Turn every gate off while a TTS line plays, then restore the states above">
+                                        <input id="vp-bypass" type="checkbox" /> off while speaking
+                                    </label>
+                                </div>
                                <div class="vk-tts-speed">
                                    <label class="vk-tts-slider" title="Speech rate; 1.0x is the engine's natural rate">
                                        <span class="vk-micro-label">speed</span>
@@ -25,6 +25,8 @@ export interface AudioServiceOptions {
    getVoiceContext?: () => Promise<{ guildId?: string | null; channelId?: string | null } | undefined>;
    getSpeakerName?: (userId: string) => Promise<string | undefined>;
    onTranscript?: (event: Record<string, unknown>) => void;
+    /** Called true when a TTS burst starts and false when the queue drains. */
+    onSpeakActivity?: (active: boolean) => void;
 }

 export interface SpeakResult {
@@ -51,6 +53,8 @@ export class AudioService {
    private readonly speakerIntervals: SpeakerInterval[] = [];
    private readonly speakerNames = new Map<string, string>();
    private playing = false;
+    /** True while a TTS burst holds the voice-processing bypass. */
+    private bypassed = false;
    private duckEnabled = true;
    private nextId = 1;
    private ducked = false;
@@ -381,6 +385,14 @@ export class AudioService {
        }

        this.playing = true;
+        if (!this.bypassed) {
+            this.bypassed = true;
+            try {
+                this.options.onSpeakActivity?.(true);
+            } catch {
+                // Best effort: speech still plays.
+            }
+        }
        try {
            // Streaming word mode: the wrapper writes the WAV while it synthesizes, so
            // playback starts on the first word instead of after the whole line.
@@ -408,6 +420,14 @@ export class AudioService {
        } finally {
            this.playing = false;
            await this.applyDuck(false);
+            if (this.bypassed && this.queue.length === 0) {
+                this.bypassed = false;
+                try {
+                    this.options.onSpeakActivity?.(false);
+                } catch {
+                    // Best effort.
+                }
+            }
            void this.pump();
        }
    }
@@ -186,6 +186,19 @@ const media = new MediaService({
    log: line => console.log(`[media] ${line}`)
 });

+/** State saved before a TTS burst turns Discord's mic processing off. */
+let voiceProcessingSaved: Record<string, boolean> | null = null;
+
+function restingVoiceProcessing(): Record<string, boolean> {
+    const prefs = preferences.get().voice;
+    return {
+        noiseSuppression: prefs.noiseSuppression,
+        echoCancellation: prefs.echoCancellation,
+        automaticGainControl: prefs.automaticGainControl,
+        noiseCancellation: prefs.noiseCancellation
+    };
+}
+
 const audio = new AudioService({
    cacheDir: process.env.VESKSLOP_CACHE_DIR,
    transcripts,
@@ -199,6 +212,32 @@ const audio = new AudioService({
        preRollMs: Number(process.env.VESKSLOP_VAD_PRE_ROLL_MS ?? DEFAULT_VAD.preRollMs),
        maxSegmentMs: Number(process.env.VESKSLOP_VAD_MAX_SEGMENT_MS ?? DEFAULT_VAD.maxSegmentMs)
    },
+    onSpeakActivity: active => {
+        const prefs = preferences.get().voice;
+        if (!prefs.bypassWhileSpeaking) return;
+
+        const client = core;
+        if (!client || !coreConnected) return;
+
+        if (active) {
+            voiceProcessingSaved = null;
+            void client
+                .voiceProcessing({
+                    noiseSuppression: false,
+                    echoCancellation: false,
+                    automaticGainControl: false,
+                    noiseCancellation: false
+                })
+                .then((result: any) => {
+                    voiceProcessingSaved = result?.before ?? null;
+                })
+                .catch(() => undefined);
+        } else {
+            const restore = voiceProcessingSaved ?? restingVoiceProcessing();
+            voiceProcessingSaved = null;
+            void client.voiceProcessing(restore).catch(() => undefined);
+        }
+    },
    getSelfId: async () => (core ? await core.getSelfId() : undefined),
    getVoiceContext: async () => {
        if (!core) return {};
@@ -481,7 +520,14 @@ function buildServer(): McpServer {
            }
        },
        async ({ channelId, mute }) =>
-            jsonText(await pacedVoice(() => requireCore().voiceJoin(channelId, mute ?? true)))
+            jsonText(
+                await pacedVoice(async () => {
+                    const joined = await requireCore().voiceJoin(channelId, mute ?? true);
+                    // The core joins with processing off; restore the resting state.
+                    await requireCore().voiceProcessing(restingVoiceProcessing()).catch(() => undefined);
+                    return joined;
+                })
+            )
    );

    server.registerTool(
@@ -497,18 +543,27 @@ function buildServer(): McpServer {
                echoCancellation: z.boolean().optional(),
                automaticGainControl: z.boolean().optional(),
                noiseCancellation: z.boolean().optional(),
-                inputVolume: z.number().min(0).max(200).optional().describe("Discord input volume percent")
+                inputVolume: z.number().min(0).max(200).optional().describe("Discord input volume percent"),
+                readOnly: z.boolean().optional().describe("Only report the current gates, change nothing")
            }
        },
-        async ({ noiseSuppression, echoCancellation, automaticGainControl, noiseCancellation, inputVolume }) =>
+        async ({
+            noiseSuppression,
+            echoCancellation,
+            automaticGainControl,
+            noiseCancellation,
+            inputVolume,
+            readOnly
+        }) =>
            jsonText(
                await requireCore().voiceProcessing({
                    ...(noiseSuppression !== undefined ? { noiseSuppression } : {}),
                    ...(echoCancellation !== undefined ? { echoCancellation } : {}),
                    ...(automaticGainControl !== undefined ? { automaticGainControl } : {}),
                    ...(noiseCancellation !== undefined ? { noiseCancellation } : {}),
-                    ...(inputVolume !== undefined ? { inputVolume } : {})
-                })
+                    ...(inputVolume !== undefined ? { inputVolume } : {}),
+                    ...(readOnly === true ? { readOnly: true } : {})
+                } as any)
            )
    );

@@ -128,6 +128,16 @@ export interface MediaTunables {
    };
 }

+/** Discord's microphone processing gates, plus a TTS bypass. */
+export interface VoicePreferences {
+    noiseSuppression: boolean;
+    echoCancellation: boolean;
+    automaticGainControl: boolean;
+    noiseCancellation: boolean;
+    /** Turn every gate off while a TTS line plays, then restore the state above. */
+    bypassWhileSpeaking: boolean;
+}
+
 export interface MediaPreferences {
    /** Combined size limit for the media DB root: databases, downloads, previews. */
    limitBytes: number;
@@ -143,6 +153,7 @@ export interface WebPreferences {
    visibleGuilds: string[] | null;
    tts: TtsPreferences;
    audio: AudioPreferences;
+    voice: VoicePreferences;
    favourites: FavouritesPreferences;
    media: MediaPreferences;
 }
@@ -173,6 +184,13 @@ const DEFAULTS: WebPreferences = {
        streamWords: true
    },
    audio: { duck: true },
+    voice: {
+        noiseSuppression: false,
+        echoCancellation: false,
+        automaticGainControl: false,
+        noiseCancellation: false,
+        bypassWhileSpeaking: true
+    },
    favourites: { users: [], channels: [] },
    media: {
        limitBytes: 30 * 1024 * 1024 * 1024,
@@ -313,6 +331,19 @@ function normalizeExpression(value: unknown): number | null {
    return Math.round(expression);
 }

+function normalizeVoice(raw: any, fallback: VoicePreferences): VoicePreferences {
+    return {
+        noiseSuppression: typeof raw?.noiseSuppression === "boolean" ? raw.noiseSuppression : fallback.noiseSuppression,
+        echoCancellation: typeof raw?.echoCancellation === "boolean" ? raw.echoCancellation : fallback.echoCancellation,
+        automaticGainControl:
+            typeof raw?.automaticGainControl === "boolean" ? raw.automaticGainControl : fallback.automaticGainControl,
+        noiseCancellation:
+            typeof raw?.noiseCancellation === "boolean" ? raw.noiseCancellation : fallback.noiseCancellation,
+        bypassWhileSpeaking:
+            typeof raw?.bypassWhileSpeaking === "boolean" ? raw.bypassWhileSpeaking : fallback.bypassWhileSpeaking
+    };
+}
+
 function normalizeLevel(value: unknown): number | null {
    if (isUnset(value)) return null;
    const number = Number(value);
@@ -399,6 +430,10 @@ export class PreferencesStore {
            };
        }

+        if (patch.voice !== undefined) {
+            this.preferences.voice = normalizeVoice(patch.voice, this.preferences.voice);
+        }
+
        if (patch.media !== undefined) {
            this.preferences.media = normalizeMedia(patch.media, this.preferences.media);
        }
@@ -468,6 +503,7 @@ export class PreferencesStore {
            audio: {
                duck: raw?.audio?.duck !== false
            },
+            voice: normalizeVoice(raw?.voice, DEFAULTS.voice),
            media: normalizeMedia(raw?.media, DEFAULTS.media),
            favourites: {
                users: Array.isArray(raw?.favourites?.users)
@@ -484,7 +484,14 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
    app.post(
        "/api/preferences",
        wrap(async (req, res) => {
-            const { visibleGuilds, tts, audio, favourites, media: mediaPrefs } = req.body ?? {};
+            const {
+                visibleGuilds,
+                tts,
+                audio,
+                voice: voicePrefs,
+                favourites,
+                media: mediaPrefs
+            } = req.body ?? {};

            if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
                throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
@@ -549,8 +556,24 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
                deps.audio.setDuckEnabled(patch.audio.duck);
            }

+            if (voicePrefs !== undefined) {
+                patch.voice = {
+                    noiseSuppression: voicePrefs?.noiseSuppression === true,
+                    echoCancellation: voicePrefs?.echoCancellation === true,
+                    automaticGainControl: voicePrefs?.automaticGainControl === true,
+                    noiseCancellation: voicePrefs?.noiseCancellation === true,
+                    bypassWhileSpeaking: voicePrefs?.bypassWhileSpeaking !== false
+                };
+            }
+
            const updated = deps.preferences.update(patch);
            if (mediaPrefs !== undefined) deps.media.applyTunables();
+            if (voicePrefs !== undefined) {
+                const core = deps.getCore();
+                if (core && deps.isCoreConnected()) {
+                    void core.voiceProcessing(patch.voice).catch(() => undefined);
+                }
+            }
            res.json({ ok: true, result: updated });
        })
    );
@@ -655,6 +678,20 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {

            const result = await pacedVoice(() => requireCore().voiceJoin(channelId, mute));
            deps.getPuppet()?.appendAction({ type: "voice", action: "join", channelId, mute });
+
+            // The core joins with processing off; put the user's resting state back.
+            const resting = deps.preferences.get().voice;
+            if (deps.isCoreConnected()) {
+                void requireCore()
+                    .voiceProcessing({
+                        noiseSuppression: resting.noiseSuppression,
+                        echoCancellation: resting.echoCancellation,
+                        automaticGainControl: resting.automaticGainControl,
+                        noiseCancellation: resting.noiseCancellation
+                    })
+                    .catch(() => undefined);
+            }
+
            res.json({ ok: true, result });
        })
    );
@@ -58,9 +58,20 @@ try {
    });
    const page = await context.newPage();
    page.on("console", message => {
-        if (message.type() === "error") results.consoleErrors.push(message.text());
+        if (message.type() !== "error") return;
+        // Resource errors carry no URL; the response listener checks those precisely.
+        if (message.text().startsWith("Failed to load resource")) return;
+        results.consoleErrors.push(message.text());
    });
    page.on("pageerror", error => results.consoleErrors.push(String(error)));
+    page.on("response", response => {
+        if (response.status() < 400) return;
+
+        // The grid falls back to the original CDN link when nothing is cached; expired
+        // signed links 404 by design and the tile shows its placeholder.
+        if (/^https:\/\/(cdn|media)\.discordapp\.(com|net)\//.test(response.url())) return;
+        results.consoleErrors.push(`HTTP ${response.status()} ${response.url()}`);
+    });

    await page.goto(URL, { waitUntil: "domcontentloaded", timeout: 30000 });
    await page.waitForSelector("#scope-media", { timeout: 20000 });