From 302436f95fd50917c23e215a794b3d64d6d7c1f9 Mon Sep 17 00:00:00 2001 From: Fisto Date: Tue, 15 Sep 2026 15:09:23 +0200 Subject: [PATCH] Add MicroslopSam TTS, Flextalk voice profiles, effects, and engine controls --- .gitignore | 4 +- CURRENT.md | 122 ++++- TTS-DESIGN.md | 477 ++++++++++++++++++ docs/INDEX.md | 2 + docs/core-bridge.md | 18 +- docs/daemon.md | 16 +- docs/voice.md | 101 +++- docs/webui.md | 34 +- microslopsam/.gitignore | 8 + microslopsam/README.md | 97 ++++ microslopsam/SCOPE.md | 26 + microslopsam/docs/NOTES.md | 99 ++++ microslopsam/docs/TTS-DESIGN.md | 477 ++++++++++++++++++ microslopsam/host/sapi4host.c | 352 ++++++++++++++ microslopsam/setup/inf2reg.py | 112 +++++ microslopsam/setup/install-deps.sh | 37 ++ microslopsam/setup/install.sh | 84 ++++ microslopsam/setup/run.sh | 28 ++ microslopsam/setup/setup-flextalk.sh | 95 ++++ microslopsam/setup/setup-prefix.sh | 51 ++ microslopsam/setup/veskslop-tts.sh | 308 ++++++++++++ poc/package.json | 3 +- poc/public/app.js | 697 ++++++++++++++++++++++++--- poc/public/index.html | 124 ++++- poc/public/style.css | 229 ++++++++- poc/scripts/audio-doctor.sh | 83 ++++ poc/src/audio-service.ts | 48 +- poc/src/audio.ts | 106 +++- poc/src/control-bridge.ts | 196 +++++++- poc/src/core-client.ts | 12 + poc/src/core-process.ts | 34 +- poc/src/mock-core.ts | 18 + poc/src/preferences.ts | 151 +++++- poc/src/puppet.ts | 5 + poc/src/tts.ts | 244 +++++++++- poc/src/webui.ts | 111 ++++- src/main/ipcCommands.ts | 31 +- src/main/mainWindow.ts | 9 +- src/main/puppetBridge.ts | 41 +- src/renderer/puppet/actions.ts | 145 +++++- src/renderer/puppet/events.ts | 14 +- src/shared/puppetProtocol.ts | 6 + 42 files changed, 4675 insertions(+), 180 deletions(-) create mode 100644 TTS-DESIGN.md create mode 100644 microslopsam/.gitignore create mode 100644 microslopsam/README.md create mode 100644 microslopsam/SCOPE.md create mode 100644 microslopsam/docs/NOTES.md create mode 100644 microslopsam/docs/TTS-DESIGN.md create mode 100644 microslopsam/host/sapi4host.c create mode 100755 microslopsam/setup/inf2reg.py create mode 100755 microslopsam/setup/install-deps.sh create mode 100755 microslopsam/setup/install.sh create mode 100755 microslopsam/setup/run.sh create mode 100755 microslopsam/setup/setup-flextalk.sh create mode 100755 microslopsam/setup/setup-prefix.sh create mode 100755 microslopsam/setup/veskslop-tts.sh create mode 100644 poc/scripts/audio-doctor.sh diff --git a/.gitignore b/.gitignore index be8fd61..7a10a1a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ node_modules .env .DS_Store .idea/ -.pnpm-store/ \ No newline at end of file +.pnpm-store/ +# Ops notes with credentials; never commit. +GITEA.md diff --git a/CURRENT.md b/CURRENT.md index d4664ee..bf8d31c 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -1,6 +1,6 @@ # CURRENT.md — session state -Last updated: 2026-09-14 +Last updated: 2026-09-15 This file tracks the work state. Read it first. Update it before you stop. @@ -12,6 +12,17 @@ mobile draft at 390px. ## In progress +- Flextalk voice profile, end to end. The profile window is a movable panel that recenters on each + open; it carries the character sliders with engine defaults and +/-2 snapping, five named + presets, radio-kit effects (high-pass 40-400 Hz, echo 0-100, word mode with a gap in ms), and + test lines (Attention, Laboratory, Critical, Health, Radiation, plus two custom lines in + `localStorage`) that post the current profile straight to `/api/audio/speak`. Fixed the test + buttons reading a stale expression from saved state; they now read the slider like Speak does. +- Root cause of the profile never reaching the engine (2026-09-15): the wrapper wrote the Flextalk + `.reg` with `echo`, and dash's `echo` treats `\2` in `...\Watson\2.0\...` as an escape, which + corrupted the key so `wine reg import` was a silent no-op (it still returns 0). The wrapper now + uses `printf`, validates that every profile value is numeric, and rejects a `.reg` containing + control characters. Documented in TTS-DESIGN.md. - The Veskslop agent lives in `agent/` (Go 1.25+, Eino v0.9.19, remote llama.cpp over the OpenAI-compatible API, daemon tools over MCP Streamable HTTP). It is disabled by default (`VESKSLOP_AGENT_ENABLED=1`), takes tasks on `POST /task`, and logs every tool call. Smoke-tested @@ -24,6 +35,115 @@ mobile draft at 390px. 390x844. Next: transcript and TTS box in the Call tab, per the 1c design. - Fixed a pre-existing web UI bug: `loadDms` called the nonexistent `renderDirect`, so the DM rail only rendered from other paths. It now calls `renderDmRail()`. +- Fixed the speaking marks: the core turned Discord's numeric `speaking: 0` into `true`, so the + equalizer bars never stopped. The bars now fade in on start and decay out on stop (500 ms), with + a 2.5 s idle timeout for missed stops; a mute or channel change ends the state at once. Muted + members show no bars. +- Browser audio: **Client audio** now runs the browser unlock in the click (microphone permission, + AudioContext created and resumed, output sink applied) before it starts listening. Verified + headless: 52 PCM frames played through a running context. The button label no longer wipes the + icon markup. +- Capture routing: nothing moved the client's playback streams into `veskslop_capture`, so the + browser monitor streamed digital silence. The audio manager now adopts matching streams + (`VESKSLOP_AUDIO_CLIENT_MATCH`, default `vesktop,electron,chromium,discord`) on device setup, + monitor start, and every 5 s while listening or capturing. +- LAN audio verified end to end on 2026-09-14: a tone stream on `veskslop_tts` was moved to + `veskslop_capture` (daemon log: `moved stream 81 to veskpop_capture`) and the browser, connected + to `https://192.168.0.178:18801/`, played 256+ PCM frames with a peak of 4096 (a stream already + on the capture sink peaked at 29274). +- Speaking bars are always in the layout now: `scaleY(0)` and transparent when idle, they scale up + with the fade-in at start, run the equalizer animation while speaking, and stay on until an + explicit stop, a mute, or a channel change. Live data showed repeated `speaking: true` pulses and + no stop events inside an 8 s window, so the idle decay is removed: bars stick rather than + disappear. The fade runs the equalizer inside a shrinking container, so the amplitude drops + smoothly instead of the block collapsing at full height. The core treats a missing `speaking` + flag as a start pulse and accepts `user_id`. Core-side changes need a rebuild plus a client + restart. +- Added `poc/scripts/audio-doctor.sh` (`pnpm audio:doctor`): default sink, sink inputs, audio + status with adoption counters, a 5 s `volumedetect` on `veskslop_capture.monitor`, and a live + event peek. Run it in a real call to trace a silent browser monitor. +- Live doctor output (2026-09-14): the `vesktop`/`electron` streams sit on `veskslop_capture` + (sink 40) already, `adopted: 0`, and `VOICE_SPEAKING` events arrive. Client audio routing is + therefore in place. Bot TTS works. Capture was off (`capturing: false`), so the transcript was + empty; Vosk transcribes correctly when invoked with the model env. +- Live call audio works now (the user confirmed TTS is audible and the silent call was partly a + per-user volume set to zero for some members, plus another issue). The `volume` read returns + `null` because this client build exposes no `UserVolumeStore`; the setter reports + `NOT_SUPPORTED`. Volume is not a usable control here. +- External TTS engine support (external project, first target is a command wrapper): the contract + lives in `TTS-DESIGN.md`. Veskslop now has nullable `tts.speed` (0.5-2.0) and `tts.pitch` + (-12..+12) preferences, per-line overrides on `/api/audio/speak` and MCP `speak`, `{speed}`/ + `{pitch}` placeholders, `VESKSLOP_TTS_SPEED`/`_PITCH` exports, and speed/pitch sliders in the TTS + panel. A custom command advertises its voices through `VESKSLOP_TTS_VOICES`; the espeak list is + hidden while a custom command is set. Verified with a wrapper that logged argv and env: + preferences and overrides both arrive. Reading the engine's `GET /voices` for the HTTP provider + remains future work. +- The MicroslopSam engine project is now part of the tree in `microslopsam/` (copied from the + separate project; its `build/` output and voice data stay ignored). The wrapper was renamed + `setup/vekslop-tts.sh` -> `setup/veskslop-tts.sh` and every reference and configuration export + was fixed. The engine also corrected the Flextalk pitch finding (REG_DWORD bug), so Keith Bell + and Kerry Watson now support pitch and carry no `no-pitch` flag. +- Implemented the Flextalk Expression profile parameter end to end: `VESKSLOP_TTS_VOICES` marks the + two voices as `expression=59` / `expression=100`; the wrapper writes the registry value before + synthesis; Veskslop has `tts.expression` (0-100 nullable), the `{expression}` placeholder, + `VESKSLOP_TTS_EXPRESSION`, an HTTP/MCP field, and an Expression slider that appears only for + those voices (default shown until moved, null leaves the registry untouched). Verified: 0/59/100 + produce distinct audio, the registry reaches 0x14 for 20, and the UI shows 25/100/default states. +- Fixed the null-value coercion in the preference normalizers (`Number(null) === 0` turned a + cleared speed/expression into 0.5/0); unset now stays null. +- Renderer-down handling polished: the bridge logs one warning per outage instead of a stack per + polled command, and the `navigator.languages` request catches the new timeout rejection instead + of raising an unhandled promise warning. (The `exitCode: 9` renderer kills in the live log were + client restarts; the daemon's retry handled the startup race as designed.) +- The TTS panel has speed presets (0.5x, 0.75x, 1x, 1.25x, 1.5x) under the fine-adjust slider, and + the pitch control is hidden entirely for voices that report `supportsPitch: false`. (Flextalk + voices no longer report that; see the pitch correction above.) Verified: preset clicks update the + slider/label and save. +- TTS playback has a lead-in of silence before each line (`VESKSLOP_TTS_LEAD_MS`, default 150 ms, + max 2000, added with `adelay` in `ffplay`; status reports `ttsLeadMs`) so Discord's voice + pipeline is settled before the first syllable. Verified on the sink monitor: first sound at + 0.02 s with 0 ms and 0.62 s with 600 ms. +- Speech settings have a **Start engine** button that warms the selected engine + (`POST /api/audio/tts/warm`, MCP `start_tts_engine`): it runs one throwaway synthesis so the + command engine's Xvfb/Wine/host are up. Verified with MicroslopSam: cold 1732 ms, warm 505 ms, + and the UI shows "MicroslopSam ready in 835 ms". The warm text is a full sentence because their + two-pass `loudnorm` fails with `inf` on very short input. +- Startup and crash robustness: `sendRendererCommand` now rejects immediately when the client + renderer is destroyed or crashed, and times out after 15 s, so a dead frame no longer logs + Electron's "Render frame was disposed" stack once per poll. Device auto-connect retries up to + 10 times every 2.5 s, because the bridge socket is ready before the renderer's modules load + (the live error was `Reflect.get called on non-object` from a lazy lookup at startup). Verified + with the mock: attempt 1 failed, attempt 2 set the devices. +- TTS engines are selectable: the Speech settings and the TTS panel list espeak-ng, the command + engine (`VESKSLOP_TTS_NAME`, default "custom engine"), and the HTTP API; unavailable entries are + disabled and Auto resolves to the command engine first. Selecting an engine switches the voice + list (`/api/audio/tts/voices?engine=`) and hides the language row for single-language engines. + The TTS panel now has a voice select next to the speed/pitch sliders. Fixed a slider bug where a + null speed preference (`Number(null) === 0`) clamped the slider to 0.5 and then persisted it. +- TTS controls: the TTS panel has a working **duck on voice** toggle (stored in preferences, + reported as `duckEnabled`). Ducking was verified live with the mock: the `veskslop_tts` sink + drops to 30% while TTS plays and someone speaks, and returns to 100% after. The Transcript tab + in the console has **Start** and **Stop** buttons; the status line shows `capturing`/`capture + off`. The espeak-ng voice list now includes the `m1`–`m7`, `f1`–`f5`, `croak`, and `whisper` + variants (`+`), so each language has about 15 voices instead of one. +- Speaking state is now authoritative: the daemon polls the core's `voice.speaking` action + (`ChannelRTCStore.getSpeakingParticipants`) every second and synthesizes the missing stop/start + events; dispatcher `VOICE_SPEAKING` hints feed the same set so stale users clear. This fixes the + bars sticking for everyone and the transcript attribution, which credited the first member + because speaker intervals never closed. Verified with the mock: a `speaking: true` event cleared + within one poll. +- The client streams sit uncorked, unmuted, and at 100% on `veskslop_capture`, yet the monitor is + silent. Gateway `VOICE_SPEAKING` events do not prove inbound RTP, and the TTS path only proves + outbound, so the remaining suspect is the client's inbound voice state (stuck deafen or a broken + voice connection), not the sink routing. `/health` now returns `selfId` so the bot's own member + flags can be picked out of the voice member list; the doctor prints it. +- Client output is still silent at the sink (`-91 dB`) although the streams are uncorked, unmuted, + and at 100%, and a headless Chromium tone into the same sink measured `-7.4 dB`. Added per-user + volume reading and writing to find Discord-side silencing: `user.volume` / `user.setVolume` in + the core, `/api/voice/volume` (GET and POST), MCP `get_user_volume` / `set_user_volume`, and + `volume` in every voice member. A volume of 0 (or an output volume of 0) silences a user even + when Pulse shows a healthy stream. `voice.join` also clears persisted self-deafen now. + The doctor prints output and per-user volumes. - Voice audio is implemented: daemon-managed virtual devices (`veskslop_tts`, `veskslop_mic`, `veskslop_capture`), TTS providers (local command + HTTP API), queued playback through `ffplay`, ducking on `VOICE_SPEAKING`, and WAV capture. The daemon starts `pulseaudio` when `pactl` cannot diff --git a/TTS-DESIGN.md b/TTS-DESIGN.md new file mode 100644 index 0000000..e752a70 --- /dev/null +++ b/TTS-DESIGN.md @@ -0,0 +1,477 @@ +# TTS-DESIGN.md — building a TTS engine for Veskslop + +This document is the integration contract for a text-to-speech engine that Veskslop will use. It is +written to be self-contained: an engine developer can work from this file alone. + +## How Veskslop uses TTS + +1. A line of text is queued (`POST /api/audio/speak` or the MCP `speak` tool). +2. The daemon asks the engine for an audio file (command mode) or audio bytes (HTTP mode). +3. `ffplay` decodes the file and plays it into the PulseAudio null sink `veskslop_tts`. +4. `veskslop_tts.monitor` is exposed as the source `veskslop_mic`. +5. The Vesktop client uses `veskslop_mic` as its microphone, so the call hears the speech. +6. While TTS plays and another person speaks, the daemon lowers the `veskslop_tts` sink volume to + 30% and restores it after. This is "ducking". The engine does not implement ducking. + +Lines are serialized: one synthesis, then playback, then the next line. There is no streaming +playback path and no partial-audio protocol. The engine returns a complete audio object. + +## Integration modes + +### HTTP provider (recommended) + +Set `VESKSLOP_TTS_API_URL`. Select the API provider with `VESKSLOP_TTS_PROVIDER=api` or the +**API TTS** choice in the TTS panel. The daemon sends: + +``` +POST +Content-Type: application/json +Authorization: Bearer (only when the key is set) + +{ "text": "Hello there.", "voice": "anna" } +``` + +The response body must be the audio bytes. `Content-Type` selects the file extension: `audio/mpeg` +or `audio/mp3` → `.mp3`, `audio/ogg` → `.ogg`, `audio/wav` or `audio/x-wav` → `.wav`, anything else +is treated as `.mp3`. Prefer `audio/wav`. + +A non-2xx response is an error. Return a short JSON or text body with the reason; the daemon logs +it and drops the line. + +Use the HTTP mode for any engine with a model load time. It keeps the model warm and avoids the +per-line process start cost. + +### Local command provider + +Set `VESKSLOP_TTS_CMD`. It takes precedence over the built-in espeak-ng engine. Veskslop replaces +these placeholders, splits the string on whitespace, and runs the result directly (no shell): + +| Placeholder | Replaced with | +| --- | --- | +| `{out}` | Output file path, extension `.wav`. Required. | +| `{text_file}` | A UTF-8 text file holding the line. | +| `{voice}` | The selected voice ID, or an empty string. | +| `{speed}` | The rate multiplier, or an empty string when unset. | +| `{pitch}` | The pitch offset in semitones, or an empty string when unset. | +| `{expression}` | The Flextalk Expression profile value, or an empty string when unset. | + +The process also receives `VESKSLOP_TTS_VOICE`, `VESKSLOP_TTS_SPEED`, +`VESKSLOP_TTS_PITCH`, and `VESKSLOP_TTS_EXPRESSION` when those are set. Write the audio to +`{out}` and exit 0. Non-zero exits are errors; stderr is captured into the log message, so keep it +short. + +Command mode is best for fast, stateless engines. It has no shell quoting: paths with spaces and +shell tricks cannot work. Use a small wrapper script if you need options. + +## Audio output requirements + +| Property | Recommendation | Notes | +| --- | --- | --- | +| Container | WAV (RIFF) | Any format ffmpeg decodes works, but WAV avoids decoder startup cost. | +| Codec | PCM signed 16-bit little-endian | `pcm_s16le`. | +| Sample rate | 48000 Hz | 44100 Hz is fine. Discord re-encodes to 48 kHz Opus; a 48 kHz source avoids one resample. | +| Channels | Mono | Discord voice is effectively mono. Stereo is accepted but adds no value. | +| Duration | One line of speech | No intro/outro music, no chimes. | +| Peak level | About -3 dBFS | Leave headroom; ducking and Discord gain must not clip. | +| Loudness | Consistent across lines | Aim for a stable integrated loudness (roughly -16 to -20 LUFS) so lines do not jump in volume. | +| Silence | Trim start to under 100 ms, keep roughly 20-50 ms of padding | Long leading silence delays the first audible word. | +| Edges | 5-10 ms fade in and out | Prevents clicks. | +| DC offset | None | Center the waveform. | + +Veskslop does not normalize, limit, or re-encode the audio. Playback is `ffplay` at 100% sink +volume, so the engine's levels are what Discord hears. + +## Voices + +A voice ID is an opaque string to Veskslop. Prefer short, stable, lowercase ASCII identifiers such +as `anna`, `sv-anna`, or `en-male-2`. The ID can contain letters, digits, `-`, `_`, and `+`. + +If the engine has multiple voices, expose them so a UI can list them: + +``` +GET /voices +[ + { + "id": "anna", + "name": "Anna", + "language": "sv", + "gender": "female", + "speed": { "min": 0.5, "max": 2.0, "default": 1.0 }, + "pitch": { "min": -12, "max": 12, "default": 0, "unit": "semitones" } + } +] +``` + +Note: Veskslop currently lists voices from espeak-ng only. A custom engine's voices appear after +Veskslop adds support for reading this endpoint (see "Veskslop changes needed"). Until then, set +the voice through the engine's own configuration and leave the Veskslop voice setting empty. + +## Speed and pitch + +Speed and pitch are engine settings, not Veskslop concepts today. Veskslop passes only `text`, +`voice`, and `provider`. No UI or preference exists for speed or pitch. + +Recommended semantics for the engine: + +- **speed**: a multiplier, `1.0` is normal, useful range `0.5` to `2.0`. Do not change pitch when + changing speed unless the user asks for it. +- **pitch**: semitones, `0` is the voice's natural pitch, useful range `-12` to `+12`. State the + unit in the `/voices` response so clients do not guess. + +Recommended HTTP shape (extras are optional; an engine may ignore them): + +``` +{ "text": "...", "voice": "anna", "speed": 1.15, "pitch": -2 } +``` + +Recommended command shape: `{speed}` and `{pitch}` are substituted when set, and the same values +are exported as `VESKSLOP_TTS_SPEED` and `VESKSLOP_TTS_PITCH`. Read either; the environment form +survives placeholder-free commands. + +### How Veskslop should add first-class settings + +Implemented on the Veskslop side (2026-09-15): + +- `tts` preferences carry `speed` (0.5–2.0) and `pitch` (-12–+12, semitones), both nullable. +- `/api/audio/speak` and the MCP `speak` tool accept optional `speed` and `pitch` that override + the preferences for that line. +- The command provider substitutes `{speed}`/`{pitch}`/`{expression}` and exports + `VESKSLOP_TTS_SPEED` / `VESKSLOP_TTS_PITCH` / `VESKSLOP_TTS_EXPRESSION` only when the value is set. +- The HTTP provider adds `speed`, `pitch`, and `expression` to the request body only when set. +- The TTS panel has speed presets, speed/pitch sliders, and an Expression slider that appears only + for voices that declare it (`expression` / `expression=NN` flags). Changes save to preferences. +- Command mode advertises voices through `VESKSLOP_TTS_VOICES`. Entries are separated by commas or + newlines and use `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or + `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch` (engine ignores + pitch) and `expression` / `expression=NN` (Flextalk profile parameter with its registry default). + `/api/audio/tts/voices` reports `supportsPitch` and `supportsExpression` / + `expressionDefault`; the TTS panel hides controls the voice does not support. While a custom + command is set, the engine's declared voices replace the espeak list. Reading `/voices` for the + HTTP provider is still future work. + +Do not encode settings into the voice string (for example `anna+speed1.2`). It breaks voice lists +and is hard to validate. + +## Text handling + +- UTF-8 input. Lines arrive trimmed and are at most 1000 characters. +- Do not parse or require SSML. Plain text is the contract. SSML support can be additive. +- Emoji, URLs, markdown, and code blocks can appear. Veskslop does not sanitize them. The engine + should not fail on an unknown character; skip or verbalize it as it prefers. +- Keep punctuation-based prosody. A period, comma, question mark, and newline are the only hints + for pacing. + +## Errors, limits, and behavior + +- One request = one complete audio object. No chunked playback, no callbacks. +- Synthesis blocks the queue. Target under about 500 ms for a short line; state a realistic budget + if the engine is slower. A warm HTTP server is the way to meet this. +- Requests are serialized by Veskslop, so the engine does not need a request queue, but it should + tolerate repeated back-to-back requests. +- The daemon never retries a failed line. Make failures visible in the response and stderr. +- Do not write to the `veskslop_tts` sink or the `veskslop_mic` source directly. The daemon owns + playback routing. + +## Optional: caching + +Veskslop does not cache audio. The engine may cache by `hash(text + voice + speed + pitch)` and +return the cached file or bytes. This is useful for repeated lines and for expensive models. + +## Veskslop changes needed + +Status as of 2026-09-15: + +1. `speed` and `pitch` are first-class: preferences, per-request overrides on `/api/audio/speak` + and MCP `speak`, env exports and placeholders for the command provider, request fields for the + HTTP provider, and two sliders in the TTS panel. +2. Command-mode voices come from `VESKSLOP_TTS_VOICES`. Reading the engine's `GET /voices` for the + HTTP provider is not implemented yet. Until then, declare voices with the env var or configure + the voice inside the wrapper. +3. No changes are needed for the audio format; the engine post-processes to the required output. + +Until `/voices` is wired, use defaults configured in the engine. + +## Acceptance checklist + +- [ ] Engine starts and answers on a stable URL or command. +- [ ] Output is mono PCM WAV at 48 kHz (or 44.1 kHz), peak near -3 dBFS. +- [ ] First audible word starts within about 500 ms of the request. +- [ ] Speech is clean: no clicks, no clipping, no long leading silence. +- [ ] Lines are consistent in loudness. +- [ ] `voice` accepts stable ASCII IDs; unknown IDs fall back to a default instead of failing. +- [ ] `speed` and `pitch` are optional and documented in `/voices`. +- [ ] Errors return a short reason and do not leave the engine wedged. +- [ ] No files are written outside the engine's own cache or the `{out}` path. + +--- + +## Clarifications from the engine side (MicroslopSam) + +_Added 2026-09-15 by the engine implementer. The contract above stands; these points +record what the engine needs to interpret, add, or hide._ + +### Output pipeline (WAV requirements) + +- Raw SAPI4 output is mono PCM s16le WAV but not at the target rate: the MSTTS voices + (Sam/Mary/Mike) render at 22050 Hz, the Flextalk voices at 11025 Hz. +- Raw Flextalk output **clips** (peak 32768, RMS about 22000) and MSTTS is much quieter, + so one shared level target is required. The engine post-processes every line: + resample to 48000 Hz, trim leading silence (<100 ms), 5-10 ms fades, DC removal, and + level normalization (peak about -3 dBFS, integrated loudness -16 to -20 LUFS). +- Implemented in `setup/veskslop-tts.sh` with ffmpeg: high-pass at 40 Hz, silence trim at + both ends (trim-start, reverse, trim-start, reverse), 8 ms fades, then a **fixed gain** + computed from an EBU R128 measurement (`ebur128=peak=true`): + `gain = min(-16 LUFS target, -3 dBFS true-peak clamp)`, then 48 kHz mono s16le. +- `loudnorm` was rejected: on short lines (under about 3 s) the loudness range is 0 and + dynamic mode attenuated a 2.8 s line to -37.9 LUFS instead of about -18 LUFS; two-pass + linear mode showed the same result. The fixed gain is steady for one-line utterances, + as Veskslop suggested. +- Verified: Keith Bell -18.4 LUFS, Kerry Watson -17.9 LUFS, Sam -20.6 LUFS; every file is + 48 kHz mono s16le at -3.0 dBFS true peak. The WAV handed to Veskslop is compliant; the + "Veskslop does not normalize" rule holds. + +### Voice IDs + +- Engine voice names are "Sam", "Mary", "Mike", "Keith Bell", "Kerry Watson", plus MSTTS + variants (telephone, hall, stadium, space) and TruVoice names that the engine lists but + has no installed data for. +- Assigned stable IDs: `sam`, `mary`, `mike`, `keith-bell`, `kerry-watson`; variants get + `mike-telephone`, `mike-hall`, `mary-stadium`, `mike-space`, and so on. Unknown IDs fall + back to `sam`. +- `/voices` (HTTP mode) reports `language: "en"`, gender, and the speed/pitch ranges below. + +### Speed and pitch mapping (measured 2026-09-15) + +- SAPI4 uses engine units, not multipliers or semitones. Measured on the dev host with a + correctly typed Flextalk install: + - MSTTS Sam: speed 30-450, natural about 150; pitch 50-200 with a real audible effect. + - Flextalk: speed 50-390, natural about 150 (durations at 100/200/300 fit a natural rate + near 140-150); pitch 50-250 works. Measured F0: Kerry 0 st = 209 Hz, -6 st = 148 Hz, + +6 st = 250 Hz (clamped); Keith 0 st = 119 Hz, +6 st = 170 Hz. +- **Correction to the earlier note**: the previous "Flextalk ignores pitch / natural 300" + finding was an artifact of the speaker registry values being stored as `REG_SZ`. The + engine requires `REG_DWORD` for numeric speaker values; with strings it synthesizes a + monotone buzz at maximum pitch. Fixed in `setup/setup-flextalk.sh`; no `no-pitch` flag + is needed. +- Mapping implemented in `setup/veskslop-tts.sh`: `speed 1.0` = natural rate 150 for every + voice, scaled by the multiplier and clamped to the engine range. `pitch` semitones scale + the voice's natural pitch unit by `2^(st/12)` and clamp to the range. Natural units are + about 100 (MSTTS), 106 (Keith Bell) and 186 (Kerry Watson). The engines' F0 response is + approximate, not semitone-calibrated. +- `{setting}` is unused; the wrapper reads `VESKSLOP_TTS_SPEED` and + `VESKSLOP_TTS_PITCH` from the environment when Veskslop later exports them. + +### Text handling + +- SAPI4 is ANSI/CP1252. The wrapper converts UTF-8 to CP1252 and maps emoji and + unmappable characters to spaces. The host passes `TTSDATAFLAG_TAGGED`, so SAPI4 control + tags in the text are enabled at the engine level; tagged input is not part of this + contract and is untested. Plain text (no SSML) is the contract. +- Lines up to 1000 characters are well within engine limits. + +### Other voice adjustments (beyond speed) + +All voices implement SAPI4 `ITTSAttributes`, so **pitch, speed and volume** exist at the +API level. The Veskslop contract exposes only speed and pitch; volume is available in the +host as `sapi4host -V ` and is not sent by Veskslop. + +- **Volume** (measured 2026-09-15): + - Flextalk: linear, range 0-65535, default 65535; half (32767) is about -6 dB, 0 is + silence. + - MSTTS: default is 0xFFFFFFFF; 0 is silence, but the response is not linear (for + example 4096 gives about -24 dB while 65535 and larger are full scale). Treat volume + as effectively on/off for MSTTS. +- **Flextalk per-speaker profile** (applied at voice load, not runtime-adjustable): + values live in `HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{GUID}` (see + `docs/NOTES.md` for the full list and defaults). `Pitch`/`Speaking Rate` set the natural + voice; **Expression** was verified to change the synthesis audibly (Keith Bell at + Expression 0 / 59 / 100 yields different waveforms with the same duration). Changing a + profile requires a registry edit and re-selecting the voice. +- **MSTTS voice character** is fixed by the `.vce`/`.cfg` data; the engine exposes no + per-voice profile in the registry. Its effect variants (telephone, in Hall, in Stadium, + in Space) are separate voice modes with separate data, exposed as distinct voice IDs + (`mike-telephone`, `mary-hall`, ...), not runtime effects. +- Not exposed by this integration: SSML, prosody/style tags, or SAPI4 attribute + extensions beyond pitch/speed/volume. + +### Flextalk profile parameters (how to implement) + +Each Flextalk speaker has a profile the engine reads when the voice is loaded. It lives in +the registry; **every numeric value must be `REG_DWORD`**: + + HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{11260610-C84A-11ce-AEC4-0000C0E9CB87} # Keith Bell + HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{000031CB-3699-551F-AED0-6B4893382D57} # Kerry Watson + +| Value (REG_DWORD unless noted) | Keith Bell | Kerry Watson | Measured effect | +| --- | --- | --- | --- | +| Pitch | 27 | 65 | Natural pitch (F0); the same scale `ITTSAttributes` shifts at runtime | +| Speaking Rate | 33 | 32 | Natural speech rate (about 150 wpm equivalent) | +| Volume | 100 | 100 | Output gain; 0 = silence | +| Expression | 59 | 100 | Audible; changes the waveform at equal duration | +| Prominences | 17 | 17 | Audible; strong prosody/emphasis effect (F0 median 122 -> 197 Hz at 100) | +| Head Size | 46 | 45 | Audible; brightness/level (peak 20001 -> 32205 at 100) | +| Lowering | 40 | 40 | Audible but subtle | +| Age | 30 | 30 | No measured effect on synthesis (informational) | +| Gender (REG_SZ) | "" | "" | No measured effect on synthesis (informational) | +| Mode Name / Speaker Name (REG_SZ) | "Keith Bell" | "Kerry Watson" | Mode name SAPI4 shows for the voice | + +How to apply a change: + +1. `setup/setup-flextalk.sh` writes these defaults, so re-running it restores them. +2. Profiles are read at voice load. The host loads the engine per request, so the next + synthesis after a registry change uses the new profile; no engine restart is needed. +3. Example (lower and slow Keith Bell), with `WINEPREFIX` pointing at the project prefix: + + wine reg add 'HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{11260610-C84A-11ce-AEC4-0000C0E9CB87}' \ + /v Pitch /t REG_DWORD /d 20 /f + wine reg add '...' /v "Speaking Rate" /t REG_DWORD /d 28 /f + + Never use `REG_SZ` for numeric values: the engine reads garbage and synthesizes a + monotone buzz at maximum pitch. +4. Not exposed by the Veskslop contract (which sends only text/voice/speed/pitch). If + profile control is wanted later, the clean place is the wrapper: read an env var such + as `VESKSLOP_TTS_PROFILE` (`key=value` list or JSON), write the registry values before + synthesis, then restore. That keeps the Veskslop interface unchanged. +5. Runtime `ITTSAttributes` pitch/speed/volume layer on top of the profile: the profile + defines the natural values, the attributes shift them per request. + +### Transport and latency + +- Installation is one command: `setup/install.sh` (system dependencies with sudo, wine + prefix + SAPI4 runtime + MSTTS voices, Flextalk extraction/registration, host build). + Re-running is safe; a fresh machine needs nothing else. +- Command provider, implemented as `setup/veskslop-tts.sh`: + `VESKSLOP_TTS_CMD="/setup/veskslop-tts.sh {out} {text_file}"` and + `VESKSLOP_TTS_VOICES="$(/setup/veskslop-tts.sh --voices)"`. The wrapper reads the + text file, transliterates UTF-8 to CP1252, starts or reuses an Xvfb when `DISPLAY` is + unset, calls the engine, then post-processes (trim, fades, measured fixed gain, 48 kHz + mono s16le). A stale private Xvfb is retried once. +- Measured on the dev host: engine-only synthesis is about 290 ms wall for Sam and about + 40 ms for Flextalk, including process start; the full wrapper adds the ffmpeg + measure/apply passes, about 630-880 ms per short line. That is above the 500 ms target; + the known optimization is a per-voice static gain (skip the measurement pass), expected + to land around 400-500 ms. +- Manual checks: `setup/run.sh -l`, `setup/run.sh -v "Keith Bell" -t "Hello." -o out.wav` + (raw engine WAV), and the wrapper above for normalized output. +- HTTP provider (later): a persistent process returning `audio/wav` bytes and `/voices`, + with optional caching by `hash(text + voice + speed + pitch)`. + +### Quirks the wrapper must hide + +- Flextalk numeric speaker settings must be `REG_DWORD`; `REG_SZ` values make the engine + synthesize a buzz. `setup/setup-flextalk.sh` writes them correctly. +- The mode list includes TruVoice names with no installed voice data; filter those out of + `/voices` (or mark them `unavailable`). +- Wine needs a working X display; the project helper starts a persistent Xvfb. + +### Licensing + +- The engines and voice data are proprietary Microsoft/AT&T/L&H artifacts. Keep this a + local integration; do not redistribute the DLLs or voice data with Veskslop. + +### Extra acceptance items for the engine + +- [x] Post-processed WAV is 48 kHz mono s16le, peak about -3 dBFS, no clipping, fades present. +- [x] Leading silence under 100 ms. +- [x] Voice ID fallback works; TruVoice entries filtered. +- [x] Latency recorded (Sam about 288 ms, Flextalk about 41 ms on the dev host). + +## Responses from the Veskslop side + +_Added 2026-09-15._ + +- **Env vars and placeholders**: `VESKSLOP_TTS_SPEED` and `VESKSLOP_TTS_PITCH` are exported only + when a value is set; `{speed}` and `{pitch}` substitute the same values. Reading the environment + is the safer option because it works even when the command has no placeholders. +- **Defaults**: a missing value means "engine default". Leave the sliders at 1.00x and 0 st in + Veskslop to keep the fields out of the request entirely. +- **`loudnorm` caution**: single-pass `loudnorm` can pump or over-amplify noise on short lines. + Set the target explicitly (`loudnorm=I=-16:TP=-1.5:LRA=11`) and verify a few lines with + `ebur128`. A fixed gain that hits about -3 dBFS peak with a consistent RMS is often steadier for + one-line utterances. Ducking lowers the sink to 30%, so keep peaks conservative and loudness + even between lines. +- **Display**: the daemon does not export `DISPLAY` to TTS children. The wrapper must start or + find its own Xvfb and set `DISPLAY` itself, as the project helper already does. Do not rely on + the client's display. +- **Engine selection**: the Speech settings and the TTS panel list every configured engine. The + command engine appears under `VESKSLOP_TTS_NAME` (default "custom engine") and is selectable + directly; Auto uses the command engine first. Selecting it hides the language row and shows the + `VESKSLOP_TTS_VOICES` list. +- **Start engine**: the Speech settings have a **Start engine** button (`POST /api/audio/tts/warm`, + MCP `start_tts_engine`) that runs one throwaway synthesis to warm the command engine (Xvfb, Wine, + host). Keep the wrapper able to handle a short sentence; the warm-up uses "The quick brown fox + jumps over the lazy dog." and reports the elapsed milliseconds. Two-pass `loudnorm` must not fail + on that input. +- **Lead-in silence**: Veskslop prepends a short silence at playback time (`VESKSLOP_TTS_LEAD_MS`, + default 150 ms) so Discord's voice pipeline is ready before the first syllable. It is added with + `adelay` in the player, not in the audio file. The engine needs no change and can keep its + 5–10 ms edge fades. +- **Voice declarations**: list only usable IDs in `VESKSLOP_TTS_VOICES`. Veskslop hides the + espeak list while a custom command is set, so a missing declaration means an empty voice list, + not a fallback. The list accepts commas or newlines. Optional flags: `no-pitch` (hide the pitch + control) and `expression` / `expression=NN` (show the Expression slider and use NN as its + default). +- **Flextalk pitch**: works now that the speaker registry values are `REG_DWORD`; no flag is + needed and Veskslop shows pitch for Keith Bell and Kerry Watson. +- **Expression profile**: implemented for the Flextalk voices. Veskslop stores `tts.expression` + (0-100, nullable), sends it per line (`{expression}` / `VESKSLOP_TTS_EXPRESSION` / the HTTP body), + and shows the slider only for voices flagged `expression=NN`. A null value leaves the registry + untouched, so the voice keeps its engine default. +- **Licensing**: keep the engine DLLs and voice data local and out of any repository; Veskslop + never bundles or redistributes them. + +## Current integration path (Veskslop side, 2026-09-15) + +Everything below is implemented. The engine stays unchanged; the wrapper and the daemon do the work. + +### Voice metadata + +`VESKSLOP_TTS_VOICES` entries may carry flags after the language field: + + id|Name|gender|language|flags + +Flags are `+`-separated: + +- `no-pitch` - hide the pitch control for this voice. +- `expression=NN` - Flextalk profile voice; NN is the default Expression value. +- `defaults=pitch:27:rate:33:prominences:17:head_size:46:lowering:40:expression:59` - + per-voice engine defaults. The UI shows "default" at these values and snaps to them + within +/-2 units. + +### Wrapper contract (`microslopsam/setup/veskslop-tts.sh`) + +Inputs: `{out}`, `{text_file}`, optional positional voice/speed/pitch/expression, and: + +| Env | Meaning | +| --- | --- | +| `VESKSLOP_TTS_PROFILE` | `pitch:27:...`/`key=value,...` overrides; the wrapper writes the full profile (defaults + overrides) to the Wine registry with `wine reg import` before every synthesis. | +| `VESKSLOP_TTS_HIGHPASS` | High-pass Hz, 40-400 (default 40). | +| `VESKSLOP_TTS_ECHO` | Mono echo 0-100 (`aecho`), default 0. | +| `VESKSLOP_TTS_WORDS` / `VESKSLOP_TTS_WORD_GAP_MS` | Word-by-word synthesis with silence gaps (Flextalk only). | + +Placeholders `{profile}`, `{highpass}`, `{echo}`, `{words}`, `{word_gap_ms}` mirror the env vars. +`wine regedit` fails headlessly but returns 0: use `wine reg import`. A missing profile value +must always write the engine default so no stale registry state survives. + +### Daemon and UI path + +1. Preferences (`tts`): `profile` (changed keys only), `profilePresets` (5 named slots), + `highpass`, `echo`, `words`, `wordGapMs`, plus `speed`, `pitch`, `expression`, `voice`. +2. `/api/audio/speak` and MCP `speak` accept all of them per line; the request overrides the + preferences and empty fields are omitted so the engine default applies. +3. The TTS panel opens the **Flextalk voice profile** window (movable, recentered on open) with + profile sliders, named presets, radio-kit controls, and test lines (Attention, Laboratory, + Critical, Health, Radiation, plus two persisted custom lines). Test buttons post the same + options as `speak`, so they audition the current profile. + +### Known pitfalls + +- The daemon runs from `tsx`: restart it after protocol, preference, or metadata changes. +- Static web files can be cached; hard-reload the browser after UI changes. +- `Number(null)` is 0: preference normalizers must treat null/undefined/"" as unset. +- The renderer log spam on a dead client is throttled in the bridge; background polls fail with + `CORE_UNAVAILABLE` and retry. +- Generate the Flextalk profile `.reg` with `printf`, never `echo`: dash's `echo` interprets + `\2` in `...\Watson\2.0\...` as an escape (0x02), so the key is corrupted and + `wine reg import` becomes a silent no-op (it still returns 0). The wrapper now rejects a + generated profile that contains control characters. This was the 2026-09-15 profile bug. diff --git a/docs/INDEX.md b/docs/INDEX.md index cb11deb..32a84d9 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -25,4 +25,6 @@ This file lists the project documents. | [../CURRENT.md](../CURRENT.md) | Live session state | | [../TODO.md](../TODO.md) | Ordered backlog | | [../poc/README.md](../poc/README.md) | Spike and daemon run commands | +| [../microslopsam/README.md](../microslopsam/README.md) | Bundled MicroslopSam SAPI4 TTS engine integration | +| [../TTS-DESIGN.md](../TTS-DESIGN.md) | Integration contract for an external TTS engine | | [../README.md](../README.md) | Project introduction | diff --git a/docs/core-bridge.md b/docs/core-bridge.md index 9765935..a199e40 100644 --- a/docs/core-bridge.md +++ b/docs/core-bridge.md @@ -87,13 +87,19 @@ The renderer normalizes client events and pushes them to subscribers. | `VOICE_STATE_UPDATE` | User, channel, guild, session, mute, deaf, self-mute, self-deaf | | `VOICE_SPEAKING` | User, speaking flag, channel | +Discord's dispatcher only sends reliable speaking-start events. The daemon polls the core's +`voice.speaking` action (which reads `ChannelRTCStore.getSpeakingParticipants`) once a second and +synthesizes the missing start/stop events, feeding dispatcher hints into the same set. Discord +sends `speaking` as a bitfield number, so the core treats `0` as false and a missing flag as a +start pulse. A user who mutes or leaves also emits a state update that ends the speaking state in +the UI. + ## Validation rules The core validates all input. The daemon cannot bypass these rules. - A channel ID and a user ID must match the snowflake format. The format is 17 to 20 digits. -- `content` must be a string. The content must not be empty after a trim. -- `content` must not contain control characters. +- `content` must be a string. The content must not be empty after a trim.- `content` must not contain control characters. - `content` must not be longer than 2000 characters. - The target channel must exist. The channel type must accept text. The allowed types are `GUILD_TEXT`, `DM`, `GROUP_DM`, `GUILD_ANNOUNCEMENT`, `ANNOUNCEMENT_THREAD`, `PUBLIC_THREAD`, @@ -145,7 +151,13 @@ The core rate limit is a safety net. It does not replace the daemon pacing. | `DM_UNAVAILABLE` | The client cannot create a DM channel. | | `NOT_SUPPORTED` | The client does not offer the internal action. | | `RATE_LIMITED` | The core rate limit rejected the action. | -| `INTERNAL` | An unexpected error occurred. The details go to the local log only. | +| `INTERNAL` | An unexpected error occurred. The error message is included (for example a lazy module lookup before the renderer finished loading). | + +A renderer command that is sent while the window is destroyed or crashed is rejected at once with +`CORE_UNAVAILABLE`, and a command that gets no answer times out after 15 seconds. The bridge socket +survives a renderer restart; the daemon retries background work such as device auto-connect. While +the renderer is gone, the bridge logs one warning instead of one stack per command, then stays +quiet until a command succeeds again. ## Hooks into upstream files diff --git a/docs/daemon.md b/docs/daemon.md index 9c794a3..e0a1d04 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -61,11 +61,14 @@ The daemon exposes MCP over Streamable HTTP. The default address is `http://127. | `send_dm` | Send a DM to a user. | | `get_voice_state` | Which voice channel a user is in and who else is there. | | `get_voice_overview` | Occupied voice channels in a guild and their members. | +| `get_user_volume` | Per-user playback volume and the client output volume. | +| `set_user_volume` | Set a user's playback volume (0 to 200). | | `voice_join` | Join a voice channel, muted by default. | | `voice_leave` | Leave the current voice channel. | | `voice_move` | Move to another voice channel. | | `get_audio_status` | Audio engine, TTS, queue, ducking, and capture state. | -| `speak` | Queue text to speech into the virtual microphone. | +| `speak` | Queue text to speech into the virtual microphone (`text`, `provider?`, `voice?`, `speed?`, `pitch?`). | +| `start_tts_engine` | Warm up the TTS engine so the first line is fast (`engine?`). | | `stop_audio` | Stop speech and clear the queue. | | `start_capture` / `stop_capture` | Record or stop recording the client output sink. | | `list_audio_devices` | The client's input and output audio devices. | @@ -160,10 +163,19 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the | `VESKSLOP_AUDIO_AUTOCONNECT` | `1` | Point the client at the virtual devices on connect. | | `VESKSLOP_AUDIO_MANAGE_SERVER` | `1` | Start `pulseaudio` when `pactl` cannot reach a server. | | `VESKSLOP_AUDIO_SINK` / `_SOURCE` / `_CAPTURE_SINK` | `veskslop_tts` / `veskslop_mic` / `veskslop_capture` | Virtual device names. | +| `VESKSLOP_AUDIO_CLIENT_MATCH` | `vesktop,electron,chromium,discord` | Stream name patterns moved into the capture sink. | + +The daemon warns at startup when `dist/` is older than the core sources. Rebuild with `pnpm build` +and restart the client, or new actions fail with `UNKNOWN_ACTION`. | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | +| `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). | | `VESKSLOP_FFPLAY` / `VESKSLOP_FFMPEG` | `ffplay` / `ffmpeg` | Playback and capture binaries. | -| `VESKSLOP_TTS_CMD` | unset | Local TTS command with `{out}` and `{text_file}`. | +| `VESKSLOP_TTS_CMD` | unset | Local TTS command with `{out}` and `{text_file}` (also `{voice}`, `{speed}`, `{pitch}`). | +| `VESKSLOP_TTS_NAME` | `custom engine` | Display name for the command engine in the web UI. | | `VESKSLOP_TTS_API_URL` / `_API_KEY` / `_PROVIDER` | unset | HTTP TTS provider and bearer token. | +| `VESKSLOP_TTS_VOICES` | unset | Voice list for a custom command: `id[|Name[|gender[|language]]]`, comma separated. | +| `VESKSLOP_TTS_SPEED` / `_PITCH` | unset | Exported to the command only when a value is set. | +| `VESKSLOP_TTS_EXPRESSION` | unset | Flextalk Expression profile value (0-100), exported when set. | | `VESKSLOP_TRANSCRIPT` | `/transcripts.jsonl` | Transcript store path. | | `VESKSLOP_PREFERENCES` | `/preferences.json` | Web UI preferences (visible guilds, TTS). | | `VESKSLOP_CAPTURE_AUTO` | `0` | Start capture when the bot joins a call. | diff --git a/docs/voice.md b/docs/voice.md index f4a954d..445be0f 100644 --- a/docs/voice.md +++ b/docs/voice.md @@ -33,7 +33,8 @@ The core reads `VoiceStateStore` only. It never joins a call on its own. Rules: - The channel must be a voice channel. The core validates the snowflake and the channel type. -- Join joins the call muted. The bot does not capture or send audio. +- Join joins the call muted and clears self-deafen. Discord persists self-deafen across sessions, + and a deafened client receives no call audio, which silences the monitor and the transcript. - Voice actions are state changes, not send floods. They stay out of the message pacing queue. The daemon spaces them 250-750 ms (`pacedVoice`). - The join goes through Discord's internal channel action, located at runtime. @@ -81,15 +82,108 @@ works as before. bytes. `VESKSLOP_TTS_API_KEY` adds a bearer token. - `ffplay` decodes and plays into `veskslop_tts` (`PULSE_SINK`). `VESKSLOP_FFPLAY` overrides the binary. Lines play one at a time in order. +- Each line gets a lead-in of silence before the first sample (`VESKSLOP_TTS_LEAD_MS`, default + 150 ms, maximum 2000). This gives Discord's voice pipeline a moment to settle so the first + syllable is not swallowed. The daemon adds it with `adelay` at playback; the engine's file is + unchanged. The status reports it as `ttsLeadMs`. + +### Per-user volume + +Discord stores a playback volume per user (0 to 200, default 100). **0 silences that user**, which +silences the capture sink even though the stream is uncorked and at 100%: the client mixes the user +at zero. Read it with `GET /api/voice/volume` (defaults to the current call) or the +`get_user_volume` MCP tool; the `voice.get` member list and `get_voice_state` carry `volume` too. +The response also carries the client's own `outputVolume`, which has the same effect when it is 0. + +Set it with `POST /api/voice/volume` or `set_user_volume`. The action goes through pacing and is +written to the action log as `set-volume`. + +### Engines + +The Speech settings list every configured engine: **espeak-ng**, the command engine +(`VESKSLOP_TTS_CMD`, shown as `VESKSLOP_TTS_NAME` or "custom engine"), and **HTTP API** +(`VESKSLOP_TTS_API_URL`). Unavailable engines are disabled. Auto picks the command engine first, +then espeak-ng, then the API. + +Selecting an engine switches the voice list to that engine. The language selector is hidden for +single-language engines such as the command engine; their voices come from `VESKSLOP_TTS_VOICES`. +The TTS panel carries the same engine and voice selects plus the speed and pitch sliders. + +**Start engine** in the Speech settings warms the selected engine by running one throwaway +synthesis. For the command engine this starts Xvfb, Wine, and the engine host, so the first real +line is fast. The result reports the elapsed milliseconds; espeak-ng and the HTTP API need no +warm-up. The same action is available as `POST /api/audio/tts/warm` and the MCP +`start_tts_engine` tool. + +### Voices + +The local engine is espeak-ng. `GET /api/audio/tts/voices` returns every base voice plus the +espeak variants (`m1`–`m7`, `f1`–`f5`, `croak`, `whisper`) as `+`, so each language +has roughly 15 voices. For natural voices, point `VESKSLOP_TTS_API_URL` at a network TTS (piper, +OpenAI, ElevenLabs) and pick the `api` provider; `VESKSLOP_TTS_CMD` can also call any local engine +that accepts `{out}`, `{text_file}`, and `{voice}`. + +When `VESKSLOP_TTS_CMD` is set, the daemon hides the espeak list and shows only the voices the +command declares through `VESKSLOP_TTS_VOICES`. Entries are separated by commas or newlines and +use `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or +`id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch` (hide the pitch +control) and `expression` / `expression=NN` (Flextalk Expression parameter; NN is the registry +default). The external-engine contract, including speed, pitch, expression, and voice metadata, is +in [TTS-DESIGN.md](../TTS-DESIGN.md). The bundled engine integration lives in +[`microslopsam/`](../microslopsam/README.md). + +### Speed, pitch, and expression + +The TTS panel has a speed slider (0.5x–2.0x, 1.0x is the engine's natural rate) with 0.5x, 0.75x, +1x, 1.25x, and 1.5x preset buttons, and a pitch slider (-12 to +12 semitones, 0 is natural). Both +hide for voices whose engine does not support them. Flextalk voices (Keith Bell, Kerry Watson) +also expose an **Expression** slider (0–100) that changes the voice character; it appears only for +those voices and defaults to the engine's registry value until moved. Values are stored in the +daemon preferences and travel to the provider per line; the command provider also receives +`VESKSLOP_TTS_SPEED`, `VESKSLOP_TTS_PITCH`, and `VESKSLOP_TTS_EXPRESSION`. Absent values mean +"engine default". `/api/audio/speak` and the MCP `speak` tool accept `speed`, `pitch`, and +`expression` overrides for a single line. + +### Flextalk voice profile (MicroslopSam) + +Voices flagged `expression=NN` are Flextalk voices and open the **Flextalk voice profile** window +from the TTS panel: + +- **Voice character** - Pitch, Speaking rate, Prominences, Head size, and Lowering, each showing the + voice's engine default and snapping to it within +/-2 units, plus Expression. A value equal to + the default clears the override. +- **Presets** - five named slots stored in `tts.profilePresets`; Save and Load copy the whole + profile. +- **Radio kit** - high-pass (`tts.highpass`, 40-400 Hz, default 40), mono echo (`tts.echo`, 0-100, + default 0), and word mode (`tts.words`) with a gap in ms (`tts.wordGapMs`) for the radio/robot + character. +- **Test lines** - Attention, Laboratory, Critical, Health, and Radiation, plus two custom lines + kept in `localStorage`. Each button speaks the line with the current profile, so a change can be + auditioned without touching the real queue. + +The window is movable and recenters every time it opens, so it cannot be lost off screen. Changed +profile keys travel as `profile` (with `highpass`, `echo`, `words`, and `wordGapMs`) to the command +provider, which exports `VESKSLOP_TTS_PROFILE`, `VESKSLOP_TTS_HIGHPASS`, `VESKSLOP_TTS_ECHO`, +`VESKSLOP_TTS_WORDS`, and `VESKSLOP_TTS_WORD_GAP_MS`. The wrapper contract and the registry +pitfalls (`wine reg import`; never `echo` for the `.reg` key path) are in +[TTS-DESIGN.md](../TTS-DESIGN.md). ### Browser monitor (Client audio) The header **Client audio** button streams the call to the browser: +- The daemon moves the client's playback streams into `veskslop_capture` (match on + `application.name` / `application.process.binary`, default `vesktop,electron,chromium,discord`, + `VESKSLOP_AUDIO_CLIENT_MATCH` overrides). Without the move the capture sink is silent. The adopt + pass runs when devices are set up, when the monitor starts, and every 5 seconds while the monitor + or the capture session runs. - The daemon reads `veskslop_capture.monitor` with `ffmpeg` and sends raw 48 kHz mono PCM over the WebSocket at `/audio`. - The browser plays the PCM with the Web Audio API and can send it to a chosen output device. - This is separate from the bot's microphone and TTS. Nothing the browser plays affects the call. +- Diagnose with `cd poc && pnpm audio:doctor`. It prints the default sink, the sink inputs and their + sinks, the daemon audio status (`clientMatch`, `adopted`, `lastAdoptError`), and eight seconds of + `VOICE_SPEAKING` / `VOICE_STATE_UPDATE` events. Run it during an active call. Browser microphone puppeting (talking through the bot) is not implemented. It needs a mix sink so the browser mic and TTS share `veskslop_mic`, a `pacat` input, and echo handling. @@ -98,8 +192,9 @@ the browser mic and TTS share `veskslop_mic`, a `pacat` input, and echo handling When another person speaks, the daemon lowers the `veskslop_tts` sink volume to `VESKSLOP_AUDIO_DUCK` percent (default 30) and restores it when they stop. Speaking comes from the -`VOICE_SPEAKING` event. Staying quiet while someone else talks is a queue property, not a switch. -Yield (stop mid-line) is not implemented. +`VOICE_SPEAKING` event. The TTS panel has a **duck on voice** toggle; it is stored in the daemon +preferences and reported as `duckEnabled` by `/api/audio`. Staying quiet while someone else talks +is a queue property, not a switch. Yield (stop mid-line) is not implemented. ### Capture and transcript diff --git a/docs/webui.md b/docs/webui.md index 97a02b1..35b59db 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -64,9 +64,15 @@ With plain HTTP, `navigator.mediaDevices` only exists on localhost; remote brows - Right column: **In voice** (favourited users in a call, grouped by call), **Favourites**, and **Open DMs & groups** as quick access. The guild's own occupied channels live in the left rail. - Speaking users show animated level bars and a bold name (design 2c: audio → video → share marks). + The bars scale up when speaking starts and stay on until an explicit stop, a mute, or a channel + change. A missing stop event leaves them on by design; they do not decay on their own. The + fade-out runs the equalizer inside a shrinking container, so the amplitude drops to nothing + instead of the block collapsing at full height. Muted members never show bars. Nicknames come from the guild member store; DMs fall back to the username. - Console: Tool calls, Events (live SSE), Transcript. The ticker shows the newest event. The - Transcript tab lists captured segments (time, speaker, text, WAV) and updates live. + Transcript tab lists captured segments (time, speaker, text, WAV) and updates live. It has + **Start**, **Stop**, and **Refresh** buttons; the status line shows `capturing` or `capture off` + plus the segment count. - Opening a chat fetches the latest 50 messages from Discord into the client cache. The refresh button by the channel header re-fetches. Live message events read the cache. Edited messages show an `edited` marker and replies show `↳ replying to `. @@ -131,9 +137,19 @@ The Settings button opens a slide-over: - **Client** — a collapsible group with: - **Bot audio** — engine status, TTS provider state, a **Connect devices** button, and the capture toggle. These are the daemon host's devices. - - **Speech** — the TTS engine (Auto / Local / API), language, and voice. Voices are grouped by - base language and filtered by the chosen language; English, Swedish, German, Danish, - Norwegian, Russian, and Cherokee are pinned on top. Stored in the daemon preferences. + - **Speech** — the TTS engine (Auto plus every configured engine: espeak-ng, the command engine + by its `VESKSLOP_TTS_NAME`, and the HTTP API), language, and voice. **Start engine** warms the + selected engine (starts Xvfb, Wine, and the host for the command engine). Selecting an engine + switches the voice list to that engine; the language row hides for single-language engines. + Voices are grouped by base language and filtered by the chosen language; English, Swedish, + German, Danish, Norwegian, Russian, and Cherokee are pinned on top. Stored in the daemon + preferences. Speed (0.5x–2.0x with 0.5/0.75/1/1.25/1.5 presets), pitch (-12 to +12 + semitones), and an Expression slider (0–100, Flextalk voices only) live in the TTS panel and + save to the same preferences; controls the selected voice does not support stay hidden. For + Flextalk voices the TTS panel also opens the **Flextalk voice profile** window: a movable panel + that recenters on each open, with character sliders, five named presets, radio-kit effects + (high-pass, echo, word mode), and test lines (Attention, Laboratory, Critical, Health, + Radiation, plus two custom) that speak immediately. - **Display** — the timestamp timezone. Defaults to the browser's zone (`Intl.DateTimeFormat().resolvedOptions().timeZone`); pick any IANA zone to override. Stored in `localStorage` and applied to messages, the transcript, the event log, the action log, and the @@ -141,7 +157,9 @@ The Settings button opens a slide-over: - **Browser audio** — puppeting controls: the browser output device for Client audio, a placeholder mic select, and the call-audio state. **Enable browser audio** asks for the microphone permission, which browsers require before they list output devices. Separate from - the bot's devices. + the bot's devices. Clicking **Client audio** in the header does the same: it asks for the + permission, unlocks the audio context in the same click, and then starts listening. A second + click stops it. - **Server** — a collapsible group with: - **Core process** — state, last exit, output, Start / Stop / Restart. - **Account login** — token file state, QR login, captcha. @@ -200,6 +218,8 @@ replaceable. | GET | `/api/dms` | `limit?` | Cached DM and group DM channels with recipients | | GET | `/api/voice` | `userId?`, `guildId?` | One user's voice state | | GET | `/api/voice/overview` | `guildId` | Occupied voice channels and members | +| GET | `/api/voice/volume` | `userIds?` comma-separated | Per-user volumes plus output volume; defaults to the current call | +| POST | `/api/voice/volume` | `userId`, `volume` | Set a user's playback volume (0 to 200) | | POST | `/api/voice/join` | `channelId`, `mute?` | Join a voice channel (muted by default) | | POST | `/api/voice/leave` | none | Leave the current voice channel | | POST | `/api/voice/move` | `channelId` | Move to another voice channel | @@ -207,11 +227,11 @@ replaceable. | POST | `/api/preferences` | `visibleGuilds?`, `tts?` | Update preferences. Stored in `/preferences.json`. | | GET | `/api/audio` | none | Audio engine, TTS, queue, ducking, capture state | | GET | `/api/audio/devices` | none | The client's input and output devices | -| GET | `/api/audio/tts/voices` | none | Local TTS voices (espeak-ng) | +| GET | `/api/audio/tts/voices` | `engine?` | Voices for the selected engine (espeak-ng or `VESKSLOP_TTS_VOICES`) | | POST | `/api/audio/connect` | none | Re-apply the virtual devices to the client | | WS | `/audio` | `{"type":"listen","on":bool}` | Raw 48 kHz mono PCM of the call | | POST | `/api/audio/devices` | `inputDeviceId?`, `outputDeviceId?` | Set the client's devices | -| POST | `/api/audio/speak` | `text`, `provider?`, `voice?` | Queue text to speech | +| POST | `/api/audio/speak` | `text`, `provider?`, `voice?`, `speed?`, `pitch?` | Queue text to speech | | POST | `/api/audio/stop` | none | Stop speech and clear the queue | | POST | `/api/audio/capture` | `start` | Start or stop capture | | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | diff --git a/microslopsam/.gitignore b/microslopsam/.gitignore new file mode 100644 index 0000000..735b6e4 --- /dev/null +++ b/microslopsam/.gitignore @@ -0,0 +1,8 @@ +build/ +*.o +*.a +*.so +*.exe +*.dll +setup/out-*.reg +*.wav diff --git a/microslopsam/README.md b/microslopsam/README.md new file mode 100644 index 0000000..b1ef54d --- /dev/null +++ b/microslopsam/README.md @@ -0,0 +1,97 @@ +# MicroslopSam + +Run SAPI4 text-to-speech voices on Linux. + +Primary approach: a **Winelib host** — a native Linux ELF that uses Wine's +libraries to load the original 32-bit SAPI4 runtime and TTS engine DLLs. + +Voice targets: + +- Microsoft Sam / Mary / Mike — MSTTS engine (`msttssyn.dll` + `.vce` data) +- AT&T Flextalk 2.1 — "Keith Bell" and "Kerry Watson" (`FlexTalk.dll`) + +Long-term: replace Wine-provided pieces with own shims (rehost) for a +pure-native build, and keep a path to a fully portable engine. + +## Status (working) + +- Winelib host builds and runs: `build/sapi4host.exe` (launcher) + + `build/sapi4host.exe.so` (native 32-bit ELF). +- Voices enumerate and synthesize: + - MSTTS: Sam, Mary, Mike (+ telephone and effect variants) + - AT&T Flextalk: Keith Bell, Kerry Watson +- Test output: `build/test_sam.wav` (22.05 kHz), `build/test_Keith_Bell.wav`, + `build/test_Kerry_Watson.wav` (11.025 kHz). + +## Build + +Everything in one command (idempotent; sudo only for system packages): + + setup/install.sh + +Manual steps, if needed: + + sudo setup/install-deps.sh # Wine/Winelib/helpers + setup/setup-prefix.sh # prefix + SAPI4 runtime + MSTTS voices + setup/setup-flextalk.sh # Flextalk extraction + registry + # host build: + winegcc -m32 -O2 -o build/sapi4host-winlib host/sapi4host.c -I../reference \ + -lole32 -luser32 -ladvapi32 + i686-w64-mingw32-gcc -O2 -static -o build/sapi4host.exe host/sapi4host.c \ + -I../reference -lole32 -luser32 -ladvapi32 -luuid + +## Usage + + setup/run.sh -l + setup/run.sh -v Sam -t "Hello." -o sam.wav + setup/run.sh -v "Keith Bell" -t "Hello." -o keith.wav -p 100 -s 150 + +Options: `-v` voice, `-t` text, `-o` output WAV, `-p` pitch, `-s` speed, `-V` volume +(volume is engine-specific; not used by the Veskslop wrapper). + +## Veskslop integration + + export VESKSLOP_TTS_CMD="$PWD/setup/veskslop-tts.sh {out} {text_file}" + export VESKSLOP_TTS_VOICES="$(setup/veskslop-tts.sh --voices)" + +The wrapper maps voice IDs (`sam`, `mary`, `mike`, `keith-bell`, `kerry-watson`, plus the +MSTTS variants), reads `VESKSLOP_TTS_VOICE`, `VESKSLOP_TTS_SPEED` (multiplier), +`VESKSLOP_TTS_PITCH` (semitones) and `VESKSLOP_TTS_EXPRESSION` (0-100, Flextalk voices +only), starts Xvfb when `DISPLAY` is unset, and writes a 48 kHz mono s16le WAV trimmed, +faded and normalized to about -16 LUFS (-3 dBFS true peak). +Speed and pitch apply to every voice (Flextalk's numeric settings must be `REG_DWORD`, +which the setup script writes). Expression is a Flextalk profile value applied at voice +load: the wrapper writes it to the Wine registry before synthesis. `--voices` marks those +voices with `expression=NN`, where NN is the registry default. Mapping details are in +`docs/TTS-DESIGN.md`. + +## Layout + +- `host/sapi4host.c` — host: voice select, pitch/speed, text -> WAV +- `setup/install.sh` — one-shot installer (runs all steps below) +- `setup/install-deps.sh` — Wine/Winelib/helpers (root) +- `setup/setup-prefix.sh` — prefix + SAPI4 runtime + MSTTS engine/voices +- `setup/setup-flextalk.sh` — Flextalk files + registry +- `setup/inf2reg.py` — INF AddReg -> `.reg` converter +- `setup/run.sh` — run helper (prefix + Xvfb) +- `setup/veskslop-tts.sh` — Veskslop command-provider wrapper (post-processing included) +- `docs/NOTES.md` — registry/CLSID facts, runtime RE notes, gotchas +- `docs/TTS-DESIGN.md` — integration contract with Veskslop plus engine clarifications + +## Known issues + +- `xvfb-run` is flaky on this host. `setup/run.sh` starts a persistent Xvfb + (`:77`) instead; Wine needs a working X display because the runtime creates + windows. +- Flextalk numeric speaker settings must be `REG_DWORD` (the installer script handles + this); `REG_SZ` values cause a monotone buzz. Pitch and speed work on all voices. +- The Flextalk archive lacks `data2.cab`; `unshield -O` extracts `data1.cab` + but one tail file (`WatsonUNInstall.dll`) is incomplete. + +## Reference material (not in this repo) + +- `../reference/speech.h` — SAPI4 SDK header +- `../reference/SAPI4-CLI`, `../reference/TETYYS-SAPI4` — host examples +- `../Microsoft-Sam-Mary-Mike-TruVoice-WSAPI4` — installers/payloads +- `../flextalk` — Flextalk installer archive + extraction +- `/mnt/ghidra/decomp/{msttssyn.dll,speech.dll}/` — decompiled corpora diff --git a/microslopsam/SCOPE.md b/microslopsam/SCOPE.md new file mode 100644 index 0000000..c9baa5e --- /dev/null +++ b/microslopsam/SCOPE.md @@ -0,0 +1,26 @@ +# Scope + +## Goals + +1. Synthesize speech with the MSTTS voices (Sam, Mary, Mike) on Linux. +2. Synthesize speech with the AT&T Flextalk 2.1 voices ("Keith Bell", + "Kerry Watson") on Linux, through the same host. +3. Ship as a native Linux binary (Winelib first, pure-native later). +4. Keep the engine contract limited to what `speech.h` defines, so other + SAPI4 engines can be added without host changes. +5. Text in, WAV out. Pitch and speed adjustable. + +## Approach + +- Phase 1 (current): Winelib host + Wine-provided runtime. Fastest path to + working audio, and it doubles as the reference oracle for later work. +- Phase 2: replace Wine-provided parts with own shims one at a time + (file I/O, threads, OLE2, window/message pump, finally MSVC EH/SEH). +- Phase 3: pure-native rehost of `msttssyn.dll`; no Wine libraries. + +## Non-goals (for now) + +- GUI, SAPI4 telephony (`spchtel.dll`), Voice Command (`vcmd`), Agent support. +- TruVoice voices (possible later; same L&H engine family). +- 64-bit build (the engines are 32-bit). +- Multi-platform audio (PipeWire/ALSA output comes after WAV works). diff --git a/microslopsam/docs/NOTES.md b/microslopsam/docs/NOTES.md new file mode 100644 index 0000000..de90d3b --- /dev/null +++ b/microslopsam/docs/NOTES.md @@ -0,0 +1,99 @@ +# Notes + +## Registry facts + +From `spchapi/SPCHAPI.INF` (`%49000%` = `%25%\speech` = `C:\windows\speech`): + +- `CLSID_TTSEnumerator` `{D67C0280-C743-11cd-80E5-00AA003E4B50}` -> `Speech.dll` +- "Audio Destination to File Object" `{D4623720-E4B9-11cf-8D56-00A0C9034A7E}` -> `Speech.dll` +- "Audio Destination Object" `{CB96B400-C743-11cd-80E5-00AA003E4B50}` -> `Speech.dll` +- Lexicon dialog `{92655FB1-ADF9-11d1-BEB9-006008317CE8}` -> `Speech.dll` +- Phoneme converter `{B9F11A96-90E3-11d0-8D77-00A0C9034A7E}` -> `Speech.dll` + +From `msttsl/mstts.inf` (`[TTSReg]`): + +- Engine `MSTTSSyn` = `{E0725551-286F-11d0-8E73-00A0C9083363}` -> `MSTTSSYN.dll` +- Engine install dir: `Common Files\microsoft shared\SpeechEngines\TTS`. +- Enumerates voices by scanning `*.vce` in its own directory. + +## Flextalk registration (from the installer's `setup.ins`) + +- Engine value: `HKLM\Software\Voice\TextToSpeech\Engine` + `"AT&T FlexTalk" = "{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}"` +- Engine subkey (forces mode enumeration): `...\Engine\{FFE78E00-...}` with + `Dirty = 1` (REG_DWORD). The runtime rewrites `Count` + `Mode0..N` and + clears `Dirty` after enumerating. +- `HKCR\CLSID\{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}\InprocServer32` = + `\EngineDLLs\FlexTalk.dll`, `ThreadingModel = Apartment`. + Also registered: `{778EAAE2-499F-11CF-B0BA-444553540000}` -> same DLL. +- Directories: `HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Directories` + `RootDirectory = C:\FlexTalk\`. +- Dictionaries: `...\FlexTalk\2.0\Dictionaries`: + `StandardDictionary="2 ft_sdict.dll"`, `ExceptionDictionary="0 ft_udict.dll"`, + `HomographDictionary="1 ft_hgrap.dll"`, `ProperNameDictionary="3 ft_namsa.dll"`, + `LetterToSoundDictionary="4 ft_l2s.dll"`. +- Options: `...\FlexTalk\2.0\Options` `Gender=Male`, `SampleRate=11025`. +- Speakers (per-voice settings; names come from these values). **All numeric + values must be REG_DWORD**: with REG_SZ the engine reads garbage and + synthesizes a monotone buzz (pitch fixed at max, pitch control dead). + - `{11260610-C84A-11ce-AEC4-0000C0E9CB87}`: Mode Name / Speaker Name + "Keith Bell", Pitch 27, Speaking Rate 33, Head Size 46, Expression 59, + Lowering 40, Prominences 17, Age 30, Volume 100. + - `{000031CB-3699-551F-AED0-6B4893382D57}`: "Kerry Watson", Pitch 65, + Speaking Rate 32, Head Size 45, Expression 100, Lowering 40, + Prominences 17, Age 30, Volume 100. +- Voice data: `EngineData\AE1M08LC.*`, `AE1M11LC.*` (male, 8/11 kHz) and + `AE2F08LC.*`, `AE2F11LC.*` (female); `.INV` inventory, `.NDX` index, + `.COD` codebook, plus shared `DICT_CI.*`, `phondef`, `AMPRULES`, `ENDINGS`. + +Extraction: `unshield -O -d extracted x installer/flextalk/data1.cab` +(`-O` = old compression; the archive.org package has no `data2.cab`, so the +final file `WatsonUNInstall.dll` is incomplete). + +## SAPI4 runtime engine discovery (`speech.dll`) + +- `FUN_635747a4` reads `HKLM\Software\Voice\TextToSpeech\Engine` values + (value name = engine name, data = CLSID string), validates the GUID with + `FUN_635712e4`, then opens a *subkey named by the CLSID string*. +- `FUN_635758b1` (TTS mode-list ctor) reads `Dirty`; when non-zero/missing it + calls `CoCreateInstance(CLSID, ..., IID @635791c0)` and pulls modes via + vtable+0xC into 0xaf0-byte records, writes `Mode%d` (REG_BINARY), `Count`, + then clears `Dirty`. When `Dirty=0` it reads the cached `Mode%d` blobs. +- Practical rule: to add an engine, register the `Engine` value + the CLSID + `InprocServer32` + an engine subkey with `Dirty=1` (a missing subkey also + triggers enumeration). + +## Environment gotchas + +- Wine needs a working X display (the runtime creates windows). `xvfb-run -a` + proved flaky here; `setup/run.sh` starts a persistent `Xvfb :77` instead. +- Wine reports `err:ole:apartment_add_dll couldn't load in-process dll` when + the `InprocServer32` path does not exist. Check the file copy first + (this bit us: a bad `unshield` run created empty directories). +- Generate registry `.reg` files with `printf`, never `echo`: dash's `echo` + interprets `\2` in `...\Watson\2.0\...` as 0x02, corrupting the key so that + `wine reg import` is a silent no-op (it still returns 0). The wrapper rejects + a generated profile containing control characters. +- 32-bit engines require a 32-bit prefix (`WINEARCH=win32`). +- `sudo` on this machine requires a password; installs are manual. + +## Engine units (measured) + +- speed: MSTTS range 30-450, Flextalk 50-390; the natural rate is about 150 for every + voice (duration at 100/200/300 fits a natural value near 140-150). +- pitch: engine units, roughly linear in F0 near the natural point (~1.1 Hz per unit). + Natural values: about 100 (MSTTS), 106 (Keith Bell), 186 (Kerry Watson); Flextalk max 250. + Verified with the corrected DWORD registry: Kerry 0 st = 209 Hz, -6 st = 148 Hz, + +6 st = 250 Hz (clamped at max); Keith 0 st = 119 Hz, +6 st = 170 Hz. +- The wrapper maps semitones as `natural * 2^(st/12)` and clamps to the range. +- volume: available through `ITTSAttributes` (host flag `-V`). Flextalk is linear + (0-65535, default 65535; half is about -6 dB). MSTTS is non-linear (0 = silence, + 4096 about -24 dB, 65535+ full scale); treat it as on/off. +- Flextalk per-speaker profile values are applied at voice load; `Expression` was + verified audible (0/59/100 produce different waveforms at equal duration). Changing + them requires a registry edit and re-selecting the voice. + +## Decompiled corpora (Ghidra) + +- `/mnt/ghidra/decomp/msttssyn.dll/` (786 functions, engine) +- `/mnt/ghidra/decomp/speech.dll/` (1030 functions, SAPI4 runtime) diff --git a/microslopsam/docs/TTS-DESIGN.md b/microslopsam/docs/TTS-DESIGN.md new file mode 100644 index 0000000..e752a70 --- /dev/null +++ b/microslopsam/docs/TTS-DESIGN.md @@ -0,0 +1,477 @@ +# TTS-DESIGN.md — building a TTS engine for Veskslop + +This document is the integration contract for a text-to-speech engine that Veskslop will use. It is +written to be self-contained: an engine developer can work from this file alone. + +## How Veskslop uses TTS + +1. A line of text is queued (`POST /api/audio/speak` or the MCP `speak` tool). +2. The daemon asks the engine for an audio file (command mode) or audio bytes (HTTP mode). +3. `ffplay` decodes the file and plays it into the PulseAudio null sink `veskslop_tts`. +4. `veskslop_tts.monitor` is exposed as the source `veskslop_mic`. +5. The Vesktop client uses `veskslop_mic` as its microphone, so the call hears the speech. +6. While TTS plays and another person speaks, the daemon lowers the `veskslop_tts` sink volume to + 30% and restores it after. This is "ducking". The engine does not implement ducking. + +Lines are serialized: one synthesis, then playback, then the next line. There is no streaming +playback path and no partial-audio protocol. The engine returns a complete audio object. + +## Integration modes + +### HTTP provider (recommended) + +Set `VESKSLOP_TTS_API_URL`. Select the API provider with `VESKSLOP_TTS_PROVIDER=api` or the +**API TTS** choice in the TTS panel. The daemon sends: + +``` +POST +Content-Type: application/json +Authorization: Bearer (only when the key is set) + +{ "text": "Hello there.", "voice": "anna" } +``` + +The response body must be the audio bytes. `Content-Type` selects the file extension: `audio/mpeg` +or `audio/mp3` → `.mp3`, `audio/ogg` → `.ogg`, `audio/wav` or `audio/x-wav` → `.wav`, anything else +is treated as `.mp3`. Prefer `audio/wav`. + +A non-2xx response is an error. Return a short JSON or text body with the reason; the daemon logs +it and drops the line. + +Use the HTTP mode for any engine with a model load time. It keeps the model warm and avoids the +per-line process start cost. + +### Local command provider + +Set `VESKSLOP_TTS_CMD`. It takes precedence over the built-in espeak-ng engine. Veskslop replaces +these placeholders, splits the string on whitespace, and runs the result directly (no shell): + +| Placeholder | Replaced with | +| --- | --- | +| `{out}` | Output file path, extension `.wav`. Required. | +| `{text_file}` | A UTF-8 text file holding the line. | +| `{voice}` | The selected voice ID, or an empty string. | +| `{speed}` | The rate multiplier, or an empty string when unset. | +| `{pitch}` | The pitch offset in semitones, or an empty string when unset. | +| `{expression}` | The Flextalk Expression profile value, or an empty string when unset. | + +The process also receives `VESKSLOP_TTS_VOICE`, `VESKSLOP_TTS_SPEED`, +`VESKSLOP_TTS_PITCH`, and `VESKSLOP_TTS_EXPRESSION` when those are set. Write the audio to +`{out}` and exit 0. Non-zero exits are errors; stderr is captured into the log message, so keep it +short. + +Command mode is best for fast, stateless engines. It has no shell quoting: paths with spaces and +shell tricks cannot work. Use a small wrapper script if you need options. + +## Audio output requirements + +| Property | Recommendation | Notes | +| --- | --- | --- | +| Container | WAV (RIFF) | Any format ffmpeg decodes works, but WAV avoids decoder startup cost. | +| Codec | PCM signed 16-bit little-endian | `pcm_s16le`. | +| Sample rate | 48000 Hz | 44100 Hz is fine. Discord re-encodes to 48 kHz Opus; a 48 kHz source avoids one resample. | +| Channels | Mono | Discord voice is effectively mono. Stereo is accepted but adds no value. | +| Duration | One line of speech | No intro/outro music, no chimes. | +| Peak level | About -3 dBFS | Leave headroom; ducking and Discord gain must not clip. | +| Loudness | Consistent across lines | Aim for a stable integrated loudness (roughly -16 to -20 LUFS) so lines do not jump in volume. | +| Silence | Trim start to under 100 ms, keep roughly 20-50 ms of padding | Long leading silence delays the first audible word. | +| Edges | 5-10 ms fade in and out | Prevents clicks. | +| DC offset | None | Center the waveform. | + +Veskslop does not normalize, limit, or re-encode the audio. Playback is `ffplay` at 100% sink +volume, so the engine's levels are what Discord hears. + +## Voices + +A voice ID is an opaque string to Veskslop. Prefer short, stable, lowercase ASCII identifiers such +as `anna`, `sv-anna`, or `en-male-2`. The ID can contain letters, digits, `-`, `_`, and `+`. + +If the engine has multiple voices, expose them so a UI can list them: + +``` +GET /voices +[ + { + "id": "anna", + "name": "Anna", + "language": "sv", + "gender": "female", + "speed": { "min": 0.5, "max": 2.0, "default": 1.0 }, + "pitch": { "min": -12, "max": 12, "default": 0, "unit": "semitones" } + } +] +``` + +Note: Veskslop currently lists voices from espeak-ng only. A custom engine's voices appear after +Veskslop adds support for reading this endpoint (see "Veskslop changes needed"). Until then, set +the voice through the engine's own configuration and leave the Veskslop voice setting empty. + +## Speed and pitch + +Speed and pitch are engine settings, not Veskslop concepts today. Veskslop passes only `text`, +`voice`, and `provider`. No UI or preference exists for speed or pitch. + +Recommended semantics for the engine: + +- **speed**: a multiplier, `1.0` is normal, useful range `0.5` to `2.0`. Do not change pitch when + changing speed unless the user asks for it. +- **pitch**: semitones, `0` is the voice's natural pitch, useful range `-12` to `+12`. State the + unit in the `/voices` response so clients do not guess. + +Recommended HTTP shape (extras are optional; an engine may ignore them): + +``` +{ "text": "...", "voice": "anna", "speed": 1.15, "pitch": -2 } +``` + +Recommended command shape: `{speed}` and `{pitch}` are substituted when set, and the same values +are exported as `VESKSLOP_TTS_SPEED` and `VESKSLOP_TTS_PITCH`. Read either; the environment form +survives placeholder-free commands. + +### How Veskslop should add first-class settings + +Implemented on the Veskslop side (2026-09-15): + +- `tts` preferences carry `speed` (0.5–2.0) and `pitch` (-12–+12, semitones), both nullable. +- `/api/audio/speak` and the MCP `speak` tool accept optional `speed` and `pitch` that override + the preferences for that line. +- The command provider substitutes `{speed}`/`{pitch}`/`{expression}` and exports + `VESKSLOP_TTS_SPEED` / `VESKSLOP_TTS_PITCH` / `VESKSLOP_TTS_EXPRESSION` only when the value is set. +- The HTTP provider adds `speed`, `pitch`, and `expression` to the request body only when set. +- The TTS panel has speed presets, speed/pitch sliders, and an Expression slider that appears only + for voices that declare it (`expression` / `expression=NN` flags). Changes save to preferences. +- Command mode advertises voices through `VESKSLOP_TTS_VOICES`. Entries are separated by commas or + newlines and use `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or + `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch` (engine ignores + pitch) and `expression` / `expression=NN` (Flextalk profile parameter with its registry default). + `/api/audio/tts/voices` reports `supportsPitch` and `supportsExpression` / + `expressionDefault`; the TTS panel hides controls the voice does not support. While a custom + command is set, the engine's declared voices replace the espeak list. Reading `/voices` for the + HTTP provider is still future work. + +Do not encode settings into the voice string (for example `anna+speed1.2`). It breaks voice lists +and is hard to validate. + +## Text handling + +- UTF-8 input. Lines arrive trimmed and are at most 1000 characters. +- Do not parse or require SSML. Plain text is the contract. SSML support can be additive. +- Emoji, URLs, markdown, and code blocks can appear. Veskslop does not sanitize them. The engine + should not fail on an unknown character; skip or verbalize it as it prefers. +- Keep punctuation-based prosody. A period, comma, question mark, and newline are the only hints + for pacing. + +## Errors, limits, and behavior + +- One request = one complete audio object. No chunked playback, no callbacks. +- Synthesis blocks the queue. Target under about 500 ms for a short line; state a realistic budget + if the engine is slower. A warm HTTP server is the way to meet this. +- Requests are serialized by Veskslop, so the engine does not need a request queue, but it should + tolerate repeated back-to-back requests. +- The daemon never retries a failed line. Make failures visible in the response and stderr. +- Do not write to the `veskslop_tts` sink or the `veskslop_mic` source directly. The daemon owns + playback routing. + +## Optional: caching + +Veskslop does not cache audio. The engine may cache by `hash(text + voice + speed + pitch)` and +return the cached file or bytes. This is useful for repeated lines and for expensive models. + +## Veskslop changes needed + +Status as of 2026-09-15: + +1. `speed` and `pitch` are first-class: preferences, per-request overrides on `/api/audio/speak` + and MCP `speak`, env exports and placeholders for the command provider, request fields for the + HTTP provider, and two sliders in the TTS panel. +2. Command-mode voices come from `VESKSLOP_TTS_VOICES`. Reading the engine's `GET /voices` for the + HTTP provider is not implemented yet. Until then, declare voices with the env var or configure + the voice inside the wrapper. +3. No changes are needed for the audio format; the engine post-processes to the required output. + +Until `/voices` is wired, use defaults configured in the engine. + +## Acceptance checklist + +- [ ] Engine starts and answers on a stable URL or command. +- [ ] Output is mono PCM WAV at 48 kHz (or 44.1 kHz), peak near -3 dBFS. +- [ ] First audible word starts within about 500 ms of the request. +- [ ] Speech is clean: no clicks, no clipping, no long leading silence. +- [ ] Lines are consistent in loudness. +- [ ] `voice` accepts stable ASCII IDs; unknown IDs fall back to a default instead of failing. +- [ ] `speed` and `pitch` are optional and documented in `/voices`. +- [ ] Errors return a short reason and do not leave the engine wedged. +- [ ] No files are written outside the engine's own cache or the `{out}` path. + +--- + +## Clarifications from the engine side (MicroslopSam) + +_Added 2026-09-15 by the engine implementer. The contract above stands; these points +record what the engine needs to interpret, add, or hide._ + +### Output pipeline (WAV requirements) + +- Raw SAPI4 output is mono PCM s16le WAV but not at the target rate: the MSTTS voices + (Sam/Mary/Mike) render at 22050 Hz, the Flextalk voices at 11025 Hz. +- Raw Flextalk output **clips** (peak 32768, RMS about 22000) and MSTTS is much quieter, + so one shared level target is required. The engine post-processes every line: + resample to 48000 Hz, trim leading silence (<100 ms), 5-10 ms fades, DC removal, and + level normalization (peak about -3 dBFS, integrated loudness -16 to -20 LUFS). +- Implemented in `setup/veskslop-tts.sh` with ffmpeg: high-pass at 40 Hz, silence trim at + both ends (trim-start, reverse, trim-start, reverse), 8 ms fades, then a **fixed gain** + computed from an EBU R128 measurement (`ebur128=peak=true`): + `gain = min(-16 LUFS target, -3 dBFS true-peak clamp)`, then 48 kHz mono s16le. +- `loudnorm` was rejected: on short lines (under about 3 s) the loudness range is 0 and + dynamic mode attenuated a 2.8 s line to -37.9 LUFS instead of about -18 LUFS; two-pass + linear mode showed the same result. The fixed gain is steady for one-line utterances, + as Veskslop suggested. +- Verified: Keith Bell -18.4 LUFS, Kerry Watson -17.9 LUFS, Sam -20.6 LUFS; every file is + 48 kHz mono s16le at -3.0 dBFS true peak. The WAV handed to Veskslop is compliant; the + "Veskslop does not normalize" rule holds. + +### Voice IDs + +- Engine voice names are "Sam", "Mary", "Mike", "Keith Bell", "Kerry Watson", plus MSTTS + variants (telephone, hall, stadium, space) and TruVoice names that the engine lists but + has no installed data for. +- Assigned stable IDs: `sam`, `mary`, `mike`, `keith-bell`, `kerry-watson`; variants get + `mike-telephone`, `mike-hall`, `mary-stadium`, `mike-space`, and so on. Unknown IDs fall + back to `sam`. +- `/voices` (HTTP mode) reports `language: "en"`, gender, and the speed/pitch ranges below. + +### Speed and pitch mapping (measured 2026-09-15) + +- SAPI4 uses engine units, not multipliers or semitones. Measured on the dev host with a + correctly typed Flextalk install: + - MSTTS Sam: speed 30-450, natural about 150; pitch 50-200 with a real audible effect. + - Flextalk: speed 50-390, natural about 150 (durations at 100/200/300 fit a natural rate + near 140-150); pitch 50-250 works. Measured F0: Kerry 0 st = 209 Hz, -6 st = 148 Hz, + +6 st = 250 Hz (clamped); Keith 0 st = 119 Hz, +6 st = 170 Hz. +- **Correction to the earlier note**: the previous "Flextalk ignores pitch / natural 300" + finding was an artifact of the speaker registry values being stored as `REG_SZ`. The + engine requires `REG_DWORD` for numeric speaker values; with strings it synthesizes a + monotone buzz at maximum pitch. Fixed in `setup/setup-flextalk.sh`; no `no-pitch` flag + is needed. +- Mapping implemented in `setup/veskslop-tts.sh`: `speed 1.0` = natural rate 150 for every + voice, scaled by the multiplier and clamped to the engine range. `pitch` semitones scale + the voice's natural pitch unit by `2^(st/12)` and clamp to the range. Natural units are + about 100 (MSTTS), 106 (Keith Bell) and 186 (Kerry Watson). The engines' F0 response is + approximate, not semitone-calibrated. +- `{setting}` is unused; the wrapper reads `VESKSLOP_TTS_SPEED` and + `VESKSLOP_TTS_PITCH` from the environment when Veskslop later exports them. + +### Text handling + +- SAPI4 is ANSI/CP1252. The wrapper converts UTF-8 to CP1252 and maps emoji and + unmappable characters to spaces. The host passes `TTSDATAFLAG_TAGGED`, so SAPI4 control + tags in the text are enabled at the engine level; tagged input is not part of this + contract and is untested. Plain text (no SSML) is the contract. +- Lines up to 1000 characters are well within engine limits. + +### Other voice adjustments (beyond speed) + +All voices implement SAPI4 `ITTSAttributes`, so **pitch, speed and volume** exist at the +API level. The Veskslop contract exposes only speed and pitch; volume is available in the +host as `sapi4host -V ` and is not sent by Veskslop. + +- **Volume** (measured 2026-09-15): + - Flextalk: linear, range 0-65535, default 65535; half (32767) is about -6 dB, 0 is + silence. + - MSTTS: default is 0xFFFFFFFF; 0 is silence, but the response is not linear (for + example 4096 gives about -24 dB while 65535 and larger are full scale). Treat volume + as effectively on/off for MSTTS. +- **Flextalk per-speaker profile** (applied at voice load, not runtime-adjustable): + values live in `HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{GUID}` (see + `docs/NOTES.md` for the full list and defaults). `Pitch`/`Speaking Rate` set the natural + voice; **Expression** was verified to change the synthesis audibly (Keith Bell at + Expression 0 / 59 / 100 yields different waveforms with the same duration). Changing a + profile requires a registry edit and re-selecting the voice. +- **MSTTS voice character** is fixed by the `.vce`/`.cfg` data; the engine exposes no + per-voice profile in the registry. Its effect variants (telephone, in Hall, in Stadium, + in Space) are separate voice modes with separate data, exposed as distinct voice IDs + (`mike-telephone`, `mary-hall`, ...), not runtime effects. +- Not exposed by this integration: SSML, prosody/style tags, or SAPI4 attribute + extensions beyond pitch/speed/volume. + +### Flextalk profile parameters (how to implement) + +Each Flextalk speaker has a profile the engine reads when the voice is loaded. It lives in +the registry; **every numeric value must be `REG_DWORD`**: + + HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{11260610-C84A-11ce-AEC4-0000C0E9CB87} # Keith Bell + HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{000031CB-3699-551F-AED0-6B4893382D57} # Kerry Watson + +| Value (REG_DWORD unless noted) | Keith Bell | Kerry Watson | Measured effect | +| --- | --- | --- | --- | +| Pitch | 27 | 65 | Natural pitch (F0); the same scale `ITTSAttributes` shifts at runtime | +| Speaking Rate | 33 | 32 | Natural speech rate (about 150 wpm equivalent) | +| Volume | 100 | 100 | Output gain; 0 = silence | +| Expression | 59 | 100 | Audible; changes the waveform at equal duration | +| Prominences | 17 | 17 | Audible; strong prosody/emphasis effect (F0 median 122 -> 197 Hz at 100) | +| Head Size | 46 | 45 | Audible; brightness/level (peak 20001 -> 32205 at 100) | +| Lowering | 40 | 40 | Audible but subtle | +| Age | 30 | 30 | No measured effect on synthesis (informational) | +| Gender (REG_SZ) | "" | "" | No measured effect on synthesis (informational) | +| Mode Name / Speaker Name (REG_SZ) | "Keith Bell" | "Kerry Watson" | Mode name SAPI4 shows for the voice | + +How to apply a change: + +1. `setup/setup-flextalk.sh` writes these defaults, so re-running it restores them. +2. Profiles are read at voice load. The host loads the engine per request, so the next + synthesis after a registry change uses the new profile; no engine restart is needed. +3. Example (lower and slow Keith Bell), with `WINEPREFIX` pointing at the project prefix: + + wine reg add 'HKLM\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{11260610-C84A-11ce-AEC4-0000C0E9CB87}' \ + /v Pitch /t REG_DWORD /d 20 /f + wine reg add '...' /v "Speaking Rate" /t REG_DWORD /d 28 /f + + Never use `REG_SZ` for numeric values: the engine reads garbage and synthesizes a + monotone buzz at maximum pitch. +4. Not exposed by the Veskslop contract (which sends only text/voice/speed/pitch). If + profile control is wanted later, the clean place is the wrapper: read an env var such + as `VESKSLOP_TTS_PROFILE` (`key=value` list or JSON), write the registry values before + synthesis, then restore. That keeps the Veskslop interface unchanged. +5. Runtime `ITTSAttributes` pitch/speed/volume layer on top of the profile: the profile + defines the natural values, the attributes shift them per request. + +### Transport and latency + +- Installation is one command: `setup/install.sh` (system dependencies with sudo, wine + prefix + SAPI4 runtime + MSTTS voices, Flextalk extraction/registration, host build). + Re-running is safe; a fresh machine needs nothing else. +- Command provider, implemented as `setup/veskslop-tts.sh`: + `VESKSLOP_TTS_CMD="/setup/veskslop-tts.sh {out} {text_file}"` and + `VESKSLOP_TTS_VOICES="$(/setup/veskslop-tts.sh --voices)"`. The wrapper reads the + text file, transliterates UTF-8 to CP1252, starts or reuses an Xvfb when `DISPLAY` is + unset, calls the engine, then post-processes (trim, fades, measured fixed gain, 48 kHz + mono s16le). A stale private Xvfb is retried once. +- Measured on the dev host: engine-only synthesis is about 290 ms wall for Sam and about + 40 ms for Flextalk, including process start; the full wrapper adds the ffmpeg + measure/apply passes, about 630-880 ms per short line. That is above the 500 ms target; + the known optimization is a per-voice static gain (skip the measurement pass), expected + to land around 400-500 ms. +- Manual checks: `setup/run.sh -l`, `setup/run.sh -v "Keith Bell" -t "Hello." -o out.wav` + (raw engine WAV), and the wrapper above for normalized output. +- HTTP provider (later): a persistent process returning `audio/wav` bytes and `/voices`, + with optional caching by `hash(text + voice + speed + pitch)`. + +### Quirks the wrapper must hide + +- Flextalk numeric speaker settings must be `REG_DWORD`; `REG_SZ` values make the engine + synthesize a buzz. `setup/setup-flextalk.sh` writes them correctly. +- The mode list includes TruVoice names with no installed voice data; filter those out of + `/voices` (or mark them `unavailable`). +- Wine needs a working X display; the project helper starts a persistent Xvfb. + +### Licensing + +- The engines and voice data are proprietary Microsoft/AT&T/L&H artifacts. Keep this a + local integration; do not redistribute the DLLs or voice data with Veskslop. + +### Extra acceptance items for the engine + +- [x] Post-processed WAV is 48 kHz mono s16le, peak about -3 dBFS, no clipping, fades present. +- [x] Leading silence under 100 ms. +- [x] Voice ID fallback works; TruVoice entries filtered. +- [x] Latency recorded (Sam about 288 ms, Flextalk about 41 ms on the dev host). + +## Responses from the Veskslop side + +_Added 2026-09-15._ + +- **Env vars and placeholders**: `VESKSLOP_TTS_SPEED` and `VESKSLOP_TTS_PITCH` are exported only + when a value is set; `{speed}` and `{pitch}` substitute the same values. Reading the environment + is the safer option because it works even when the command has no placeholders. +- **Defaults**: a missing value means "engine default". Leave the sliders at 1.00x and 0 st in + Veskslop to keep the fields out of the request entirely. +- **`loudnorm` caution**: single-pass `loudnorm` can pump or over-amplify noise on short lines. + Set the target explicitly (`loudnorm=I=-16:TP=-1.5:LRA=11`) and verify a few lines with + `ebur128`. A fixed gain that hits about -3 dBFS peak with a consistent RMS is often steadier for + one-line utterances. Ducking lowers the sink to 30%, so keep peaks conservative and loudness + even between lines. +- **Display**: the daemon does not export `DISPLAY` to TTS children. The wrapper must start or + find its own Xvfb and set `DISPLAY` itself, as the project helper already does. Do not rely on + the client's display. +- **Engine selection**: the Speech settings and the TTS panel list every configured engine. The + command engine appears under `VESKSLOP_TTS_NAME` (default "custom engine") and is selectable + directly; Auto uses the command engine first. Selecting it hides the language row and shows the + `VESKSLOP_TTS_VOICES` list. +- **Start engine**: the Speech settings have a **Start engine** button (`POST /api/audio/tts/warm`, + MCP `start_tts_engine`) that runs one throwaway synthesis to warm the command engine (Xvfb, Wine, + host). Keep the wrapper able to handle a short sentence; the warm-up uses "The quick brown fox + jumps over the lazy dog." and reports the elapsed milliseconds. Two-pass `loudnorm` must not fail + on that input. +- **Lead-in silence**: Veskslop prepends a short silence at playback time (`VESKSLOP_TTS_LEAD_MS`, + default 150 ms) so Discord's voice pipeline is ready before the first syllable. It is added with + `adelay` in the player, not in the audio file. The engine needs no change and can keep its + 5–10 ms edge fades. +- **Voice declarations**: list only usable IDs in `VESKSLOP_TTS_VOICES`. Veskslop hides the + espeak list while a custom command is set, so a missing declaration means an empty voice list, + not a fallback. The list accepts commas or newlines. Optional flags: `no-pitch` (hide the pitch + control) and `expression` / `expression=NN` (show the Expression slider and use NN as its + default). +- **Flextalk pitch**: works now that the speaker registry values are `REG_DWORD`; no flag is + needed and Veskslop shows pitch for Keith Bell and Kerry Watson. +- **Expression profile**: implemented for the Flextalk voices. Veskslop stores `tts.expression` + (0-100, nullable), sends it per line (`{expression}` / `VESKSLOP_TTS_EXPRESSION` / the HTTP body), + and shows the slider only for voices flagged `expression=NN`. A null value leaves the registry + untouched, so the voice keeps its engine default. +- **Licensing**: keep the engine DLLs and voice data local and out of any repository; Veskslop + never bundles or redistributes them. + +## Current integration path (Veskslop side, 2026-09-15) + +Everything below is implemented. The engine stays unchanged; the wrapper and the daemon do the work. + +### Voice metadata + +`VESKSLOP_TTS_VOICES` entries may carry flags after the language field: + + id|Name|gender|language|flags + +Flags are `+`-separated: + +- `no-pitch` - hide the pitch control for this voice. +- `expression=NN` - Flextalk profile voice; NN is the default Expression value. +- `defaults=pitch:27:rate:33:prominences:17:head_size:46:lowering:40:expression:59` - + per-voice engine defaults. The UI shows "default" at these values and snaps to them + within +/-2 units. + +### Wrapper contract (`microslopsam/setup/veskslop-tts.sh`) + +Inputs: `{out}`, `{text_file}`, optional positional voice/speed/pitch/expression, and: + +| Env | Meaning | +| --- | --- | +| `VESKSLOP_TTS_PROFILE` | `pitch:27:...`/`key=value,...` overrides; the wrapper writes the full profile (defaults + overrides) to the Wine registry with `wine reg import` before every synthesis. | +| `VESKSLOP_TTS_HIGHPASS` | High-pass Hz, 40-400 (default 40). | +| `VESKSLOP_TTS_ECHO` | Mono echo 0-100 (`aecho`), default 0. | +| `VESKSLOP_TTS_WORDS` / `VESKSLOP_TTS_WORD_GAP_MS` | Word-by-word synthesis with silence gaps (Flextalk only). | + +Placeholders `{profile}`, `{highpass}`, `{echo}`, `{words}`, `{word_gap_ms}` mirror the env vars. +`wine regedit` fails headlessly but returns 0: use `wine reg import`. A missing profile value +must always write the engine default so no stale registry state survives. + +### Daemon and UI path + +1. Preferences (`tts`): `profile` (changed keys only), `profilePresets` (5 named slots), + `highpass`, `echo`, `words`, `wordGapMs`, plus `speed`, `pitch`, `expression`, `voice`. +2. `/api/audio/speak` and MCP `speak` accept all of them per line; the request overrides the + preferences and empty fields are omitted so the engine default applies. +3. The TTS panel opens the **Flextalk voice profile** window (movable, recentered on open) with + profile sliders, named presets, radio-kit controls, and test lines (Attention, Laboratory, + Critical, Health, Radiation, plus two persisted custom lines). Test buttons post the same + options as `speak`, so they audition the current profile. + +### Known pitfalls + +- The daemon runs from `tsx`: restart it after protocol, preference, or metadata changes. +- Static web files can be cached; hard-reload the browser after UI changes. +- `Number(null)` is 0: preference normalizers must treat null/undefined/"" as unset. +- The renderer log spam on a dead client is throttled in the bridge; background polls fail with + `CORE_UNAVAILABLE` and retry. +- Generate the Flextalk profile `.reg` with `printf`, never `echo`: dash's `echo` interprets + `\2` in `...\Watson\2.0\...` as an escape (0x02), so the key is corrupted and + `wine reg import` becomes a silent no-op (it still returns 0). The wrapper now rejects a + generated profile that contains control characters. This was the 2026-09-15 profile bug. diff --git a/microslopsam/host/sapi4host.c b/microslopsam/host/sapi4host.c new file mode 100644 index 0000000..53048b0 --- /dev/null +++ b/microslopsam/host/sapi4host.c @@ -0,0 +1,352 @@ +/* + * sapi4host.c — minimal SAPI4 text-to-speech host. + * + * Builds as: + * - Winelib (native Linux ELF): winegcc -m32 -o sapi4host sapi4host.c ... + * - PE (runs under wine): i686-w64-mingw32-gcc -o sapi4host.exe ... + * + * Usage: + * sapi4host -l + * sapi4host -v -t -o [-p ] [-s ] + * + * Flow (see reference/speech.h): + * CLSID_TTSEnumerator -> ITTSFind::Find/Select + * CLSID_AudioDestFile -> IAudioFile::Set (WAV) + * ITTSAttributes::PitchSet/SpeedSet + * ITTSNotifySink -> wait for AudioStop + * ITTSCentral::AudioReset / TextData + * + * Derived from the documented SAPI4 flow used by GaryOderNichts/SAPI4-CLI + * and TETYYS/SAPI4 (both small, permissive examples). + */ + +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile LONG g_done = 0; +static volatile LONG g_started = 0; + +/* ------------------------------------------------------------------ */ +/* ITTSNotifySink */ +/* ------------------------------------------------------------------ */ + +static HRESULT STDMETHODCALLTYPE N_QueryInterface(ITTSNotifySinkA *This, + REFIID riid, LPVOID *ppv) +{ + *ppv = NULL; + if (IsEqualIID(riid, &IID_IUnknown) || + IsEqualIID(riid, &IID_ITTSNotifySink)) { + *ppv = (LPVOID)This; + return S_OK; + } + return E_NOINTERFACE; +} + +static ULONG STDMETHODCALLTYPE N_AddRef(ITTSNotifySinkA *This) +{ + (void)This; + return 1; +} + +static ULONG STDMETHODCALLTYPE N_Release(ITTSNotifySinkA *This) +{ + (void)This; + return 1; +} + +static HRESULT STDMETHODCALLTYPE N_AttribChanged(ITTSNotifySinkA *This, DWORD id) +{ + (void)This; (void)id; + return S_OK; +} + +static HRESULT STDMETHODCALLTYPE N_AudioStart(ITTSNotifySinkA *This, QWORD ts) +{ + (void)This; (void)ts; + InterlockedExchange(&g_started, 1); + return S_OK; +} + +static HRESULT STDMETHODCALLTYPE N_AudioStop(ITTSNotifySinkA *This, QWORD ts) +{ + (void)This; (void)ts; + InterlockedExchange(&g_done, 1); + return S_OK; +} + +static HRESULT STDMETHODCALLTYPE N_Visual(ITTSNotifySinkA *This, QWORD ts, + CHAR ipa, CHAR engine, + DWORD hints, PTTSMOUTH mouth) +{ + (void)This; (void)ts; (void)hints; (void)mouth; + if (ipa) + fprintf(stderr, "phoneme ipa=0x%02x engine=0x%02x\n", + (unsigned char)ipa, (unsigned char)engine); + return S_OK; +} + +static ITTSNotifySinkAVtbl g_notifyVtbl = { + N_QueryInterface, + N_AddRef, + N_Release, + N_AttribChanged, + N_AudioStart, + N_AudioStop, + N_Visual +}; + +static ITTSNotifySinkA g_notify = { &g_notifyVtbl }; + +/* ------------------------------------------------------------------ */ +/* helpers */ +/* ------------------------------------------------------------------ */ + +static void list_voices(void) +{ + PITTSENUM e = NULL; + TTSMODEINFO info; + HRESULT hr = CoCreateInstance(&CLSID_TTSEnumerator, NULL, CLSCTX_ALL, + &IID_ITTSEnum, (void **)&e); + if (FAILED(hr) || !e) { + fprintf(stderr, "cannot create TTS enumerator (0x%08lx)\n", + (unsigned long)hr); + return; + } + for (;;) { + memset(&info, 0, sizeof(info)); + if (e->lpVtbl->Next(e, 1, &info, NULL) != S_OK) + break; + printf("%s\n", info.szModeName); + } + e->lpVtbl->Release(e); +} + +static HRESULT find_voice(const char *name, TTSMODEINFO *out) +{ + PITTSFIND f = NULL; + TTSMODEINFO want, got; + HRESULT hr; + + hr = CoCreateInstance(&CLSID_TTSEnumerator, NULL, CLSCTX_ALL, + &IID_ITTSFind, (void **)&f); + if (FAILED(hr) || !f) + return hr; + + memset(&want, 0, sizeof(want)); + strncpy(want.szModeName, name, sizeof(want.szModeName) - 1); + memset(&got, 0, sizeof(got)); + + hr = f->lpVtbl->Find(f, &want, NULL, &got); + f->lpVtbl->Release(f); + if (FAILED(hr)) + return hr; + + *out = got; + return S_OK; +} + +static void usage(void) +{ + fprintf(stderr, + "usage: sapi4host -l\n" + " sapi4host -v -t -o " + " [-p pitch] [-s speed] [-V volume]\n"); +} + +/* ------------------------------------------------------------------ */ +/* main */ +/* ------------------------------------------------------------------ */ + +int main(int argc, char **argv) +{ + const char *voice = NULL, *text = NULL, *out = NULL; + int pitch = -1, speed = -1; + long volume = -1; + int i; + TTSMODEINFO mode; + + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-l") || !strcmp(argv[i], "--list")) { + CoInitialize(NULL); + list_voices(); + CoUninitialize(); + return 0; + } else if (!strcmp(argv[i], "-v")) voice = argv[++i]; + else if (!strcmp(argv[i], "-t")) text = argv[++i]; + else if (!strcmp(argv[i], "-o")) out = argv[++i]; + else if (!strcmp(argv[i], "-p")) pitch = atoi(argv[++i]); + else if (!strcmp(argv[i], "-s")) speed = atoi(argv[++i]); + else if (!strcmp(argv[i], "-V")) volume = strtol(argv[++i], NULL, 0); + else { usage(); return 2; } + } + if (!voice || !text || !out) { usage(); return 2; } + + if (FAILED(CoInitialize(NULL))) { + fprintf(stderr, "CoInitialize failed\n"); + return 1; + } + + if (FAILED(find_voice(voice, &mode))) { + fprintf(stderr, "voice not found: %s\navailable:\n", voice); + list_voices(); + CoUninitialize(); + return 1; + } + printf("voice: %s\n", mode.szModeName); + + PIAUDIOFILE af = NULL; + HRESULT hr = CoCreateInstance(&CLSID_AudioDestFile, NULL, CLSCTX_ALL, + &IID_IAudioFile, (void **)&af); + if (FAILED(hr) || !af) { + fprintf(stderr, "cannot create audio destination (0x%08lx)\n", + (unsigned long)hr); + CoUninitialize(); + return 1; + } + + PITTSCENTRAL central = NULL; + { + PITTSFIND f = NULL; + hr = CoCreateInstance(&CLSID_TTSEnumerator, NULL, CLSCTX_ALL, + &IID_ITTSFind, (void **)&f); + if (FAILED(hr) || !f) { + fprintf(stderr, "cannot create TTS find (0x%08lx)\n", + (unsigned long)hr); + af->lpVtbl->Release(af); + CoUninitialize(); + return 1; + } + hr = f->lpVtbl->Select(f, mode.gModeID, ¢ral, (LPUNKNOWN)af); + f->lpVtbl->Release(f); + if (FAILED(hr) || !central) { + fprintf(stderr, "voice select failed (0x%08lx)\n", + (unsigned long)hr); + af->lpVtbl->Release(af); + CoUninitialize(); + return 1; + } + } + + af->lpVtbl->RealTimeSet(af, (WORD)TTSATTR_MAXREALTIME); + + WCHAR wout[MAX_PATH]; + MultiByteToWideChar(CP_ACP, 0, out, -1, wout, MAX_PATH); + hr = af->lpVtbl->Set(af, wout, 1); + if (FAILED(hr)) { + fprintf(stderr, "cannot open output file (0x%08lx)\n", + (unsigned long)hr); + central->lpVtbl->Release(central); + af->lpVtbl->Release(af); + CoUninitialize(); + return 1; + } + + PITTSATTRIBUTES attr = NULL; + hr = central->lpVtbl->QueryInterface(central, &IID_ITTSAttributes, + (void **)&attr); + if (FAILED(hr) || !attr) { + fprintf(stderr, "no ITTSAttributes (0x%08lx)\n", (unsigned long)hr); + central->lpVtbl->Release(central); + af->lpVtbl->Release(af); + CoUninitialize(); + return 1; + } + + /* Pitch, speed and volume are optional: when unset, leave the engine's own + voice profile untouched (probing/setting would override it). */ + if (pitch >= 0 || speed >= 0 || volume >= 0) { + WORD pmin = 0, pmax = 0; + DWORD smin = 0, smax = 0; + + if (pitch >= 0) { + attr->lpVtbl->PitchSet(attr, TTSATTR_MINPITCH); + attr->lpVtbl->PitchGet(attr, &pmin); + attr->lpVtbl->PitchSet(attr, TTSATTR_MAXPITCH); + attr->lpVtbl->PitchGet(attr, &pmax); + if (pitch < (int)pmin) pitch = pmin; + if (pitch > (int)pmax) pitch = pmax; + attr->lpVtbl->PitchSet(attr, (WORD)pitch); + printf("pitch: %d (%d..%d)\n", pitch, pmin, pmax); + } + if (speed >= 0) { + attr->lpVtbl->SpeedSet(attr, TTSATTR_MINSPEED); + attr->lpVtbl->SpeedGet(attr, &smin); + attr->lpVtbl->SpeedSet(attr, TTSATTR_MAXSPEED); + attr->lpVtbl->SpeedGet(attr, &smax); + if (speed < (int)smin) speed = smin; + if (speed > (int)smax) speed = smax; + attr->lpVtbl->SpeedSet(attr, (DWORD)speed); + printf("speed: %d (%lu..%lu)\n", + speed, (unsigned long)smin, (unsigned long)smax); + } + if (volume >= 0) { + DWORD vdef = 0; + attr->lpVtbl->VolumeGet(attr, &vdef); + attr->lpVtbl->VolumeSet(attr, (DWORD)volume); + printf("volume: %ld (default %lu)\n", + volume, (unsigned long)vdef); + } + } + + { + DWORD regkey = 0; + hr = central->lpVtbl->Register(central, &g_notify, + IID_ITTSNotifySink, ®key); + if (FAILED(hr)) { + fprintf(stderr, "Register failed (0x%08lx)\n", (unsigned long)hr); + attr->lpVtbl->Release(attr); + central->lpVtbl->Release(central); + af->lpVtbl->Release(af); + CoUninitialize(); + return 1; + } + } + + { + SDATA data; + data.dwSize = (DWORD)(strlen(text) + 1); + data.pData = (PVOID)text; + + central->lpVtbl->AudioReset(central); + hr = central->lpVtbl->TextData(central, CHARSET_TEXT, + TTSDATAFLAG_TAGGED, data, NULL, + IID_ITTSBufNotifySink); + if (FAILED(hr)) { + fprintf(stderr, "TextData failed (0x%08lx)\n", (unsigned long)hr); + } else { + DWORD start = GetTickCount(); + while (!g_done) { + MSG msg; + while (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + if (!g_started && GetTickCount() - start > 30000) { + fprintf(stderr, "timed out waiting for audio start\n"); + break; + } + if (GetTickCount() - start > 120000) { + fprintf(stderr, "timed out waiting for audio stop\n"); + break; + } + Sleep(10); + } + } + } + + af->lpVtbl->Flush(af); + + attr->lpVtbl->Release(attr); + central->lpVtbl->Release(central); + af->lpVtbl->Release(af); + CoUninitialize(); + printf("done%s\n", g_done ? "" : " (incomplete)"); + return g_done ? 0 : 1; +} diff --git a/microslopsam/setup/inf2reg.py b/microslopsam/setup/inf2reg.py new file mode 100755 index 0000000..24eea96 --- /dev/null +++ b/microslopsam/setup/inf2reg.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Convert INF AddReg sections into a Windows/Wine .reg file. + +Example: + inf2reg.py --section APIReg --define 49000='C:\\windows\\speech' \ + spchapi/SPCHAPI.INF > spchapi.reg +""" +import argparse +import re +import sys + +ROOTS = { + "HKCR": "HKEY_CLASSES_ROOT", + "HKLM": "HKEY_LOCAL_MACHINE", + "HKCU": "HKEY_CURRENT_USER", + "HKU": "HKEY_USERS", +} + + +def expand(text, defines): + def repl(m): + key = m.group(1) + if key == "25": + return defines.get("WINDOWS", r"C:\windows") + return defines.get(key, m.group(0)) + + return re.sub(r"%(\w+)%", repl, text) + + +def reg_type(flags, data): + if not flags: + return "REG_SZ" + try: + f = int(flags, 16) if flags.lower().startswith("0x") else int(flags) + except ValueError: + return "REG_SZ" + if f & 0x00020000: + return "REG_EXPAND_SZ" + if f & 0x00010000: + return "REG_MULTI_SZ" + if f & 0x00000001: + return "REG_BINARY" + return "REG_SZ" + + +def reg_escape(value): + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def unquote(token): + token = token.strip() + if len(token) >= 2 and token[0] == '"' and token[-1] == '"': + return token[1:-1] + return token + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("inf") + ap.add_argument("--section", action="append", required=True, + help="AddReg section name (repeatable)") + ap.add_argument("--define", action="append", default=[], + help="ID=value, e.g. 49000=C:\\windows\\speech") + args = ap.parse_args() + + defines = {"WINDOWS": r"C:\windows"} + for item in args.define: + k, v = item.split("=", 1) + defines[k] = v + + wanted = set(args.section) + section = None + out = ["REGEDIT4", ""] + + with open(args.inf, "r", encoding="latin-1") as fh: + for raw in fh: + line = raw.strip() + if not line or line.startswith(";"): + continue + m = re.match(r"^\[([^\]]+)\]$", line) + if m: + section = m.group(1) + continue + if section not in wanted: + continue + parts = [p.strip() for p in line.split(",")] + if len(parts) < 4: + continue + root, key, name, flags = parts[0], parts[1], parts[2], parts[3] + data = ",".join(parts[4:]) if len(parts) > 4 else "" + if root.upper() not in ROOTS: + continue + data = expand(unquote(data), defines) + name = expand(unquote(name), defines) + if "%InstallPrompt%" in name: + continue + if not data: + continue + rtype = reg_type(flags.strip(), data) + keypath = "\\".join( + p for p in (ROOTS[root.upper()], expand(key, defines)) if p + ) + value = data if rtype == "REG_BINARY" else '"%s"' % reg_escape(data) + out.append("[%s]" % keypath) + out.append("%s=%s" % ('@' if not name else '"%s"' % name, value)) + out.append("") + + sys.stdout.write("\n".join(out) + "\n") + + +if __name__ == "__main__": + main() diff --git a/microslopsam/setup/install-deps.sh b/microslopsam/setup/install-deps.sh new file mode 100755 index 0000000..bbaba46 --- /dev/null +++ b/microslopsam/setup/install-deps.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Install Wine + 32-bit Winelib toolchain + helpers. Debian/Ubuntu. +# Run as root: sudo setup/install-deps.sh +set -eu + +[ "$(id -u)" = "0" ] || { echo "run as root: sudo $0" >&2; exit 1; } + +export DEBIAN_FRONTEND=noninteractive + +if ! dpkg --print-foreign-architectures | grep -qx i386; then + echo "== enabling i386 architecture" + dpkg --add-architecture i386 +fi + +apt-get update + +for p in \ + wine wine64 wine32:i386 \ + wine64-tools \ + libwine-dev libwine-dev:i386 \ + gcc-multilib libc6-dev-i386 \ + mingw-w64 \ + xvfb unshield +do + if apt-cache policy "$p" 2>/dev/null | grep -q "Candidate: [0-9]"; then + echo "== install $p" + apt-get install -y --no-install-recommends "$p" || echo "!! FAILED: $p" + else + echo "-- unavailable, skipped: $p" + fi +done + +echo +echo "versions:" +wine --version 2>/dev/null || echo "wine missing" +winegcc --version 2>/dev/null | head -1 || echo "winegcc missing" +i686-w64-mingw32-gcc --version 2>/dev/null | head -1 || echo "mingw missing" diff --git a/microslopsam/setup/install.sh b/microslopsam/setup/install.sh new file mode 100755 index 0000000..2c06cdd --- /dev/null +++ b/microslopsam/setup/install.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# One-shot installer for MicroslopSam. Runs all setup steps, idempotently: +# 1. system dependencies (Wine, Winelib, helpers; needs sudo) +# 2. wine prefix + SAPI4 runtime + MSTTS voices + registration +# 3. Flextalk extraction + voices + registration +# 4. host build (Winelib and PE) +# 5. quick voice check +# +# Usage: setup/install.sh +# Re-running is safe. +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +ROOT=$(CDPATH= cd -- "$HERE/.." && pwd) +ASSETS=${ASSETS:-$ROOT/../Microsoft-Sam-Mary-Mike-TruVoice-WSAPI4} +FLEXTALK_DIR=${FLEXTALK_DIR:-$ROOT/../flextalk} + +log() { printf '\n== %s\n' "$*"; } + +# 1) system dependencies ----------------------------------------------------- +if [ "$(id -u)" = "0" ]; then + log "installing system dependencies" + "$HERE/install-deps.sh" +else + log "installing system dependencies (sudo)" + sudo "$HERE/install-deps.sh" +fi + +# 2) SAPI4 runtime + MSTTS voices -------------------------------------------- +log "setting up wine prefix, SAPI4 runtime and MSTTS voices" +"$HERE/setup-prefix.sh" "$ASSETS" + +# 3) Flextalk voices ---------------------------------------------------------- +EXTRACTED="$FLEXTALK_DIR/extracted/TTS_Executables" +if [ ! -d "$EXTRACTED/EngineDLLs" ]; then + CAB="" + for c in "$FLEXTALK_DIR/installer/flextalk/data1.cab" \ + "$ASSETS/flextalk/data1.cab"; do + if [ -f "$c" ]; then CAB="$c"; break; fi + done + if [ -z "$CAB" ]; then + echo "Flextalk data1.cab not found; set FLEXTALK_DIR to the archive dir" >&2 + exit 1 + fi + command -v unshield >/dev/null 2>&1 || { + echo "unshield is required to extract the Flextalk installer" >&2 + exit 1 + } + log "extracting Flextalk installer ($CAB)" + rm -rf "$FLEXTALK_DIR/extracted" + unshield -O -d "$FLEXTALK_DIR/extracted" x "$CAB" >/dev/null +fi +log "installing Flextalk voices" +"$HERE/setup-flextalk.sh" "$EXTRACTED" + +# 4) host build --------------------------------------------------------------- +log "building the host" +mkdir -p "$ROOT/build" +winegcc -m32 -O2 -o "$ROOT/build/sapi4host-winlib" "$ROOT/host/sapi4host.c" \ + -I"$ROOT/../reference" -lole32 -luser32 -ladvapi32 +if command -v i686-w64-mingw32-gcc >/dev/null 2>&1; then + i686-w64-mingw32-gcc -O2 -static -o "$ROOT/build/sapi4host.exe" \ + "$ROOT/host/sapi4host.c" -I"$ROOT/../reference" \ + -lole32 -luser32 -ladvapi32 -luuid +fi + +# 5) check -------------------------------------------------------------------- +log "installed voices" +"$HERE/run.sh" -l || true + +cat </dev/null 2>&1 & + sleep 1 + fi + export DISPLAY=:77 +fi + +HOST="$ROOT/build/sapi4host-winlib.exe" +if [ ! -x "$HOST" ]; then + HOST="$ROOT/build/sapi4host.exe" +fi +if [ ! -x "$HOST" ]; then + echo "no host binary; build first (see README)" >&2 + exit 1 +fi + +exec "$HOST" "$@" diff --git a/microslopsam/setup/setup-flextalk.sh b/microslopsam/setup/setup-flextalk.sh new file mode 100755 index 0000000..16dc6bc --- /dev/null +++ b/microslopsam/setup/setup-flextalk.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# Install AT&T FlexTalk 2.1 (Keith Bell, Kerry Watson) into the wine prefix. +# +# Source: extracted InstallShield cabinet, e.g. +# flextalk/extracted/TTS_Executables +# Usage: setup/setup-flextalk.sh [source-dir] +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +ROOT=$(CDPATH= cd -- "$HERE/.." && pwd) +SRC=${1:-$ROOT/../flextalk/extracted/TTS_Executables} + +[ -d "$SRC/EngineDLLs" ] || { echo "source not found: $SRC" >&2; exit 1; } + +export WINEPREFIX=${WINEPREFIX:-$HOME/.wine-microslopsam} +export WINEARCH=win32 + +if [ -z "${DISPLAY:-}" ]; then + WRUN="xvfb-run -a wine" +else + WRUN="wine" +fi + +DEST="$WINEPREFIX/drive_c/FlexTalk" +echo "== files -> $DEST" +mkdir -p "$DEST" +cp -a "$SRC/." "$DEST/" + +REG="$HERE/out-flextalk.reg" +cat > "$REG" <<'EOF' +REGEDIT4 + +[HKEY_LOCAL_MACHINE\Software\Voice\TextToSpeech\Engine] +"AT&T FlexTalk"="{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}" + +[HKEY_LOCAL_MACHINE\Software\Voice\TextToSpeech\Engine\{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}] +"Dirty"=dword:00000001 + +[HKEY_CLASSES_ROOT\CLSID\{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}] +@="AT&T FlexTalk" + +[HKEY_CLASSES_ROOT\CLSID\{FFE78E00-C69C-11CE-AEC1-0000C0E9CB87}\InprocServer32] +@="C:\\FlexTalk\\EngineDLLs\\FlexTalk.dll" +"ThreadingModel"="Apartment" + +[HKEY_CLASSES_ROOT\CLSID\{778EAAE2-499F-11CF-B0BA-444553540000}\InprocServer32] +@="C:\\FlexTalk\\EngineDLLs\\FlexTalk.dll" +"ThreadingModel"="Apartment" + +[HKEY_LOCAL_MACHINE\Software\AT&T\Watson\2.0\FlexTalk\2.0\Directories] +"RootDirectory"="C:\\FlexTalk\\" + +[HKEY_LOCAL_MACHINE\Software\AT&T\Watson\2.0\FlexTalk\2.0\Dictionaries] +"StandardDictionary"="2 ft_sdict.dll" +"ExceptionDictionary"="0 ft_udict.dll" +"HomographDictionary"="1 ft_hgrap.dll" +"ProperNameDictionary"="3 ft_namsa.dll" +"LetterToSoundDictionary"="4 ft_l2s.dll" + +[HKEY_LOCAL_MACHINE\Software\AT&T\Watson\2.0\FlexTalk\2.0\Options] +"Gender"="Male" +"SampleRate"="11025" + +[HKEY_LOCAL_MACHINE\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{11260610-C84A-11ce-AEC4-0000C0E9CB87}] +"Age"=dword:0000001e +"Expression"=dword:0000003b +"Gender"="" +"Head Size"=dword:0000002e +"Lowering"=dword:00000028 +"Mode Name"="Keith Bell" +"Pitch"=dword:0000001b +"Prominences"=dword:00000011 +"Speaker Name"="Keith Bell" +"Speaking Rate"=dword:00000021 +"Volume"=dword:00000064 + +[HKEY_LOCAL_MACHINE\Software\AT&T\Watson\2.0\FlexTalk\2.0\Speakers\{000031CB-3699-551F-AED0-6B4893382D57}] +"Age"=dword:0000001e +"Expression"=dword:00000064 +"Gender"="" +"Head Size"=dword:0000002d +"Lowering"=dword:00000028 +"Mode Name"="Kerry Watson" +"Pitch"=dword:00000041 +"Prominences"=dword:00000011 +"Speaker Name"="Kerry Watson" +"Speaking Rate"=dword:00000020 +"Volume"=dword:00000064 +EOF + +echo "== registry" +$WRUN regedit /S "$REG" 2>/dev/null || true + +echo "== done; list voices:" +echo " $WRUN $ROOT/host/sapi4host.exe -l" diff --git a/microslopsam/setup/setup-prefix.sh b/microslopsam/setup/setup-prefix.sh new file mode 100755 index 0000000..9729a62 --- /dev/null +++ b/microslopsam/setup/setup-prefix.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Build a 32-bit Wine prefix, install the SAPI4 runtime, the MSTTS engine and +# the MSTTS voices, and register everything. +# +# Usage: setup/setup-prefix.sh [path-to-asset-repo] +# Default asset repo: ../Microsoft-Sam-Mary-Mike-TruVoice-WSAPI4 +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +ROOT=$(CDPATH= cd -- "$HERE/.." && pwd) +ASSETS=${1:-$ROOT/../Microsoft-Sam-Mary-Mike-TruVoice-WSAPI4} + +[ -d "$ASSETS/spchapi" ] || { echo "asset dir not found: $ASSETS" >&2; exit 1; } + +export WINEPREFIX=${WINEPREFIX:-$HOME/.wine-microslopsam} +export WINEARCH=win32 + +if [ -z "${DISPLAY:-}" ]; then + WRUN="xvfb-run -a wine" +else + WRUN="wine" +fi + +echo "== prefix: $WINEPREFIX" +mkdir -p "$WINEPREFIX" +$WRUN wineboot -u + +SPEECH_DIR="$WINEPREFIX/drive_c/windows/speech" +ENGINE_DIR="$WINEPREFIX/drive_c/Program Files/Common Files/microsoft shared/SpeechEngines/TTS" +mkdir -p "$SPEECH_DIR" "$ENGINE_DIR" + +echo "== runtime -> $SPEECH_DIR" +cp "$ASSETS"/spchapi/SPEECH.DLL "$SPEECH_DIR/" + +echo "== engine + voices -> $ENGINE_DIR" +cp "$ASSETS"/msttsl/msttssyn.dll "$ENGINE_DIR/" +cp "$ASSETS"/msttsl/*.vce "$ASSETS"/msttsl/*.cfg "$ENGINE_DIR/" + +echo "== registry" +python3 "$HERE/inf2reg.py" --section APIReg \ + --define "49000=C:\\windows\\speech" \ + "$ASSETS/spchapi/SPCHAPI.INF" > "$HERE/out-spchapi.reg" +python3 "$HERE/inf2reg.py" --section TTSReg \ + --define "49000=C:\\Program Files\\Common Files\\microsoft shared\\SpeechEngines\\TTS" \ + "$ASSETS/msttsl/mstts.inf" > "$HERE/out-mstts.reg" + +$WRUN regedit /S "$HERE/out-spchapi.reg" +$WRUN regedit /S "$HERE/out-mstts.reg" + +echo "== done" +echo "list voices: $WRUN $ROOT/host/sapi4host.exe -l" diff --git a/microslopsam/setup/veskslop-tts.sh b/microslopsam/setup/veskslop-tts.sh new file mode 100755 index 0000000..6b96170 --- /dev/null +++ b/microslopsam/setup/veskslop-tts.sh @@ -0,0 +1,308 @@ +#!/bin/sh +# Veskslop TTS command-provider wrapper for MicroslopSam. +# +# Recommended Veskslop configuration: +# VESKSLOP_TTS_CMD="/setup/veskslop-tts.sh {out} {text_file}" +# VESKSLOP_TTS_VOICES="$(setup/veskslop-tts.sh --voices)" +# +# Inputs: +# $1 {out} output WAV path +# $2 {text_file} UTF-8 text file +# $3... optional: voice id, speed multiplier, pitch semitones, expression +# env VESKSLOP_TTS_VOICE, VESKSLOP_TTS_SPEED, VESKSLOP_TTS_PITCH, +# VESKSLOP_TTS_EXPRESSION +# VESKSLOP_TTS_PROFILE Flextalk profile: key=value,key=value +# keys: pitch, rate, volume, expression, +# prominences, head_size, lowering +# VESKSLOP_TTS_HIGHPASS high-pass Hz (40-400, default 40) +# VESKSLOP_TTS_ECHO mono echo amount 0-100 (default 0) +# VESKSLOP_TTS_WORDS 1 = synthesize each word, merge (Flextalk) +# VESKSLOP_TTS_WORD_GAP_MS silence between words in ms (0-500) +# +# Output: 48 kHz mono s16le WAV, trimmed, faded, loudness-normalized. +set -eu + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +ROOT=$(CDPATH= cd -- "$HERE/.." && pwd) +export WINEPREFIX=${WINEPREFIX:-$HOME/.wine-microslopsam} +export WINEARCH=win32 + +# VESKSLOP_TTS_VOICES entries: id|Name|gender|language[|flags] +# `expression=NN` marks a Flextalk voice with an Expression profile parameter and +# its registry default. Expression is applied at voice load, so the wrapper writes +# it into the Wine registry before synthesis. +VOICES="sam|Sam|male|en,mary|Mary|female|en,mike|Mike|male|en,keith-bell|Keith Bell|male|en|expression=59+defaults=pitch:27:rate:33:prominences:17:head_size:46:lowering:40:expression:59,kerry-watson|Kerry Watson|female|en|expression=100+defaults=pitch:65:rate:32:prominences:17:head_size:45:lowering:40:expression:100,mike-telephone|Mike (for Telephone)|male|en,mary-telephone|Mary (for Telephone)|female|en,mike-hall|Mike in Hall|male|en,mike-stadium|Mike in Stadium|male|en,mike-space|Mike in Space|male|en,mary-hall|Mary in Hall|female|en,mary-stadium|Mary in Stadium|female|en,mary-space|Mary in Space|female|en" + +if [ "${1:-}" = "--voices" ]; then + echo "$VOICES" + exit 0 +fi + +OUT=${1:?missing out path} +TEXTFILE=${2:?missing text file} +VOICE=${VESKSLOP_TTS_VOICE:-${3:-sam}} +SPEED=${VESKSLOP_TTS_SPEED:-${4:-}} +PITCH=${VESKSLOP_TTS_PITCH:-${5:-}} +EXPRESSION=${VESKSLOP_TTS_EXPRESSION:-${6:-}} +PROFILE=${VESKSLOP_TTS_PROFILE:-} +HIGHPASS=${VESKSLOP_TTS_HIGHPASS:-40} +ECHO=${VESKSLOP_TTS_ECHO:-0} +WORDS=${VESKSLOP_TTS_WORDS:-0} +WORD_GAP=${VESKSLOP_TTS_WORD_GAP_MS:-120} + +case "$VOICE" in + sam) ENGVOICE="Sam" ;; + mary) ENGVOICE="Mary" ;; + mike) ENGVOICE="Mike" ;; + keith-bell) ENGVOICE="Keith Bell" ;; + kerry-watson) ENGVOICE="Kerry Watson" ;; + mike-telephone) ENGVOICE="Mike (for Telephone)" ;; + mary-telephone) ENGVOICE="Mary (for Telephone)" ;; + mike-hall) ENGVOICE="Mike in Hall" ;; + mike-stadium) ENGVOICE="Mike in Stadium" ;; + mike-space) ENGVOICE="Mike in Space" ;; + mary-hall) ENGVOICE="Mary in Hall" ;; + mary-stadium) ENGVOICE="Mary in Stadium" ;; + mary-space) ENGVOICE="Mary in Space" ;; + *) ENGVOICE="Sam" ;; +esac + +# Flextalk speakers have a registry profile; MSTTS voices do not. +case "$ENGVOICE" in + "Keith Bell") SPEAKER_GUID="{11260610-C84A-11ce-AEC4-0000C0E9CB87}"; DEF_PITCH=27; DEF_RATE=33; DEF_HEAD=46 ;; + "Kerry Watson") SPEAKER_GUID="{000031CB-3699-551F-AED0-6B4893382D57}"; DEF_PITCH=65; DEF_RATE=32; DEF_HEAD=45 ;; + *) SPEAKER_GUID=""; DEF_PITCH=100; DEF_RATE=150; DEF_HEAD=45 ;; +esac + +# Engine units (measured). Speed is words/min, natural rate near 150 for every +# voice. Pitch units differ per voice: the engine's natural pitch corresponds to +# about 100 (MSTTS), 106 (Keith Bell) and 186 (Kerry Watson). Semitones scale +# that natural value by 2^(st/12) and clamp to the engine range. +case "$ENGVOICE" in + "Keith Bell") SMAX=390; PMAX=250; NAT_PITCH=106 ;; + "Kerry Watson") SMAX=390; PMAX=250; NAT_PITCH=186 ;; + *) SMAX=450; PMAX=200; NAT_PITCH=100 ;; +esac +SET_ARGS="" +if [ -n "$SPEED" ]; then + S=$(awk -v m="$SPEED" 'BEGIN{printf "%d", 150*m+0.5}') + [ "$S" -lt 50 ] && S=50 + [ "$S" -gt "$SMAX" ] && S=$SMAX + SET_ARGS="$SET_ARGS -s $S" +fi +if [ -n "$PITCH" ]; then + P=$(awk -v st="$PITCH" -v nat="$NAT_PITCH" 'BEGIN{printf "%d", nat*exp(log(2)*st/12)+0.5}') + [ "$P" -lt 50 ] && P=50 + [ "$P" -gt "$PMAX" ] && P=$PMAX + SET_ARGS="$SET_ARGS -p $P" +fi + +# The daemon does not export DISPLAY; keep a private Xvfb for the engine. +start_xvfb() { + if ! [ -S /tmp/.X11-unix/X77 ]; then + Xvfb :77 -screen 0 1024x768x24 >/dev/null 2>&1 & + sleep 1 + fi +} +if [ -z "${DISPLAY:-}" ]; then + start_xvfb + export DISPLAY=:77 +fi + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +RAW="$TMP/raw.wav" +TXT="$TMP/text.cp1252" + +# Apply the Flextalk profile before synthesis. Profiles are read at voice load and +# the host loads the engine per request, so the values are written every call: the +# engine defaults plus any VESKSLOP_TTS_PROFILE overrides. A missing value therefore +# restores the default. +if [ -n "$SPEAKER_GUID" ]; then + P_PITCH=$DEF_PITCH + P_RATE=$DEF_RATE + P_VOLUME=100 + P_PROM=17 + P_HEAD=$DEF_HEAD + P_LOWER=40 + P_EXPR="" + + if [ -n "$PROFILE" ]; then + OLDIFS=$IFS + IFS=',' + for pair in $PROFILE; do + IFS='=' + # shellcheck disable=SC2086 + set -- $pair + IFS=',' + case "$1" in + pitch) P_PITCH=$2 ;; + rate) P_RATE=$2 ;; + volume) P_VOLUME=$2 ;; + expression) P_EXPR=$2 ;; + prominences) P_PROM=$2 ;; + head_size|headSize) P_HEAD=$2 ;; + lowering) P_LOWER=$2 ;; + esac + done + IFS=$OLDIFS + fi + + if [ -n "$EXPRESSION" ]; then P_EXPR=$EXPRESSION; fi + if [ -z "$P_EXPR" ]; then + case "$ENGVOICE" in + "Keith Bell") P_EXPR=59 ;; + *) P_EXPR=100 ;; + esac + fi + + for value in "$P_PITCH" "$P_RATE" "$P_VOLUME" "$P_EXPR" "$P_PROM" "$P_HEAD" "$P_LOWER"; do + case "$value" in + *[!0-9]*|"") + echo "profile values must be numbers" >&2 + exit 1 + ;; + esac + done + + REG_KEY="HKLM\\Software\\AT&T\\Watson\\2.0\\FlexTalk\\2.0\\Speakers\\$SPEAKER_GUID" + # Use printf, not echo: dash's echo interprets \2 in the key path as an + # escape (STX), which corrupts the key and makes the import a silent no-op. + { + printf 'Windows Registry Editor Version 5.00\n\n' + printf '[%s]\n' "$REG_KEY" + printf '"Pitch"=dword:%08x\n' "$P_PITCH" + printf '"Speaking Rate"=dword:%08x\n' "$P_RATE" + printf '"Volume"=dword:%08x\n' "$P_VOLUME" + printf '"Expression"=dword:%08x\n' "$P_EXPR" + printf '"Prominences"=dword:%08x\n' "$P_PROM" + printf '"Head Size"=dword:%08x\n' "$P_HEAD" + printf '"Lowering"=dword:%08x\n' "$P_LOWER" + } > "$TMP/profile.reg" + + # Guard against silent reg-import failures (a malformed key would corrupt + # only the import, and reg.exe can still return 0). + if LC_ALL=C grep -q '[[:cntrl:]]' "$TMP/profile.reg"; then + echo "generated Flextalk profile is malformed" >&2 + exit 1 + fi + + if ! command -v wine >/dev/null 2>&1; then + echo "wine is required to set the Flextalk profile" >&2 + exit 1 + fi + if ! wine reg import "$TMP/profile.reg" >/dev/null 2>&1; then + echo "could not write the Flextalk profile" >&2 + exit 1 + fi +fi + +# SAPI4 is ANSI/CP1252; transliterate UTF-8 (emoji become '?'). +if command -v iconv >/dev/null 2>&1; then + iconv -f UTF-8 -t WINDOWS-1252//TRANSLIT "$TEXTFILE" > "$TXT" 2>/dev/null \ + || cp "$TEXTFILE" "$TXT" +else + cp "$TEXTFILE" "$TXT" +fi + +HOST="$ROOT/build/sapi4host-winlib.exe" +if [ ! -x "$HOST" ]; then HOST="$ROOT/build/sapi4host.exe"; fi +if [ ! -x "$HOST" ]; then echo "engine host not built" >&2; exit 1; fi +if ! command -v ffmpeg >/dev/null 2>&1; then echo "ffmpeg not found" >&2; exit 1; fi + +run_host() { + # shellcheck disable=SC2086 + "$HOST" -v "$ENGVOICE" -t "$2" -o "$1" $SET_ARGS >/dev/null 2>"$TMP/err" +} + +# A stale private Xvfb can break audio-destination creation; retry once. +try_run_host() { + if run_host "$1" "$2"; then return 0; fi + + if [ "${DISPLAY:-}" = ":77" ]; then + pkill -f "Xvfb :77" 2>/dev/null || true + rm -f /tmp/.X11-unix/X77 + start_xvfb + if run_host "$1" "$2"; then return 0; fi + fi + + head -c 400 "$TMP/err" >&2 + exit 1 +} + +# Word mode (Flextalk): synthesize each word and merge with silence gaps, which +# gives the staccato radio cadence the Half-Life VOX/HEV voices are known for. +if [ "$WORDS" = "1" ] && [ -n "$SPEAKER_GUID" ]; then + GAP=$WORD_GAP + case "$GAP" in *[!0-9]*|"") GAP=120 ;; esac + [ "$GAP" -gt 500 ] && GAP=500 + + : > "$TMP/list.txt" + INDEX=0 + for word in $(cat "$TXT"); do + INDEX=$((INDEX + 1)) + try_run_host "$TMP/w$INDEX.wav" "$word" + echo "file '$TMP/w$INDEX.wav'" >> "$TMP/list.txt" + + if [ "$GAP" -gt 0 ]; then + ffmpeg -hide_banner -loglevel error -f lavfi -i "anullsrc=r=11025:cl=mono" \ + -t "$(awk -v g="$GAP" 'BEGIN{printf "%.3f", g/1000}')" -c:a pcm_s16le -y "$TMP/sil.wav" \ + >/dev/null 2>&1 + echo "file '$TMP/sil.wav'" >> "$TMP/list.txt" + fi + done + + [ "$INDEX" -gt 0 ] || { echo "no words to speak" >&2; exit 1; } + if ! ffmpeg -hide_banner -loglevel error -f concat -safe 0 -i "$TMP/list.txt" -c copy -y "$RAW" \ + >/dev/null 2>&1; then + echo "could not merge the words" >&2 + exit 1 + fi +else + try_run_host "$RAW" "$(cat "$TXT")" +fi +[ -s "$RAW" ] || { echo "engine produced no audio" >&2; exit 1; } + +# Filter chain: adjustable high-pass, silence trim, fades, optional mono echo. +case "$HIGHPASS" in *[!0-9]*|"") HIGHPASS=40 ;; esac +[ "$HIGHPASS" -lt 40 ] && HIGHPASS=40 +[ "$HIGHPASS" -gt 400 ] && HIGHPASS=400 + +case "$ECHO" in *[!0-9]*|"") ECHO=0 ;; esac +[ "$ECHO" -gt 100 ] && ECHO=100 + +SIL="silenceremove=start_periods=1:start_duration=0.05:start_silence=0.03:start_threshold=-50dB" +PRE="highpass=f=$HIGHPASS,$SIL,afade=t=in:st=0:d=0.008,areverse,$SIL,afade=t=in:st=0:d=0.008,areverse" +if [ "$ECHO" -gt 0 ]; then + DECAY=$(awk -v e="$ECHO" 'BEGIN{printf "%.2f", 0.2 + e * 0.006}') + PRE="$PRE,aecho=0.8:0.85:60:${DECAY}" +fi + +# Fixed-gain normalization: single-pass loudnorm misbehaves on short lines +# (LRA=0 causes heavy attenuation), so measure EBU R128 integrated loudness and +# true peak, then apply one gain clamped by the -3 dBFS peak target. +M=$(ffmpeg -hide_banner -nostats -i "$RAW" -af "$PRE,ebur128=peak=true" \ + -f null - 2>&1 || true) +I=$(printf '%s' "$M" | sed -n 's/ *I: *\(-\{0,1\}[0-9.]*\) *LUFS.*/\1/p' | tail -1) +TP=$(printf '%s' "$M" | sed -n 's/ *Peak: *\(-\{0,1\}[0-9.]*\) *dBFS.*/\1/p' | tail -1) +if [ -n "$I" ] && [ -n "$TP" ]; then + GAIN=$(awk -v i="$I" -v tp="$TP" 'BEGIN{ + g = -16 - i; + lim = -3 - tp; + if (lim < g) g = lim; + if (g > 30) g = 30; + if (g < -20) g = -20; + printf "%.2f", g + }') +else + GAIN=0 +fi + +TMPOUT="$TMP/out.wav" +if ! ffmpeg -hide_banner -nostats -y -i "$RAW" -af "$PRE,volume=${GAIN}dB" \ + -ar 48000 -ac 1 -c:a pcm_s16le "$TMPOUT" >/dev/null 2>"$TMP/ff"; then + head -c 400 "$TMP/ff" >&2 + exit 1 +fi +[ -s "$TMPOUT" ] || { echo "post-processing produced no audio" >&2; exit 1; } +mv -f "$TMPOUT" "$OUT" diff --git a/poc/package.json b/poc/package.json index 9a2db9d..afa14b4 100644 --- a/poc/package.json +++ b/poc/package.json @@ -15,7 +15,8 @@ "test:trigger": "tsx src/test-trigger.ts", "cdp:dispatch": "tsx src/cdp-dispatch.ts", "test:cache": "tsx src/test-cache.ts", - "test:audio": "tsx src/test-audio.ts" + "test:audio": "tsx src/test-audio.ts", + "audio:doctor": "bash scripts/audio-doctor.sh" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", diff --git a/poc/public/app.js b/poc/public/app.js index 766f2c9..aaed2ec 100644 --- a/poc/public/app.js +++ b/poc/public/app.js @@ -26,8 +26,11 @@ const state = { visibleGuildsSaved: null, timeZoneOverride: localStorage.getItem("veskslop-timezone") ?? "", speech: { provider: null, language: null, voice: null }, + profileDefaults: {}, + expressionDefault: null, scope: "guild", speaking: new Set(), + speakingFade: new Map(), people: new Map(), favourites: { users: [], channels: [] }, favouriteUsers: [], @@ -661,16 +664,19 @@ function renderVisibleGuilds() { function voiceMarks(member) { const marks = el("span", { class: "vk-marks" }); - const speaking = state.speaking.has(member.id); + const muted = member.suppress || member.mute || member.selfMute; - // Fixed order: audio → video → share. - if (speaking) { - const levels = el("span", { class: "vk-levels", title: "speaking" }); - for (let index = 0; index < 4; index++) levels.append(el("span")); - marks.append(levels); - } + // The bars are always in the layout; they scale up while speaking and + // decay back to nothing when the voice stops. + const classes = ["vk-levels"]; + if (!muted && state.speaking.has(member.id)) classes.push("speaking"); + if (!muted && state.speakingFade.has(member.id)) classes.push("stopping"); - if (member.suppress || member.mute || member.selfMute) { + const levels = el("span", { class: classes.join(" "), title: "speaking" }); + for (let index = 0; index < 4; index++) levels.append(el("span")); + marks.append(levels); + + if (muted) { const title = member.suppress ? "muted by server" : member.mute ? "server-muted" : "self-muted"; marks.append(el("span", { class: "neutral", title, html: VK.icon("micSlash") })); } @@ -702,6 +708,53 @@ function memberNameNode(member, className = "vk-name") { }); } +/* Speaking marks. Bars stay on until an explicit stop, a mute, or a channel + change: Discord's speaking events are not reliable enough for an idle decay. + The fade runs the equalizer animation inside a shrinking container, so the + amplitude drops instead of the whole block collapsing at full height. */ + +const SPEAKING_FADE_MS = 750; + +function renderSpeakingViews() { + renderVoiceRail(); + renderPeople(); + renderRightVoice(); + renderRightFavourites(); + renderCallStrip(); +} + +function scheduleSpeakingRender() { + const now = Date.now(); + if (now - state.speakingRenderAt < 250) return; + state.speakingRenderAt = now; + renderSpeakingViews(); +} + +function beginSpeaking(userId) { + const fade = state.speakingFade.get(userId); + if (fade) { + clearTimeout(fade); + state.speakingFade.delete(userId); + } + + const wasSpeaking = state.speaking.has(userId); + state.speaking.add(userId); + + if (!wasSpeaking) scheduleSpeakingRender(); +} + +function endSpeaking(userId) { + if (!state.speaking.delete(userId)) return; + + const fadeTimer = setTimeout(() => { + state.speakingFade.delete(userId); + renderSpeakingViews(); + }, SPEAKING_FADE_MS); + state.speakingFade.set(userId, fadeTimer); + + scheduleSpeakingRender(); +} + function trackPeople(members) { for (const member of members ?? []) { if (!member?.id || member.id === state.user?.id) continue; @@ -1208,6 +1261,7 @@ function toggleTts() { const panel = $("#tts-panel"); panel.hidden = !panel.hidden; $("#tts-target").textContent = `as ${state.user?.username ?? "account"} → ${channelLabel()}`; + if (!panel.hidden) void refreshSpeech(); } /* Right column */ @@ -1664,10 +1718,10 @@ function fillDeviceSelect(select, devices, defaultLabel) { async function refreshAudio() { try { const status = await VK.api.get("/api/audio"); - state.audioRecording = !!status.recording; + state.audioRecording = !!status.capturing; $("#audio-status").textContent = status.available - ? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.recording ? " · recording" : ""}` + ? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.capturing ? " · capturing" : ""}` : (status.error ?? "unavailable"); const hintRow = $("#audio-hint-row"); @@ -1678,7 +1732,15 @@ async function refreshAudio() { $("#audio-tts").textContent = `local ${tts.local ? "yes" : "no"} · api ${ tts.api ? "yes" : "no" } · player ${status.player ? "yes" : "no"} · queue ${status.queue ?? 0}`; - $("#audio-capture").textContent = status.recording ? "Stop capture" : "Start capture"; + $("#audio-capture").textContent = status.capturing ? "Stop capture" : "Start capture"; + + const duck = $("#tts-duck"); + if (duck) { + const on = status.duckEnabled !== false; + duck.classList.toggle("vk-chip-on", on); + duck.classList.toggle("vk-chip-off", !on); + duck.textContent = on ? "duck on voice" : "duck off"; + } } catch (error) { $("#audio-status").textContent = error.message; } @@ -1690,11 +1752,17 @@ async function refreshAudio() { /* Speech settings (server-side preferences). */ async function refreshSpeech() { + let engines = []; + let defaultProvider = "local"; + try { const status = await VK.api.get("/api/audio"); const tts = status.tts ?? {}; const stt = status.stt ?? {}; + engines = tts.engines ?? []; + defaultProvider = tts.defaultProvider ?? "local"; + $("#speech-status").textContent = `${tts.engine ?? "no local engine"}${tts.api ? " + api" : ""}`; $("#speech-stt").textContent = stt.local ? `local ready${stt.engine ? ` (${stt.engine})` : ""}` @@ -1705,36 +1773,77 @@ async function refreshSpeech() { // the daemon status is optional here } - let groups = []; - try { - groups = await VK.api.get("/api/audio/tts/voices"); - } catch { - // voices need a local engine - } - try { const prefs = await VK.api.get("/api/preferences"); - state.speech = prefs.tts ?? { provider: null, language: null, voice: null }; - - const languageSelect = $("#speech-language"); - languageSelect.replaceChildren(el("option", { value: "", text: "Auto" })); - for (const group of groups) { - languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` })); - } - languageSelect.value = state.speech.language ?? ""; - - fillVoiceOptions(groups, state.speech.language, state.speech.voice); - - $("#speech-provider").value = state.speech.provider ?? ""; - $("#tts-provider").value = state.speech.provider ?? ""; + state.speech = prefs.tts ?? { provider: null, language: null, voice: null, speed: null, pitch: null }; } catch { // preferences are optional } + + const selected = resolveEngine(state.speech.provider, engines, defaultProvider); + const selectedEngine = engines.find(engine => engine.id === selected); + const hasLanguage = selectedEngine?.language !== false; + + fillEngineOptions($("#speech-provider"), engines, state.speech.provider ?? selected); + fillEngineOptions($("#tts-provider"), engines, state.speech.provider ?? selected); + + let groups = []; + try { + groups = await VK.api.get(`/api/audio/tts/voices?engine=${encodeURIComponent(selected)}`); + } catch { + // voices need an engine + } + + const languageSelect = $("#speech-language"); + languageSelect.replaceChildren(el("option", { value: "", text: "Auto" })); + for (const group of groups) { + languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` })); + } + languageSelect.value = state.speech.language ?? ""; + $("#speech-language-row").hidden = !hasLanguage; + + const voices = voiceList(groups, hasLanguage ? state.speech.language : null, hasLanguage); + fillVoiceSelect($("#speech-voice"), voices, state.speech.voice); + fillVoiceSelect($("#tts-voice"), voices, state.speech.voice); + syncVoiceControls(voices, state.speech.voice); + fillSpeechSliders(state.speech); } -function fillVoiceOptions(groups, language, selected) { - const select = $("#speech-voice"); - const voices = groups.find(group => group.id === language)?.voices ?? []; +/** Resolve the engine that will actually run: provider override first, then the auto default. */ +function resolveEngine(provider, engines, defaultProvider) { + if (provider === "espeak" || provider === "command" || provider === "api") return provider; + if (defaultProvider === "espeak" || defaultProvider === "command" || defaultProvider === "api") return defaultProvider; + + const available = id => engines.some(engine => engine.id === id && engine.available); + if (available("command")) return "command"; + if (available("espeak")) return "espeak"; + return "api"; +} + +function fillEngineOptions(select, engines, value) { + if (!select) return; + + select.replaceChildren(el("option", { value: "", text: "Auto" })); + for (const engine of engines) { + const option = el("option", { + value: engine.id, + text: engine.available ? engine.name : `${engine.name} (unavailable)` + }); + option.disabled = !engine.available; + select.append(option); + } + + select.value = value; +} + +function voiceList(groups, language, hasLanguage) { + if (!hasLanguage) return (groups ?? []).flatMap(group => group.voices ?? []); + if (!language) return []; + return groups?.find(group => group.id === language)?.voices ?? []; +} + +function fillVoiceSelect(select, voices, selected) { + if (!select) return; select.replaceChildren(el("option", { value: "", text: "Default" })); for (const voice of voices) { @@ -1745,15 +1854,38 @@ function fillVoiceOptions(groups, language, selected) { select.value = selected && voices.some(voice => voice.id === selected) ? selected : ""; } -async function saveSpeech() { +function fillVoiceOptions(groups, language, selected) { + fillVoiceSelect($("#speech-voice"), voiceList(groups, language, !!language), selected); +} + +async function saveSpeech(extra = {}) { const provider = $("#speech-provider").value || null; - const language = $("#speech-language").value || null; + const language = $("#speech-language-row").hidden + ? (state.speech.language ?? null) + : $("#speech-language").value || null; const voice = $("#speech-voice").value || null; + const speed = Number($("#tts-speed").value); + const pitch = Number($("#tts-pitch").value); + const expressionEl = $("#tts-expression"); + const expression = + expressionEl.dataset.dirty === "1" || typeof state.speech.expression === "number" + ? Number(expressionEl.value) + : null; try { - const prefs = await VK.api.post("/api/preferences", { tts: { provider, language, voice } }); - state.speech = prefs.tts ?? { provider: null, language: null, voice: null }; + const prefs = await VK.api.post("/api/preferences", { + tts: { provider, language, voice, speed, pitch, expression, ...effectOptions(), ...extra } + }); + state.speech = prefs.tts ?? { + provider: null, + language: null, + voice: null, + speed: null, + pitch: null, + expression: null + }; $("#tts-provider").value = state.speech.provider ?? ""; + fillSpeechSliders(state.speech); await refreshSpeech(); toast("Speech settings saved", "ok"); } catch (error) { @@ -1761,6 +1893,205 @@ async function saveSpeech() { } } +async function warmEngine() { + const button = $("#speech-warm"); + const engine = $("#speech-provider").value || null; + const label = $("#speech-provider").selectedOptions[0]?.textContent ?? "engine"; + + button.disabled = true; + button.textContent = "Starting…"; + + try { + const result = await VK.api.post("/api/audio/tts/warm", engine ? { engine } : {}); + toast(result.warmed ? `${label} ready in ${result.ms} ms` : `${label} needs no warm-up`, "ok"); + $("#speech-status").textContent = result.warmed ? `${label} ready (${result.ms} ms)` : label; + } catch (error) { + toast(error.message, "error"); + } finally { + button.disabled = false; + button.textContent = "Start engine"; + } +} + +function fillSpeechSliders(tts) { + const speed = typeof tts?.speed === "number" ? tts.speed : 1; + const pitch = typeof tts?.pitch === "number" ? tts.pitch : 0; + + $("#tts-speed").value = Number.isFinite(speed) ? speed : 1; + $("#tts-pitch").value = Number.isFinite(pitch) ? pitch : 0; + syncSpeechSliderLabels(); +} + +function syncSpeechSliderLabels() { + const speed = Number($("#tts-speed").value); + const pitch = Number($("#tts-pitch").value); + const pitchSlider = $("#tts-pitch"); + + $("#tts-speed-value").textContent = `${speed.toFixed(2)}x`; + $("#tts-pitch-value").textContent = pitchSlider.disabled ? "n/a" : `${pitch > 0 ? "+" : ""}${pitch} st`; + + const expressionSlider = $("#tts-expression"); + if (expressionSlider) { + const explicit = expressionSlider.dataset.dirty === "1" || expressionSlider.dataset.stored === "1"; + $("#tts-expression-value").textContent = explicit ? expressionSlider.value : "default"; + } + + for (const button of $$("#tts-speed-presets .vk-chip")) { + const active = Math.abs(Number(button.dataset.speed) - speed) < 0.001; + button.classList.toggle("vk-chip-on", active); + button.classList.toggle("vk-chip-off", !active); + } +} + +/* Flextalk profile controls (VOX/HEV kit): profile values, effects, word mode. */ + +const PROFILE_SPEC = [ + ["pitch", 0, 120], + ["rate", 0, 100], + ["prominences", 0, 100], + ["head_size", 0, 100], + ["lowering", 0, 100] +]; + +function buildProfileSliders() { + const root = $("#tts-profile-sliders"); + if (!root) return; + + for (const [key, min, max] of PROFILE_SPEC) { + const slider = el("input", { type: "range", min: String(min), max: String(max), step: "1" }); + slider.id = `tts-profile-${key}`; + slider.dataset.key = key; + const value = el("span", { class: "vk-mono vk-dim", text: "default" }); + + root.append( + el( + "div", + { class: "vk-tts-profile-row" }, + el("span", { class: "vk-micro-label", text: key.replace("_", " ") }), + slider, + value + ) + ); + + slider.addEventListener("input", () => { + const engineDefault = state.profileDefaults?.[key]; + const snapped = typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2; + + if (snapped) { + slider.value = engineDefault; + slider.dataset.dirty = ""; + value.textContent = "default"; + } else { + slider.dataset.dirty = "1"; + value.textContent = slider.value; + } + }); + slider.addEventListener("change", () => void saveSpeech()); + } +} + +function profileFromControls() { + const profile = {}; + for (const [key] of PROFILE_SPEC) { + const slider = $(`#tts-profile-${key}`); + if (slider?.dataset.dirty === "1") profile[key] = Number(slider.value); + } + return profile; +} + +function fillProfileControls() { + const stored = state.speech.profile ?? {}; + fillProfileSlots(); + for (const [key] of PROFILE_SPEC) { + const slider = $(`#tts-profile-${key}`); + if (!slider) continue; + + if (typeof stored[key] === "number") { + slider.value = stored[key]; + slider.dataset.dirty = "1"; + } else { + const engineDefault = state.profileDefaults?.[key]; + if (typeof engineDefault === "number") slider.value = engineDefault; + slider.dataset.dirty = ""; + } + const value = slider.parentElement?.querySelector(".vk-mono"); + if (value) value.textContent = slider.dataset.dirty === "1" ? slider.value : "default"; + } + + const highpass = Number(state.speech.highpass); + $("#tts-highpass").value = Number.isFinite(highpass) && state.speech.highpass !== null ? highpass : 40; + $("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`; + + const echo = Number(state.speech.echo); + $("#tts-echo").value = Number.isFinite(echo) && state.speech.echo !== null ? echo : 0; + $("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off"; + + $("#tts-words").checked = state.speech.words === true; + const gap = Number(state.speech.wordGapMs); + $("#tts-word-gap").value = Number.isFinite(gap) && state.speech.wordGapMs !== null ? gap : 120; + $("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`; +} + +function fillProfileSlots() { + const select = $("#tts-profile-slot"); + if (!select) return; + + const presets = state.speech.profilePresets ?? []; + const current = select.value || "0"; + select.replaceChildren( + ...Array.from({ length: 5 }, (_, index) => { + const preset = presets[index]; + const label = preset ? `${index + 1} · ${preset.name ?? "profile"}` : `${index + 1} · empty`; + return el("option", { value: String(index), text: label }); + }) + ); + select.value = current; + + const preset = presets[Number(select.value)]; + const name = $("#tts-profile-name"); + if (name) name.value = preset?.name ?? ""; +} + +function effectOptions() { + return { + profile: profileFromControls(), + highpass: Number($("#tts-highpass").value), + echo: Number($("#tts-echo").value), + words: $("#tts-words").checked, + wordGapMs: Number($("#tts-word-gap").value) + }; +} +function syncVoiceControls(voices, selected) { + const voice = (voices ?? []).find(entry => entry.id === selected); + const pitchable = !voice || voice.supportsPitch !== false; + + $("#tts-pitch-control").hidden = !pitchable; + $("#tts-pitch").disabled = !pitchable; + + const expressive = !!voice?.supportsExpression; + state.profileDefaults = voice?.profileDefaults ?? {}; + state.expressionDefault = typeof voice?.expressionDefault === "number" ? voice.expressionDefault : null; + if (expressive) fillProfileControls(); + + { + const slider = $("#tts-expression"); + const pref = state.speech.expression; + + if (typeof pref === "number") { + slider.value = pref; + } else if (typeof voice.expressionDefault === "number") { + slider.value = voice.expressionDefault; + } else { + slider.value = 50; + } + + slider.dataset.stored = typeof pref === "number" ? "1" : "0"; + delete slider.dataset.dirty; + } + + syncSpeechSliderLabels(); +} + /* Browser audio (puppeting) — separate from the bot's own audio devices. */ async function refreshBrowserAudio() { @@ -1793,19 +2124,38 @@ async function refreshBrowserAudio() { } async function grantBrowserAudio() { + if (await ensureBrowserAudio()) toast("Browser audio enabled", "ok"); +} + +/** Get the browser ready for playback: permission, device labels, and a live + * AudioContext. Must run inside the click so the context is not suspended. */ +async function ensureBrowserAudio() { if (!navigator.mediaDevices?.getUserMedia) { toast("Browser audio needs HTTPS or localhost", "error"); - return; + return false; } try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); for (const track of stream.getTracks()) track.stop(); - toast("Browser audio enabled", "ok"); - await refreshBrowserAudio(); } catch (error) { toast(`Could not enable browser audio: ${error.message}`, "error"); + return false; } + + if (!state.clientAudio.ctx) { + state.clientAudio.ctx = new AudioContext({ sampleRate: 48000 }); + applyBrowserSink(); + } + + if (state.clientAudio.ctx.state === "suspended") { + await state.clientAudio.ctx.resume().catch(error => { + console.warn("[client-audio] could not resume the audio context:", error); + }); + } + + await refreshBrowserAudio(); + return true; } function applyBrowserSink() { @@ -1824,6 +2174,8 @@ function playPcm(arrayBuffer) { } const context = state.clientAudio.ctx; + if (context.state === "suspended") void context.resume(); + const samples = new Int16Array(arrayBuffer); const buffer = context.createBuffer(1, samples.length, 48000); const data = buffer.getChannelData(0); @@ -1846,7 +2198,10 @@ function updateClientAudioButton() { if (!button) return; button.classList.toggle("active", state.clientAudio.on); - button.textContent = state.clientAudio.on ? "Client audio on" : "Client audio"; + const label = button.querySelector(".vk-btn-text"); + const text = state.clientAudio.on ? "Client audio on" : "Client audio"; + if (label) label.textContent = text; + else button.textContent = text; const status = $("#browser-audio-status"); if (status) status.textContent = state.clientAudio.on ? "on" : "off"; @@ -1911,9 +2266,17 @@ function stopClientAudio() { updateClientAudioButton(); } -function toggleClientAudio() { - if (state.clientAudio.on || state.clientAudio.ws) stopClientAudio(); - else startClientAudio(); +async function toggleClientAudio() { + if (state.clientAudio.on || state.clientAudio.ws) { + stopClientAudio(); + return; + } + + // Clicking for audio is the consent: unlock the browser in the same gesture + // so playback is not blocked by a suspended AudioContext. + if (!(await ensureBrowserAudio())) return; + + startClientAudio(); } function syncTtsControls() { @@ -1921,6 +2284,17 @@ function syncTtsControls() { if (speak) speak.disabled = !state.takeover; } +async function setCapture(start) { + try { + await VK.api.post("/api/audio/capture", { start }); + toast(start ? "Capture started" : "Capture stopped", "ok"); + await refreshTranscript(); + void refreshAudio(); + } catch (error) { + toast(error.message, "error"); + } +} + async function speakText() { if (!state.takeover) { toast("Composer is read-only — turn Takeover on to speak", "error"); @@ -1932,7 +2306,20 @@ async function speakText() { try { const provider = $("#tts-provider").value; - await VK.api.post("/api/audio/speak", { text, provider }); + const speed = Number($("#tts-speed").value); + const pitch = Number($("#tts-pitch").value); + const expressionEl = $("#tts-expression"); + const expression = + expressionEl.dataset.dirty === "1" ? Number(expressionEl.value) : state.speech.expression ?? undefined; + + await VK.api.post("/api/audio/speak", { + text, + provider, + speed, + pitch, + ...(typeof expression === "number" ? { expression } : {}), + ...effectOptions() + }); $("#tts-status").textContent = "queued"; toast("Queued for speech", "ok"); void refreshAudio(); @@ -1961,6 +2348,7 @@ function transcriptLine(segment) { async function refreshTranscript() { try { const result = await VK.api.get("/api/transcript?limit=200"); + const capture = await VK.api.get("/api/audio").catch(() => null); const list = $("#transcript-list"); list.replaceChildren(); @@ -1969,7 +2357,7 @@ async function refreshTranscript() { list.append( el("div", { class: "vk-empty", - text: "No captured segments yet. Start capture in Settings → Client → Bot audio." + text: "No captured segments yet. Press Start above." }) ); } else { @@ -1978,9 +2366,10 @@ async function refreshTranscript() { } const active = [...(result.sessions ?? [])].reverse().find(session => !session.endedAt); + const captureState = capture?.capturing ? "capturing" : "capture off"; $("#transcript-status").textContent = active - ? `session ${active.id.slice(-8)} · ${segments.length} segments` - : `${segments.length} segments`; + ? `${captureState} · session ${active.id.slice(-8)} · ${segments.length} segments` + : `${captureState} · ${segments.length} segments`; } catch (error) { toast(error.message, "error"); } @@ -2139,22 +2528,16 @@ function handleLiveEvent(event) { } if (event.type === "VOICE_SPEAKING") { - if (event.speaking) state.speaking.add(event.userId); - else state.speaking.delete(event.userId); - - const now = Date.now(); - if (now - state.speakingRenderAt > 500) { - state.speakingRenderAt = now; - renderVoiceRail(); - renderPeople(); - renderRightVoice(); - renderRightFavourites(); - renderCallStrip(); - } + if (event.speaking) beginSpeaking(event.userId); + else endSpeaking(event.userId); return; } if (event.type === "VOICE_STATE_UPDATE") { + // A mute or a channel change ends speaking immediately: a muted member + // never transmits, and the stop event may not arrive. + if (!event.channelId || event.mute || event.selfMute) endSpeaking(event.userId); + const now = Date.now(); if (now - state.voiceReloadAt > 1000) { state.voiceReloadAt = now; @@ -2282,6 +2665,18 @@ function init() { if (isMobileLayout()) setMobileTab("activity"); }); $("#transcript-refresh").addEventListener("click", () => void refreshTranscript()); + $("#transcript-start").addEventListener("click", () => void setCapture(true)); + $("#transcript-stop").addEventListener("click", () => void setCapture(false)); + $("#tts-duck").addEventListener("click", async () => { + const on = !$("#tts-duck").classList.contains("vk-chip-on"); + try { + await VK.api.post("/api/preferences", { audio: { duck: on } }); + toast(on ? "TTS ducking on" : "TTS ducking off", "ok"); + void refreshAudio(); + } catch (error) { + toast(error.message, "error"); + } + }); $("#channel-refresh").addEventListener("click", () => void loadMessages({ refresh: true })); $("#guilds-show-all").addEventListener("click", () => { @@ -2310,13 +2705,193 @@ function init() { void saveSpeech(); }); + for (const id of ["tts-speed", "tts-pitch"]) { + $(`#${id}`).addEventListener("input", syncSpeechSliderLabels); + $(`#${id}`).addEventListener("change", () => void saveSpeech()); + } + $("#tts-expression").addEventListener("input", () => { + const slider = $("#tts-expression"); + const engineDefault = state.expressionDefault; + const snapped = + typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2; + + if (snapped) { + slider.value = engineDefault; + slider.dataset.dirty = ""; + } else { + slider.dataset.dirty = "1"; + } + syncSpeechSliderLabels(); + }); + $("#tts-expression").addEventListener("change", () => void saveSpeech()); + buildProfileSliders(); + for (const id of ["tts-highpass", "tts-echo", "tts-word-gap"]) { + $(`#${id}`).addEventListener("input", () => { + $("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`; + $("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off"; + $("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`; + }); + $(`#${id}`).addEventListener("change", () => void saveSpeech()); + } + $("#tts-words").addEventListener("change", () => void saveSpeech()); + $("#profile-open").addEventListener("click", () => { + const panel = document.querySelector(".vk-profile-panel"); + if (panel) { + panel.style.left = ""; + panel.style.top = ""; + panel.style.transform = ""; + } + $("#profile-overlay").hidden = false; + fillProfileControls(); + }); + $("#profile-close").addEventListener("click", () => { + $("#profile-overlay").hidden = true; + }); + $("#profile-overlay").addEventListener("click", event => { + if (window.__profileDragged) { + window.__profileDragged = false; + return; + } + if (event.target === $("#profile-overlay")) $("#profile-overlay").hidden = true; + }); + + (() => { + const panel = document.querySelector(".vk-profile-panel"); + const handle = $("#profile-drag"); + if (!panel || !handle) return; + + let dragging = false; + let startX = 0; + let startY = 0; + let originX = 0; + let originY = 0; + + handle.addEventListener("mousedown", event => { + const rect = panel.getBoundingClientRect(); + dragging = true; + startX = event.clientX; + startY = event.clientY; + originX = rect.left; + originY = rect.top; + panel.style.transform = "none"; + panel.style.left = `${originX}px`; + panel.style.top = `${originY}px`; + event.preventDefault(); + }); + + window.addEventListener("mousemove", event => { + if (!dragging) return; + if (Math.abs(event.clientX - startX) + Math.abs(event.clientY - startY) > 3) { + window.__profileDragged = true; + } + panel.style.left = `${originX + event.clientX - startX}px`; + panel.style.top = `${originY + event.clientY - startY}px`; + }); + + window.addEventListener("mouseup", () => { + dragging = false; + }); + })(); + + for (const slot of ["1", "2"]) { + const input = $(`#tts-test-custom-${slot}`); + input.value = localStorage.getItem(`veskslop-test-line-${slot}`) ?? ""; + input.addEventListener("input", () => + localStorage.setItem(`veskslop-test-line-${slot}`, input.value) + ); + } + + $$(".vk-profile-tests button").forEach(button => { + button.addEventListener("click", async () => { + let text = button.dataset.test ?? ""; + if (button.dataset.testCustom) { + text = $(`#tts-test-custom-${button.dataset.testCustom}`).value.trim(); + if (!text) { + toast("Custom line is empty", "error"); + return; + } + } + + try { + const expressionEl = $("#tts-expression"); + const expression = + expressionEl.dataset.dirty === "1" + ? Number(expressionEl.value) + : state.speech.expression ?? undefined; + + await VK.api.post("/api/audio/speak", { + text, + provider: $("#tts-provider").value, + ...effectOptions(), + ...(typeof expression === "number" ? { expression } : {}) + }); + toast(`Playing "${text}"`, "ok"); + } catch (error) { + toast(error.message, "error"); + } + }); + }); + $("#tts-profile-slot").addEventListener("change", () => { + const preset = (state.speech.profilePresets ?? [])[Number($("#tts-profile-slot").value)]; + $("#tts-profile-name").value = preset?.name ?? ""; + }); + $("#tts-profile-save").addEventListener("click", async () => { + const slot = Number($("#tts-profile-slot").value); + const presets = (state.speech.profilePresets ?? [null, null, null, null, null]).slice(0, 5); + const name = $("#tts-profile-name").value.trim() || `Profile ${slot + 1}`; + presets[slot] = { + name, + ...profileFromControls(), + ...(typeof state.speech.expression === "number" ? { expression: state.speech.expression } : {}) + }; + await saveSpeech({ profilePresets: presets }); + toast(`Saved "${name}" to slot ${slot + 1}`, "ok"); + }); + $("#tts-profile-load").addEventListener("click", async () => { + const slot = Number($("#tts-profile-slot").value); + const preset = (state.speech.profilePresets ?? [])[slot]; + if (!preset) { + toast(`Slot ${slot + 1} is empty`, "error"); + return; + } + for (const [key] of PROFILE_SPEC) { + const slider = $(`#tts-profile-${key}`); + if (!slider || typeof preset[key] !== "number") continue; + slider.value = preset[key]; + slider.dataset.dirty = "1"; + } + if (typeof preset.expression === "number") { + $("#tts-expression").value = preset.expression; + $("#tts-expression").dataset.dirty = "1"; + } + fillProfileControls(); + syncSpeechSliderLabels(); + await saveSpeech(); + toast(`Loaded "${preset.name ?? `Profile ${slot + 1}`}"`, "ok"); + }); + $("#tts-speed-presets").addEventListener("click", event => { + const button = event.target.closest("button[data-speed]"); + if (!button) return; + + $("#tts-speed").value = button.dataset.speed; + syncSpeechSliderLabels(); + void saveSpeech(); + }); + syncSpeechSliderLabels(); + $("#speech-provider").addEventListener("change", () => void saveSpeech()); + $("#speech-warm").addEventListener("click", () => void warmEngine()); $("#speech-language").addEventListener("change", () => { $("#speech-voice").value = ""; void saveSpeech(); }); $("#speech-voice").addEventListener("change", () => void saveSpeech()); + $("#tts-voice").addEventListener("change", () => { + $("#speech-voice").value = $("#tts-voice").value; + void saveSpeech(); + }); + $("#scope-guild").addEventListener("click", () => void applyScope("guild")); $("#scope-dm").addEventListener("click", () => void applyScope("dm")); diff --git a/poc/public/index.html b/poc/public/index.html index fd7dc28..d855c92 100644 --- a/poc/public/index.html +++ b/poc/public/index.html @@ -129,12 +129,34 @@
Voice settings - - duck on voice + + + queue lines +
+ +
+ + + + + +
+
+ +
@@ -216,6 +238,8 @@
no capture session + +
@@ -307,13 +331,14 @@
TTS engine - + + + +
-
+
Language + default +
+ + +
+
+ Presets +
+
+ + + + +
+
+ +
+
+ Radio kit + VOX / HEV +
+
+ highpass + + 40Hz +
+
+ echo + + off +
+
+ + + 120ms +
+
+
+
+ Test lines +
+
+ + + + + + + + + +
+
+
+
+
+
diff --git a/poc/public/style.css b/poc/public/style.css index cc6b635..7bbb8b8 100644 --- a/poc/public/style.css +++ b/poc/public/style.css @@ -950,8 +950,8 @@ a.vk-attachment:hover { .vk-composer-input { flex: 1; min-height: 38px; - max-height: 160px; - resize: none; + max-height: 50vh; + resize: vertical; padding: 8px 10px; border-radius: var(--radius-sm); background: var(--color-bg); @@ -1016,6 +1016,46 @@ a.vk-attachment:hover { padding: 9px 11px 8px; } +.vk-tts-slider { + display: flex; + align-items: center; + gap: 6px; + flex: none; +} + +.vk-tts-slider input[type="range"] { + width: 84px; + accent-color: var(--color-accent); +} + +.vk-tts-slider .vk-mono { + min-width: 34px; + text-align: right; +} + +.vk-tts-speed { + display: flex; + flex-direction: column; + gap: 4px; +} + +.vk-tts-presets { + display: flex; + gap: 4px; + padding-left: 2px; +} + +.vk-tts-presets .vk-chip { + padding: 3px 7px; + font-size: 10px; +} + +.vk-engine-row { + display: flex; + align-items: center; + gap: 6px; +} + .vk-chip { display: flex; align-items: center; @@ -1027,6 +1067,14 @@ a.vk-attachment:hover { box-shadow: inset 0 0 0 1px var(--color-accent-800); color: var(--color-neutral-300); font: 400 11px/1 ui-monospace, Menlo, monospace; + border: none; + cursor: pointer; +} + +.vk-chip-off { + background: var(--color-bg); + box-shadow: inset 0 0 0 1px var(--color-neutral-800); + color: var(--color-neutral-500); } .vk-chip-on { @@ -1047,8 +1095,9 @@ a.vk-attachment:hover { .vk-tts-field { width: 100%; - max-height: 150px; - resize: none; + min-height: 64px; + max-height: 40vh; + resize: vertical; border: none; outline: none; background: transparent; @@ -1203,7 +1252,11 @@ a.vk-attachment:hover { } .vk-transcript-toolbar .vk-small-btn { - margin-left: auto; + margin-left: 0; +} + +.vk-transcript-toolbar #transcript-status { + margin-right: auto; } .vk-transcript-list { @@ -1266,6 +1319,9 @@ a.vk-attachment:hover { align-items: flex-end; gap: 2px; height: 11px; + transform-origin: bottom; + transform: scaleY(0); + opacity: 0; } .vk-levels span { @@ -1274,9 +1330,49 @@ a.vk-attachment:hover { border-radius: 1px; background: var(--color-accent); transform-origin: bottom; +} + +.vk-levels.speaking { + opacity: 1; + transform: scaleY(1); + animation: vk-levels-in 200ms ease-out; +} + +.vk-levels.speaking span, +.vk-levels.stopping span { animation: vk-lvl 0.7s ease-in-out infinite; } +.vk-levels.stopping { + animation: vk-levels-out 700ms ease-in forwards; +} + +@keyframes vk-levels-in { + from { + opacity: 0; + transform: scaleY(0.05); + } + to { + opacity: 1; + transform: scaleY(1); + } +} + +@keyframes vk-levels-out { + 0% { + opacity: 1; + transform: scaleY(1); + } + 60% { + opacity: 0.8; + transform: scaleY(0.35); + } + 100% { + opacity: 0; + transform: scaleY(0.02); + } +} + .vk-levels span:nth-child(2) { animation-delay: 0.1s; } @@ -1308,6 +1404,19 @@ a.vk-attachment:hover { } @media (prefers-reduced-motion: reduce) { + .vk-levels, + .vk-levels.speaking, + .vk-levels.stopping { + animation: none; + } + .vk-levels.speaking { + opacity: 1; + transform: scaleY(1); + } + .vk-levels.stopping { + opacity: 0; + transform: scaleY(0); + } .vk-levels span { animation: none; transform: scaleY(0.6); @@ -1482,6 +1591,116 @@ a.vk-attachment:hover { display: none; } +/* Flextalk voice profile: a bottom drawer over the chat, separate from the TTS row. */ + +.vk-profile-panel { + position: fixed; + left: 50%; + top: 10vh; + transform: translateX(-50%); + width: min(760px, 100%); + max-height: 76vh; + display: flex; + flex-direction: column; + background: var(--color-surface); + box-shadow: var(--shadow-lg); + border-radius: var(--radius-lg) var(--radius-lg) 0 0; + animation: vk-drawer-in 180ms ease-out; +} + +.vk-profile-body { + flex: 1; + min-height: 0; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 16px; + padding: 6px 16px 18px; +} + +.vk-profile-section { + display: flex; + flex-direction: column; + gap: 8px; +} + +.vk-profile-section + .vk-profile-section { + padding-top: 14px; + box-shadow: inset 0 1px 0 var(--color-neutral-800); +} + +.vk-profile-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 6px 20px; +} + +.vk-profile-row { + display: flex; + align-items: center; + gap: 8px; +} + +.vk-profile-row input[type="range"] { + flex: 1; + min-width: 80px; + accent-color: var(--color-accent); +} + +.vk-profile-row .vk-mono { + min-width: 44px; + text-align: right; +} + +.vk-profile-row .input { + flex: 1; + min-height: 28px; +} + +#profile-drag { + cursor: move; + user-select: none; +} + +.vk-profile-tests { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; +} + +.vk-test-custom { + width: 110px; + min-height: 26px; +} + +@keyframes vk-drawer-in { + from { + transform: translateY(26px); + opacity: 0; + } + to { + transform: translateY(0); + opacity: 1; + } +} + +@media (max-width: 768px) { + .vk-profile-panel { + max-height: 82vh; + } + + .vk-profile-grid { + grid-template-columns: minmax(0, 1fr); + } +} + +@media (prefers-reduced-motion: reduce) { + .vk-profile-panel { + animation: none; + } +} + .vk-settings-panel { width: min(680px, 100%); height: 100%; diff --git a/poc/scripts/audio-doctor.sh b/poc/scripts/audio-doctor.sh new file mode 100644 index 0000000..044b4da --- /dev/null +++ b/poc/scripts/audio-doctor.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Veskslop audio doctor. Run it on the server while the bot sits in a call and +# people are talking. It shows where client audio goes and what the daemon sees. +set -uo pipefail + +PORT="${WEBUI_PORT:-8789}" + +section() { printf '\n== %s ==\n' "$1"; } + +section "host addresses" +hostname -I 2>/dev/null || true + +section "default sink" +pactl get-default-sink 2>&1 || true + +section "sinks" +pactl list short sinks 2>&1 || true + +section "sink inputs: who is playing, and into which sink" +if command -v pactl >/dev/null; then + pactl list short sink-inputs 2>&1 || true + echo + echo "-- vesktop stream state (Corked / Mute / Volume) --" + pactl list sink-inputs 2>&1 | grep -E "Sink Input|Sink:|Corked:|Mute:|Volume:|application\.(name|process\.binary)|media\.name" || true +fi + +section "microphone sources the client reads" +if command -v pactl >/dev/null; then + pactl list short source-outputs 2>&1 || true + echo + echo "-- source output names --" + pactl list source-outputs 2>&1 | grep -E "Source Output #|Source:|application\.(name|process\.binary)|media\.name" || true +fi + +section "client processes" +ps -eo pid,args 2>/dev/null | grep -iE "vesktop|electron" | grep -v grep || echo "no veskstop/electron process found" + +section "pulse clients" +if command -v pactl >/dev/null; then + pactl list short clients 2>&1 || true +fi + +section "daemon audio status" +if command -v curl >/dev/null; then + curl -sk "https://127.0.0.1:${PORT}/api/audio" || curl -s "http://127.0.0.1:${PORT}/api/audio" + echo +else + echo "curl is missing" +fi + +section "bot identity" +if command -v curl >/dev/null; then + curl -sk "https://127.0.0.1:${PORT}/health" || curl -s "http://127.0.0.1:${PORT}/health" + echo +fi + +section "bot voice state" +if command -v curl >/dev/null; then + curl -sk "https://127.0.0.1:${PORT}/api/voice" || curl -s "http://127.0.0.1:${PORT}/api/voice" + echo +fi + +section "volumes: client output and per-user" +if command -v curl >/dev/null; then + curl -sk "https://127.0.0.1:${PORT}/api/voice/volume" || curl -s "http://127.0.0.1:${PORT}/api/voice/volume" + echo +fi + +section "capture sink level for 5 seconds (talk during this)" +if command -v ffmpeg >/dev/null; then + ffmpeg -hide_banner -loglevel info -f pulse -i "veskslop_capture.monitor" -t 5 \ + -af volumedetect -f null - 2>&1 | grep -E "mean_volume|max_volume" || true +else + echo "ffmpeg is missing" +fi + +section "live events for 8 seconds (talk during this)" +if command -v curl >/dev/null; then + curl -skN --max-time 8 "https://127.0.0.1:${PORT}/api/events" 2>/dev/null \ + | grep -E "VOICE_SPEAKING|VOICE_STATE_UPDATE" | head -10 || true +fi + +printf '\nDaemon terminal lines to look for: [audio] moved stream, [capture] auto-start, ffmpeg errors.\n' diff --git a/poc/src/audio-service.ts b/poc/src/audio-service.ts index d1304e4..cfda289 100644 --- a/poc/src/audio-service.ts +++ b/poc/src/audio-service.ts @@ -10,7 +10,7 @@ import { AudioManager } from "./audio.js"; import type { CaptureSegment } from "./capture.js"; import { CapturePipeline } from "./capture.js"; import type { Stt } from "./stt.js"; -import { Tts, type TtsOptions, type TtsVoice } from "./tts.js"; +import { Tts, type TtsOptions, type TtsProvider, type TtsVoice } from "./tts.js"; import type { TranscriptSegment, TranscriptSession, TranscriptStore } from "./transcript.js"; import { DEFAULT_VAD, type VadConfig } from "./vad.js"; @@ -50,10 +50,12 @@ export class AudioService { private readonly speakerIntervals: SpeakerInterval[] = []; private readonly speakerNames = new Map(); private playing = false; + private duckEnabled = true; private nextId = 1; private ducked = false; private selfId?: string; private capture?: CapturePipeline; + private captureAdoptTimer?: ReturnType; private session?: TranscriptSession; private segmentCount = 0; private initialized = false; @@ -94,6 +96,7 @@ export class AudioService { stt: this.stt.status(), queue: this.queue.length, speaking: this.playing, + duckEnabled: this.duckEnabled, capturing: !!this.capture, sessionId: this.session?.id ?? null, segments: this.segmentCount, @@ -101,6 +104,16 @@ export class AudioService { }; } + /** Enable or disable ducking the TTS while others speak. */ + setDuckEnabled(on: boolean): void { + this.duckEnabled = on; + void this.applyDuck(this.shouldDuck()); + } + + private shouldDuck(): boolean { + return this.duckEnabled && this.playing && this.speakers.size > 0; + } + async speak(text: string, options: TtsOptions = {}): Promise { await this.init(); @@ -109,12 +122,18 @@ export class AudioService { const tts = this.tts.status(); const provider = options.provider ?? tts.defaultProvider; - if (provider === "local" && !tts.local) { - throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)"); - } if (provider === "api" && !tts.api) { throw new Error("API TTS is not configured (VESKSLOP_TTS_API_URL)"); } + if (provider === "command" && !tts.command) { + throw new Error("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)"); + } + if (provider === "espeak" && !tts.espeak) { + throw new Error("espeak-ng is not installed"); + } + if (provider === "local" && !tts.local) { + throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)"); + } const trimmed = text.trim(); if (!trimmed) throw new Error("text is required"); @@ -156,7 +175,7 @@ export class AudioService { } this.pruneIntervals(); - await this.applyDuck(this.playing && this.speakers.size > 0); + await this.applyDuck(this.shouldDuck()); } async startCapture(): Promise<{ sessionId: string }> { @@ -185,6 +204,8 @@ export class AudioService { }); this.capture.start(); + void this.manager.adoptClientStreams(); + this.captureAdoptTimer = setInterval(() => void this.manager.adoptClientStreams(), 5000); this.session = session; this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION", session }); @@ -192,6 +213,8 @@ export class AudioService { } stopCapture(): void { + clearInterval(this.captureAdoptTimer); + this.captureAdoptTimer = undefined; this.capture?.stop(); this.capture = undefined; @@ -209,8 +232,12 @@ export class AudioService { return this.manager.startMonitor(onData, onError); } - ttsVoices(): Promise { - return this.tts.voices(); + ttsVoices(engine?: TtsProvider): Promise { + return this.tts.voices(engine); + } + + warmTts(engine?: TtsProvider) { + return this.tts.warm(engine); } async dispose(): Promise { @@ -259,7 +286,10 @@ export class AudioService { try { const text = await this.stt.transcribe(record.wav); - if (!text) return; + if (!text) { + console.warn(`[capture] transcription empty for ${record.wav}`); + return; + } this.transcripts.setSegmentText(record.id, text); this.options.onTranscript?.({ type: "TRANSCRIPT_TEXT", id: record.id, text }); @@ -321,7 +351,7 @@ export class AudioService { this.playing = true; try { const file = await this.tts.synthesize(item.text, item.options); - await this.applyDuck(this.speakers.size > 0); + await this.applyDuck(this.shouldDuck()); await this.manager.play(file); } catch (error) { console.error("[audio] speak failed:", (error as Error).message); diff --git a/poc/src/audio.ts b/poc/src/audio.ts index 958581b..8f69e76 100644 --- a/poc/src/audio.ts +++ b/poc/src/audio.ts @@ -13,6 +13,7 @@ const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts"; const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic"; const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture"; const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30); +const LEAD_MS = Math.min(Math.max(Number(process.env.VESKSLOP_TTS_LEAD_MS ?? 150) || 0, 0), 2000); export class AudioUnavailableError extends Error {} @@ -26,10 +27,15 @@ export interface AudioManagerStatus { captureSink: string; ducked: boolean; duckPercent: number; + /** Lead-in silence in milliseconds before each TTS line. */ + ttsLeadMs: number; playing: boolean; recording: boolean; player: boolean; ffmpeg: boolean; + clientMatch: string[]; + adopted: number; + lastAdoptError?: string; } interface CommandResult { @@ -69,6 +75,8 @@ export class AudioManager { private backend: "pulse" | null = null; private playerAvailable = false; private ffmpegAvailable = false; + private adoptedTotal = 0; + private lastAdoptError?: string; async detect(): Promise { const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]); @@ -128,13 +136,24 @@ export class AudioManager { captureSink: CAPTURE_SINK, ducked: this.ducked, duckPercent: DUCK_PERCENT, + ttsLeadMs: LEAD_MS, playing: !!this.player, recording: !!this.recorder, player: this.playerAvailable, - ffmpeg: this.ffmpegAvailable + ffmpeg: this.ffmpegAvailable, + clientMatch: this.clientPatterns(), + adopted: this.adoptedTotal, + lastAdoptError: this.lastAdoptError }; } + private clientPatterns(): string[] { + return (process.env.VESKSLOP_AUDIO_CLIENT_MATCH ?? "vesktop,electron,chromium,discord") + .split(",") + .map(part => part.trim().toLowerCase()) + .filter(Boolean); + } + /** Create the virtual devices when they are missing. Safe to call twice. */ async ensureDevices(): Promise { if (!this.available) await this.detect(); @@ -173,6 +192,76 @@ export class AudioManager { ]); this.modules.push(Number(id)); } + + await this.adoptClientStreams(); + } + + /** + * Move the client's playback streams into the capture sink. Without this the + * capture sink stays silent and the browser monitor streams zero samples. + */ + async adoptClientStreams(): Promise { + if (!this.available) return 0; + + const patterns = this.clientPatterns(); + if (!patterns.length) return 0; + + const sinks = await run("pactl", ["list", "short", "sinks"]); + if (sinks.code !== 0) { + this.lastAdoptError = sinks.stderr.trim() || "pactl list sinks failed"; + return 0; + } + + const captureIndex = sinks.stdout + .split("\n") + .map(line => line.split(/\s+/)) + .find(fields => fields[1] === CAPTURE_SINK)?.[0]; + if (captureIndex === undefined) return 0; + + const list = await run("pactl", ["list", "sink-inputs"]); + if (list.code !== 0) { + this.lastAdoptError = list.stderr.trim() || "pactl list sink-inputs failed"; + return 0; + } + + let moved = 0; + let index: string | undefined; + let sinkIndex: string | undefined; + let matchText = ""; + + const tryMove = async () => { + if (!index || sinkIndex === captureIndex) return; + if (!patterns.some(pattern => matchText.includes(pattern))) return; + + const result = await run("pactl", ["move-sink-input", index, CAPTURE_SINK]); + if (result.code === 0) { + moved++; + this.adoptedTotal++; + console.log(`[audio] moved stream ${index} to ${CAPTURE_SINK}`); + } else { + this.lastAdoptError = result.stderr.trim() || `move-sink-input ${index} failed`; + } + }; + + for (const line of list.stdout.split("\n")) { + const header = line.match(/^Sink Input #(\d+)/); + if (header) { + await tryMove(); + index = header[1]; + sinkIndex = undefined; + matchText = ""; + continue; + } + + const sink = line.match(/^\s*Sink:\s*(\d+)/); + if (sink) sinkIndex = sink[1]; + + const property = line.match(/^\s*(application\.name|application\.process\.binary)\s*=\s*"?([^"]+)"?/); + if (property) matchText += ` ${property[2].toLowerCase()}`; + } + await tryMove(); + + return moved; } /** Stop children and unload the modules this daemon created. */ @@ -191,8 +280,15 @@ export class AudioManager { if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available"); const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay"; + + // Lead-in silence: give Discord's voice pipeline a moment to settle + // before the first syllable, so the attack is not swallowed. + const args = ["-nodisp", "-autoexit", "-loglevel", "error"]; + if (LEAD_MS > 0) args.push("-af", `adelay=${LEAD_MS}:all=1`); + args.push(file); + await new Promise((resolve, reject) => { - const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], { + const child = spawn(binary, args, { env: { ...process.env, PULSE_SINK: SINK }, stdio: ["ignore", "ignore", "pipe"] }); @@ -278,6 +374,11 @@ export class AudioManager { } const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg"; + + // The client may start or restart its stream while we listen. + void this.adoptClientStreams(); + const adoptTimer = setInterval(() => void this.adoptClientStreams(), 5000); + const child = spawn( binary, [ @@ -308,6 +409,7 @@ export class AudioManager { }); return () => { + clearInterval(adoptTimer); try { child.kill("SIGTERM"); } catch { diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index d608146..421e020 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -22,7 +22,7 @@ import { Stt } from "./stt.js"; import { TranscriptStore } from "./transcript.js"; import { DEFAULT_VAD } from "./vad.js"; import { CoreClient, coreSocketPath } from "./core-client.js"; -import { CoreProcess } from "./core-process.js"; +import { CoreProcess, staleBundleWarning } from "./core-process.js"; import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js"; import { PreferencesStore } from "./preferences.js"; import { Puppet } from "./puppet.js"; @@ -49,6 +49,13 @@ async function connectLoop(): Promise { puppet = new Puppet(client, audio); client.onEvent(event => { + // Speaking hints add to the reconciled set so the poller can + // clear users the store no longer reports as speaking. + if (event?.type === "VOICE_SPEAKING" && event.userId) { + if (event.speaking) speakingUsers.add(String(event.userId)); + else speakingUsers.delete(String(event.userId)); + } + void puppet!.enqueueEvent(event); void audio.onEvent(event); webui.broadcast(event); @@ -56,6 +63,7 @@ async function connectLoop(): Promise { client.onClose(() => { console.warn("[daemon] core bridge disconnected"); + stopSpeakingPoll(); coreConnected = false; core = undefined; puppet = undefined; @@ -66,7 +74,8 @@ async function connectLoop(): Promise { coreConnected = true; console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`); webui.broadcast({ type: "CORE_STATUS", coreConnected: true }); - void autoConnectAudio(client); + startSpeakingPoll(); + void autoConnectAudioWithRetry(client); return; } catch (error) { @@ -89,6 +98,69 @@ function requirePuppet(): Puppet { return puppet; } +/* + * Discord's dispatcher only sends speaking-start events, so the UI and the + * transcript attribution need the store's authoritative set. Poll it once a + * second and synthesize the missing stop/start events. Dispatcher events feed + * the same set so a user the store no longer reports is cleared. + */ + +let speakingTimer: ReturnType | undefined; +const speakingUsers = new Set(); + +function emitCoreEvent(event: any): void { + void puppet?.enqueueEvent(event); + void audio.onEvent(event); + webui.broadcast(event); +} + +async function pollSpeaking(): Promise { + if (!core) return; + + try { + const result = (await core.voiceSpeaking()) as { channelId?: string | null; userIds?: string[] }; + const next = new Set(result?.channelId ? (result.userIds ?? []) : []); + + for (const userId of next) { + if (!speakingUsers.has(userId)) { + emitCoreEvent({ + type: "VOICE_SPEAKING", + userId, + speaking: true, + channelId: result.channelId ?? null + }); + } + } + + for (const userId of speakingUsers) { + if (!next.has(userId)) { + emitCoreEvent({ + type: "VOICE_SPEAKING", + userId, + speaking: false, + channelId: result?.channelId ?? null + }); + } + } + + speakingUsers.clear(); + for (const userId of next) speakingUsers.add(userId); + } catch { + // The close handler stops the poller. + } +} + +function startSpeakingPoll(): void { + stopSpeakingPoll(); + speakingTimer = setInterval(() => void pollSpeaking(), 1000); +} + +function stopSpeakingPoll(): void { + if (speakingTimer) clearInterval(speakingTimer); + speakingTimer = undefined; + speakingUsers.clear(); +} + const coreProcess = new CoreProcess({ tokenFile }); const transcripts = new TranscriptStore( @@ -129,6 +201,7 @@ const audio = new AudioService({ onTranscript: event => webui.broadcast(event) }); +audio.setDuckEnabled(preferences.get().audio?.duck !== false); void audio.init(); /** Point the client at the virtual microphone and capture sink when present. */ @@ -161,11 +234,25 @@ async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolea console.log("[audio] client devices set:", JSON.stringify(result)); return { connected: true, message: "the client now uses the virtual devices" }; } catch (error) { - console.warn("[audio] device auto-connect failed:", (error as Error).message); return { connected: false, message: (error as Error).message }; } } +/** + * The bridge socket is up before the renderer finishes loading, so the first + * device calls can fail with "core is not ready" style errors. Retry. + */ +async function autoConnectAudioWithRetry(client: CoreClient): Promise { + for (let attempt = 1; attempt <= 10; attempt++) { + if (attempt > 1) await sleep(2500); + + const result = await autoConnectAudio(client); + if (result.connected) return; + + console.warn(`[audio] device auto-connect attempt ${attempt}: ${result.message}`); + } +} + const auth = new AuthService(tokenFile, update => { webui.broadcast({ type: "AUTH_QR", update }); @@ -385,12 +472,36 @@ function buildServer(): McpServer { { description: "Move the client to another voice channel.", inputSchema: { - channelId: z.string().describe("Voice channel ID") + channelId: z.string().describe("Channel ID") } }, async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId))) ); + server.registerTool( + "get_user_volume", + { + description: + "Per-user playback volume (0 to 200, 100 is default) and the client's own output volume. A volume of 0 silences that user.", + inputSchema: { + userIds: z.array(z.string()).min(1).max(50).describe("User IDs to look up") + } + }, + async ({ userIds }) => jsonText(await requireCore().userVolume(userIds)) + ); + + server.registerTool( + "set_user_volume", + { + description: "Set a user's playback volume. 0 silences the user; 100 is default; 200 is the maximum.", + inputSchema: { + userId: z.string().describe("User ID"), + volume: z.number().int().min(0).max(200).describe("Volume from 0 to 200") + } + }, + async ({ userId, volume }) => jsonText(await pacedVoice(() => requireCore().userSetVolume(userId, volume))) + ); + server.registerTool( "get_audio_status", { @@ -405,11 +516,58 @@ function buildServer(): McpServer { description: "Queue text to speech into the bot's virtual microphone.", inputSchema: { text: z.string().min(1).max(1000).describe("Text to speak"), - provider: z.enum(["local", "api"]).optional().describe("TTS provider"), - voice: z.string().optional().describe("Voice name for the provider") + provider: z + .enum(["local", "espeak", "command", "api"]) + .optional() + .describe("TTS engine: command is the configured VESKSLOP_TTS_CMD engine"), + voice: z.string().optional().describe("Voice name for the provider"), + speed: z.number().min(0.5).max(2).optional().describe("Rate multiplier, 1.0 is natural"), + pitch: z.number().min(-12).max(12).optional().describe("Pitch offset in semitones"), + expression: z + .number() + .min(0) + .max(100) + .optional() + .describe("Flextalk Expression profile value (Keith Bell, Kerry Watson)"), + profile: z + .record(z.string(), z.number()) + .optional() + .describe("Flextalk profile overrides (pitch, rate, prominences, head_size, lowering)"), + highpass: z.number().min(40).max(400).optional().describe("High-pass filter Hz"), + echo: z.number().min(0).max(100).optional().describe("Mono echo amount"), + words: z.boolean().optional().describe("Synthesize each word and merge with gaps"), + wordGapMs: z.number().min(0).max(500).optional().describe("Silence between words in ms") } }, - async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice })) + async ({ text, provider, voice, speed, pitch, expression, profile, highpass, echo, words, wordGapMs }) => + jsonText( + await audio.speak(text, { + provider, + voice, + speed, + pitch, + expression, + profile, + highpass, + echo, + words, + wordGapMs + }) + ) + ); + + server.registerTool( + "start_tts_engine", + { + description: "Start (warm up) the TTS engine so the first spoken line is fast.", + inputSchema: { + engine: z + .enum(["local", "espeak", "command", "api"]) + .optional() + .describe("Engine to start. Defaults to the configured one.") + } + }, + async ({ engine }) => jsonText(await audio.warmTts(engine)) ); server.registerTool( @@ -620,8 +778,24 @@ app.delete("/mcp", (_req, res) => { ); }); -app.get("/health", (_req, res) => { - res.json({ ok: true, coreConnected, socketPath: coreSocketPath, ownerId: puppet?.owner, queries: getQueryCount() }); +app.get("/health", async (_req, res) => { + let selfId = puppet?.self; + if (!selfId && core) { + try { + selfId = await core.getSelfId(); + } catch { + // The core may not be ready yet. + } + } + + res.json({ + ok: true, + coreConnected, + socketPath: coreSocketPath, + ownerId: puppet?.owner, + selfId, + queries: getQueryCount() + }); }); app.listen(PORT, HOST, (error?: Error) => { @@ -630,9 +804,13 @@ app.listen(PORT, HOST, (error?: Error) => { process.exit(1); } console.log(`[daemon] MCP listening on http://${HOST}:${PORT}/mcp`); + + const stale = staleBundleWarning(); + if (stale) console.warn(`[daemon] ${stale}`); }); function shutdown() { + stopSpeakingPoll(); core?.close(); void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0)); } diff --git a/poc/src/core-client.ts b/poc/src/core-client.ts index 12ef5e0..303494c 100644 --- a/poc/src/core-client.ts +++ b/poc/src/core-client.ts @@ -196,10 +196,22 @@ export class CoreClient { return this.request("voice.users", { userIds }); } + async voiceSpeaking() { + return this.request("voice.speaking", {}); + } + async userStatus(userIds: string[]) { return this.request("user.status", { userIds }); } + async userVolume(userIds: string[]) { + return this.request("user.volume", { userIds }); + } + + async userSetVolume(userId: string, volume: number) { + return this.request("user.setVolume", { userId, volume }); + } + async voiceDevices() { return this.request("voice.devices"); } diff --git a/poc/src/core-process.ts b/poc/src/core-process.ts index 4144397..8d3b993 100644 --- a/poc/src/core-process.ts +++ b/poc/src/core-process.ts @@ -9,7 +9,7 @@ */ import { spawn, type ChildProcess } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, statSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -17,7 +17,37 @@ const ROOT = fileURLToPath(new URL("../..", import.meta.url)); const DEFAULT_ARGS = "--no-sandbox --disable-gpu --disable-dev-shm-usage"; const XVFB_ARGS = "-screen 0 1280x720x24 -nolisten tcp"; -const MAX_OUTPUT_LINES = 20; +const MAX_OUTPUT_LINES = 200; + +/** + * Warn when the built client bundle predates the core sources. Without this, a + * new daemon paired with a stale client reports UNKNOWN_ACTION for new actions. + */ +export function staleBundleWarning(): string | null { + const sources = [ + "src/renderer/puppet/actions.ts", + "src/renderer/puppet/events.ts", + "src/shared/puppetProtocol.ts", + "src/main/puppetBridge.ts" + ]; + const bundles = ["dist/js/renderer.js", "dist/js/main.js"]; + + try { + let newest = 0; + for (const source of sources) newest = Math.max(newest, statSync(join(ROOT, source)).mtimeMs); + + let oldest = Number.POSITIVE_INFINITY; + for (const bundle of bundles) oldest = Math.min(oldest, statSync(join(ROOT, bundle)).mtimeMs); + + if (oldest < newest) { + return "the built client bundle is older than the core sources; run `pnpm build` and restart the client"; + } + } catch { + // A missing bundle is reported by the client start itself. + } + + return null; +} export interface CoreExit { code: number | null; diff --git a/poc/src/mock-core.ts b/poc/src/mock-core.ts index b9674bf..18f642f 100644 --- a/poc/src/mock-core.ts +++ b/poc/src/mock-core.ts @@ -100,6 +100,8 @@ const FAKE_HISTORY = Array.from({ length: 8 }, (_, index) => ({ const MOCK_VOICE_CHANNEL = "200000000000000002"; let botVoiceChannel: string | null = MOCK_VOICE_CHANNEL; let ownerVoiceChannel: string | null = MOCK_VOICE_CHANNEL; +let outputVolume: number | null = 100; +const userVolumes: Record = {}; const STATE_FAIL_MS = Number(process.env.VESKSLOP_MOCK_STATE_FAIL_MS ?? 0); const startedAt = Date.now(); @@ -165,6 +167,8 @@ function handle(action: string | undefined, params: any) { members: id === "999" ? mockVoiceMembers() : [] })) }; + case "voice.speaking": + return { channelId: botVoiceChannel, userIds: [] }; case "user.status": return { users: (params?.userIds ?? []).map((id: string) => ({ @@ -172,6 +176,20 @@ function handle(action: string | undefined, params: any) { status: id === "999" ? "online" : id === "177117296140091393" ? "idle" : "offline" })) }; + case "user.volume": + return { + outputVolume: outputVolume, + users: (params?.userIds ?? []).map((id: string) => ({ + id, + volume: userVolumes[id] ?? 100 + })) + }; + case "user.setVolume": { + const userId = String(params?.userId); + const volume = Number(params?.volume); + userVolumes[userId] = volume; + return { userId, volume, outputVolume }; + } case "voice.setDevices": return { inputDeviceId: params?.inputDeviceId ?? null, outputDeviceId: params?.outputDeviceId ?? null }; case "user.search": { diff --git a/poc/src/preferences.ts b/poc/src/preferences.ts index daf8c24..5caf60a 100644 --- a/poc/src/preferences.ts +++ b/poc/src/preferences.ts @@ -9,9 +9,61 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; export interface TtsPreferences { - provider: "local" | "api" | null; + provider: "local" | "espeak" | "command" | "api" | null; language: string | null; voice: string | null; + /** Speech-rate multiplier, 1.0 is the engine's natural rate. Null uses the engine default. */ + speed: number | null; + /** Pitch offset in semitones, 0 is the engine's natural pitch. Null uses the engine default. */ + pitch: number | null; + /** Flextalk Expression profile value, 0 to 100. Null keeps the engine's registry default. */ + expression: number | null; + /** Flextalk profile overrides (only the keys the user changed). */ + profile: Record; + /** Five saved profile slots; null means empty. `name` labels the slot. */ + profilePresets: Array | null>; + /** High-pass filter Hz, 40-400. Null uses the engine default (40). */ + highpass: number | null; + /** Mono echo amount, 0-100. Null or 0 disables. */ + echo: number | null; + /** Synthesize each word and merge with silence gaps (Flextalk only). */ + words: boolean; + /** Silence between words in milliseconds, 0-500. */ + wordGapMs: number | null; +} + +const PROFILE_RANGES: Record = { + pitch: [0, 255], + rate: [0, 100], + volume: [0, 65535], + expression: [0, 100], + prominences: [0, 100], + head_size: [0, 100], + lowering: [0, 100] +}; + +function normalizeProfile(value: unknown): Record { + const profile: Record = {}; + if (!value || typeof value !== "object") return profile; + + for (const [key, range] of Object.entries(PROFILE_RANGES)) { + const raw = (value as Record)[key]; + if (isUnset(raw)) continue; + const number = Number(raw); + if (!Number.isFinite(number) || number < range[0] || number > range[1]) continue; + profile[key] = Math.round(number); + } + + return profile; +} + +function normalizePreset(value: unknown): Record | null { + if (!value || typeof value !== "object") return null; + + const preset: Record = normalizeProfile(value); + const name = (value as Record).name; + if (typeof name === "string" && name.trim()) preset.name = name.trim().slice(0, 40); + return preset; } export interface FavouritesPreferences { @@ -19,16 +71,36 @@ export interface FavouritesPreferences { channels: string[]; } +export interface AudioPreferences { + /** Duck the TTS while someone else speaks. */ + duck: boolean; +} + export interface WebPreferences { /** null means every guild is visible. An empty array hides all. */ visibleGuilds: string[] | null; tts: TtsPreferences; + audio: AudioPreferences; favourites: FavouritesPreferences; } const DEFAULTS: WebPreferences = { visibleGuilds: null, - tts: { provider: null, language: null, voice: null }, + tts: { + provider: null, + language: null, + voice: null, + speed: null, + pitch: null, + expression: null, + profile: {}, + profilePresets: [null, null, null, null, null], + highpass: null, + echo: null, + words: false, + wordGapMs: null + }, + audio: { duck: true }, favourites: { users: [], channels: [] } }; @@ -37,6 +109,42 @@ export interface PreferencesOptions { legacyFile?: string; } +function normalizeProvider(value: unknown): TtsPreferences["provider"] { + return value === "api" || value === "local" || value === "espeak" || value === "command" ? value : null; +} + +function isUnset(value: unknown): boolean { + return value === null || value === undefined || value === ""; +} + +function normalizeSpeed(value: unknown): number | null { + if (isUnset(value)) return null; + const speed = Number(value); + if (!Number.isFinite(speed) || speed < 0.5 || speed > 2) return null; + return Math.round(speed * 100) / 100; +} + +function normalizePitch(value: unknown): number | null { + if (isUnset(value)) return null; + const pitch = Number(value); + if (!Number.isFinite(pitch) || pitch < -12 || pitch > 12) return null; + return Math.round(pitch * 10) / 10; +} + +function normalizeExpression(value: unknown): number | null { + if (isUnset(value)) return null; + const expression = Number(value); + if (!Number.isFinite(expression) || expression < 0 || expression > 100) return null; + return Math.round(expression); +} + +function normalizeRange(value: unknown, min: number, max: number): number | null { + if (isUnset(value)) return null; + const number = Number(value); + if (!Number.isFinite(number) || number < min || number > max) return null; + return Math.round(number); +} + export class PreferencesStore { private preferences: WebPreferences = structuredClone(DEFAULTS); @@ -68,9 +176,26 @@ export class PreferencesStore { if (patch.tts !== undefined) { this.preferences.tts = { - provider: patch.tts?.provider === "api" ? "api" : patch.tts?.provider === "local" ? "local" : null, + provider: normalizeProvider(patch.tts?.provider), language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null, - voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null + voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null, + speed: normalizeSpeed(patch.tts?.speed), + pitch: normalizePitch(patch.tts?.pitch), + expression: normalizeExpression(patch.tts?.expression), + profile: normalizeProfile(patch.tts?.profile), + profilePresets: Array.isArray(patch.tts?.profilePresets) + ? Array.from({ length: 5 }, (_, index) => normalizePreset(patch.tts?.profilePresets?.[index])) + : this.preferences.tts.profilePresets, + highpass: normalizeRange(patch.tts?.highpass, 40, 400), + echo: normalizeRange(patch.tts?.echo, 0, 100), + words: patch.tts?.words === true, + wordGapMs: normalizeRange(patch.tts?.wordGapMs, 0, 500) + }; + } + + if (patch.audio !== undefined) { + this.preferences.audio = { + duck: patch.audio?.duck !== false }; } @@ -112,9 +237,23 @@ export class PreferencesStore { ? [...new Set(raw.visibleGuilds.map((id: unknown) => String(id)))] : null, tts: { - provider: provider === "api" || provider === "local" ? provider : null, + provider: normalizeProvider(provider), language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null, - voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null + voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null, + speed: normalizeSpeed(raw?.tts?.speed), + pitch: normalizePitch(raw?.tts?.pitch), + expression: normalizeExpression(raw?.tts?.expression), + profile: normalizeProfile(raw?.tts?.profile), + profilePresets: Array.isArray(raw?.tts?.profilePresets) + ? Array.from({ length: 5 }, (_, index) => normalizePreset(raw.tts.profilePresets[index])) + : [null, null, null, null, null], + highpass: normalizeRange(raw?.tts?.highpass, 40, 400), + echo: normalizeRange(raw?.tts?.echo, 0, 100), + words: raw?.tts?.words === true, + wordGapMs: normalizeRange(raw?.tts?.wordGapMs, 0, 500) + }, + audio: { + duck: raw?.audio?.duck !== false }, favourites: { users: Array.isArray(raw?.favourites?.users) diff --git a/poc/src/puppet.ts b/poc/src/puppet.ts index 829c175..6fd229a 100644 --- a/poc/src/puppet.ts +++ b/poc/src/puppet.ts @@ -157,6 +157,11 @@ export class Puppet { private selfId?: string; private queue: Promise = Promise.resolve(); + /** The logged-in account's user ID, once the core has been asked for it. */ + get self(): string | undefined { + return this.selfId; + } + constructor( private readonly core: PuppetTransport, private readonly audio?: PuppetAudio diff --git a/poc/src/tts.ts b/poc/src/tts.ts index 623c574..68f9c11 100644 --- a/poc/src/tts.ts +++ b/poc/src/tts.ts @@ -7,24 +7,51 @@ */ import { spawn } from "node:child_process"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; export class TtsError extends Error {} -export type TtsProvider = "local" | "api"; +export type TtsProvider = "local" | "espeak" | "command" | "api"; export interface TtsOptions { provider?: TtsProvider; voice?: string; + /** Speech-rate multiplier; the engine maps it to its own units. */ + speed?: number; + /** Pitch offset in semitones; the engine maps it to its own units. */ + pitch?: number; + /** Flextalk Expression profile value, 0 to 100. */ + expression?: number; + /** Flextalk profile overrides: pitch, rate, volume, expression, prominences, head_size, lowering. */ + profile?: Record; + /** High-pass filter Hz, 40 to 400. */ + highpass?: number; + /** Mono echo amount, 0 to 100. */ + echo?: number; + /** Synthesize each word and merge with silence gaps. */ + words?: boolean; + /** Silence between words in milliseconds, 0 to 500. */ + wordGapMs?: number; +} + +export interface TtsEngine { + id: "espeak" | "command" | "api"; + name: string; + available: boolean; + /** false when the engine has no language dimension (single-language command engines). */ + language: boolean; } export interface TtsStatus { local: boolean; api: boolean; + espeak: boolean; + command: boolean; defaultProvider: TtsProvider; engine: string | null; + engines: TtsEngine[]; } export interface TtsVoice { @@ -32,8 +59,19 @@ export interface TtsVoice { name: string; language: string; gender: "male" | "female" | null; + /** false when the engine ignores pitch for this voice (for example Flextalk). */ + supportsPitch?: boolean; + /** true when the voice has a Flextalk-style Expression profile parameter. */ + supportsExpression?: boolean; + /** Registry default for the expression parameter, when known. */ + expressionDefault?: number; + /** Per-voice Flextalk profile defaults (pitch, rate, prominences, head_size, lowering, expression). */ + profileDefaults?: Record; } +/** espeak-ng voice variants, appended to every base voice. */ +const ESPEAK_VARIANTS = ["m1", "m2", "m3", "m4", "m5", "m6", "m7", "f1", "f2", "f3", "f4", "f5", "croak", "whisper"]; + interface CommandResult { code: number | null; stdout: string; @@ -81,11 +119,62 @@ function parseVoices(output: string): TtsVoice[] { return voices; } +/** + * Command-mode voice list. Entries are separated by commas or newlines. Each entry is + * `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or + * `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch` + * (the engine ignores pitch) and `expression` / `expression=NN` (Flextalk profile). + */ +function parseEngineVoices(raw: string): TtsVoice[] { + return raw + .split(/[,\n]+/) + .map(part => part.trim()) + .filter(Boolean) + .map((entry): TtsVoice => { + const [id, name, gender, language, flags] = entry.split("|").map(part => part?.trim()); + const flagList = flags?.split("+").map(flag => flag.trim()) ?? []; + const expression = flagList.find(flag => flag === "expression" || flag.startsWith("expression=")); + const defaults = flagList.find(flag => flag.startsWith("defaults=")); + + let profileDefaults: Record | undefined; + if (defaults) { + const parts = defaults.slice("defaults=".length).split(":"); + profileDefaults = {}; + for (let index = 0; index + 1 < parts.length; index += 2) { + const value = Number(parts[index + 1]); + if (Number.isFinite(value)) profileDefaults[parts[index]] = value; + } + } + + const expressionDefault = profileDefaults?.expression ?? + (expression?.includes("=") ? Number(expression.split("=")[1]) : undefined); + + return { + id, + name: name || id, + language: language || "en", + gender: gender === "male" || gender === "female" ? gender : null, + supportsPitch: !flagList.includes("no-pitch"), + supportsExpression: !!expression || !!profileDefaults, + ...(Number.isFinite(expressionDefault) ? { expressionDefault } : {}), + ...(profileDefaults ? { profileDefaults } : {}) + }; + }); +} + +/** Serialize a profile object to the wrapper's `key=value,key=value` form. */ +function serializeProfile(profile?: Record): string { + if (!profile) return ""; + return Object.entries(profile) + .map(([key, value]) => `${key}=${value}`) + .join(","); +} + export class Tts { private readonly dir: string; private counter = 0; private espeak = false; - private voiceCache?: TtsVoice[]; + private readonly voiceCache = new Map(); constructor(cacheDir?: string) { const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache"); @@ -109,39 +198,128 @@ export class Tts { return this.dir; } + /** Display name for the configured command engine. */ + get commandName(): string { + return process.env.VESKSLOP_TTS_NAME ?? "custom engine"; + } + async detect(): Promise { const result = await run("espeak-ng", ["--version"]); this.espeak = result.code === 0; } + engines(): TtsEngine[] { + return [ + { id: "espeak", name: "espeak-ng", available: this.espeak, language: true }, + { id: "command", name: this.commandName, available: !!this.localCommand, language: false }, + { id: "api", name: "HTTP API", available: !!this.apiUrl, language: true } + ]; + } + + private autoProvider(): TtsProvider { + if (this.localCommand) return "command"; + if (this.espeak) return "espeak"; + return this.apiUrl ? "api" : "local"; + } + status(): TtsStatus { const local = !!this.localCommand || this.espeak; const override = process.env.VESKSLOP_TTS_PROVIDER; + const allowed = new Set(["local", "espeak", "command", "api"]); return { local, api: !!this.apiUrl, - defaultProvider: override === "local" || override === "api" ? override : local ? "local" : "api", - engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null + espeak: this.espeak, + command: !!this.localCommand, + defaultProvider: allowed.has(override as TtsProvider) ? (override as TtsProvider) : this.autoProvider(), + engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null, + engines: this.engines() }; } - async voices(): Promise { - if (this.voiceCache) return this.voiceCache; - if (!this.espeak) return (this.voiceCache = []); + async voices(engine?: TtsProvider): Promise { + const key = engine ?? this.autoProvider(); + const cached = this.voiceCache.get(key); + if (cached) return cached; + + let voices: TtsVoice[] = []; + + if (key === "command" || (key === "local" && this.localCommand)) { + const declared = process.env.VESKSLOP_TTS_VOICES; + voices = declared ? parseEngineVoices(declared) : []; + } else if (key === "espeak" || key === "local") { + voices = await this.espeakVoices(); + } + + this.voiceCache.set(key, voices); + return voices; + } + + private async espeakVoices(): Promise { + if (!this.espeak) return []; const result = await run("espeak-ng", ["--voices"]); - this.voiceCache = result.code === 0 ? parseVoices(result.stdout) : []; - return this.voiceCache; + const base = result.code === 0 ? parseVoices(result.stdout) : []; + + return base.flatMap((voice): TtsVoice[] => [ + voice, + ...ESPEAK_VARIANTS.map( + (variant): TtsVoice => ({ + id: `${voice.id}+${variant}`, + name: `${voice.name} (${variant})`, + language: voice.language, + gender: variant.startsWith("m") ? "male" : variant.startsWith("f") ? "female" : null + }) + ) + ]); + } + + /** + * Start the engine so the first real line is fast: run one throwaway synthesis + * (the command wrapper starts Xvfb, Wine, and the host), then delete the file. + */ + async warm(engine?: TtsProvider): Promise<{ engine: TtsProvider; ms: number; warmed: boolean }> { + const key = engine ?? this.autoProvider(); + const start = Date.now(); + + if (key === "command" && !this.localCommand) { + throw new TtsError("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)"); + } + + const usesCommand = key === "command" || (key === "local" && !!this.localCommand); + if (!usesCommand) { + // espeak starts instantly and the HTTP provider is expected to stay warm. + return { engine: key, ms: 0, warmed: false }; + } + + const voice = process.env.VESKSLOP_TTS_VOICE || undefined; + const file = await this.viaLocalCommand("The quick brown fox jumps over the lazy dog.", { + provider: key, + voice + }); + + try { + unlinkSync(file); + } catch { + // best effort + } + + return { engine: key, ms: Date.now() - start, warmed: true }; } async synthesize(text: string, options: TtsOptions = {}): Promise { const provider = options.provider ?? this.status().defaultProvider; - return provider === "api" ? this.viaApi(text, options) : this.viaLocalCommand(text, options); + + if (provider === "api") return this.viaApi(text, options); + if (provider === "espeak") return this.viaLocalCommand(text, options, true); + if (provider === "command") return this.viaLocalCommand(text, options, false); + + return this.viaLocalCommand(text, options); } - private async viaLocalCommand(text: string, options: TtsOptions): Promise { - const command = this.localCommand; + private async viaLocalCommand(text: string, options: TtsOptions, forceEspeak = false): Promise { + const command = forceEspeak ? undefined : this.localCommand; if (!command && !this.espeak) { throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)"); } @@ -162,6 +340,14 @@ export class Tts { .replace("{out}", out) .replace("{text_file}", textFile) .replace("{voice}", options.voice ?? "") + .replace("{speed}", options.speed !== undefined ? String(options.speed) : "") + .replace("{pitch}", options.pitch !== undefined ? String(options.pitch) : "") + .replace("{expression}", options.expression !== undefined ? String(options.expression) : "") + .replace("{profile}", serializeProfile(options.profile)) + .replace("{highpass}", options.highpass !== undefined ? String(options.highpass) : "") + .replace("{echo}", options.echo !== undefined ? String(options.echo) : "") + .replace("{words}", options.words ? "1" : "") + .replace("{word_gap_ms}", options.wordGapMs !== undefined ? String(options.wordGapMs) : "") ) : ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile]; @@ -169,7 +355,24 @@ export class Tts { await new Promise((resolve, reject) => { const child = spawn(args[0], args.slice(1), { - env: { ...process.env, VESKSLOP_TTS_VOICE: options.voice ?? "" }, + env: { + ...process.env, + VESKSLOP_TTS_VOICE: options.voice ?? "", + ...(options.speed !== undefined ? { VESKSLOP_TTS_SPEED: String(options.speed) } : {}), + ...(options.pitch !== undefined ? { VESKSLOP_TTS_PITCH: String(options.pitch) } : {}), + ...(options.expression !== undefined + ? { VESKSLOP_TTS_EXPRESSION: String(options.expression) } + : {}), + ...(options.profile && Object.keys(options.profile).length + ? { VESKSLOP_TTS_PROFILE: serializeProfile(options.profile) } + : {}), + ...(options.highpass !== undefined ? { VESKSLOP_TTS_HIGHPASS: String(options.highpass) } : {}), + ...(options.echo !== undefined ? { VESKSLOP_TTS_ECHO: String(options.echo) } : {}), + ...(options.words ? { VESKSLOP_TTS_WORDS: "1" } : {}), + ...(options.wordGapMs !== undefined + ? { VESKSLOP_TTS_WORD_GAP_MS: String(options.wordGapMs) } + : {}) + }, stdio: ["ignore", "ignore", "pipe"] }); let stderr = ""; @@ -197,7 +400,18 @@ export class Tts { ? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` } : {}) }, - body: JSON.stringify({ text, ...(options.voice ? { voice: options.voice } : {}) }) + body: JSON.stringify({ + text, + ...(options.voice ? { voice: options.voice } : {}), + ...(options.speed !== undefined ? { speed: options.speed } : {}), + ...(options.pitch !== undefined ? { pitch: options.pitch } : {}), + ...(options.expression !== undefined ? { expression: options.expression } : {}), + ...(options.profile && Object.keys(options.profile).length ? { profile: options.profile } : {}), + ...(options.highpass !== undefined ? { highpass: options.highpass } : {}), + ...(options.echo !== undefined ? { echo: options.echo } : {}), + ...(options.words ? { words: true } : {}), + ...(options.wordGapMs !== undefined ? { wordGapMs: options.wordGapMs } : {}) + }) }); if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`); diff --git a/poc/src/webui.ts b/poc/src/webui.ts index 36a996c..b0e112c 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -25,7 +25,7 @@ import type { PreferencesStore, WebPreferences } from "./preferences.js"; import type { Puppet } from "./puppet.js"; import { ensureTls } from "./tls.js"; import type { TranscriptStore } from "./transcript.js"; -import type { TtsVoice } from "./tts.js"; +import type { TtsProvider, TtsVoice } from "./tts.js"; const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"]; const LANGUAGE_NAMES: Record = { @@ -401,7 +401,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { app.post( "/api/preferences", wrap(async (req, res) => { - const { visibleGuilds, tts, favourites } = req.body ?? {}; + const { visibleGuilds, tts, audio, favourites } = req.body ?? {}; if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) { throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null"); @@ -414,7 +414,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { patch.tts = { provider: tts?.provider ?? null, language: tts?.language ?? null, - voice: tts?.voice ?? null + voice: tts?.voice ?? null, + speed: tts?.speed ?? null, + pitch: tts?.pitch ?? null, + expression: tts?.expression ?? null, + profile: tts?.profile ?? {}, + profilePresets: Array.isArray(tts?.profilePresets) + ? tts.profilePresets.slice(0, 5) + : deps.preferences.get().tts.profilePresets, + highpass: tts?.highpass ?? null, + echo: tts?.echo ?? null, + words: tts?.words === true, + wordGapMs: tts?.wordGapMs ?? null }; } @@ -425,6 +436,11 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }; } + if (audio !== undefined) { + patch.audio = { duck: audio?.duck !== false }; + deps.audio.setDuckEnabled(patch.audio.duck); + } + res.json({ ok: true, result: deps.preferences.update(patch) }); }) ); @@ -486,6 +502,41 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/voice/volume", + wrap(async (req, res) => { + let userIds = req.query.userIds + ? String(req.query.userIds) + .split(",") + .map(id => id.trim()) + .filter(Boolean) + : []; + + if (!userIds.length) { + const voice = (await requireCore().voiceGet()) as { members?: Array<{ id: string }> } | undefined; + userIds = (voice?.members ?? []).map(member => member.id); + } + + if (!userIds.length) { + res.json({ ok: true, result: { outputVolume: null, users: [] } }); + return; + } + + res.json({ ok: true, result: await requireCore().userVolume(userIds) }); + }) + ); + + app.post( + "/api/voice/volume", + wrap(async (req, res) => { + const userId = String(req.body?.userId ?? ""); + const volume = Number(req.body?.volume); + const result = await pacedVoice(() => requireCore().userSetVolume(userId, volume)); + deps.getPuppet()?.appendAction({ type: "voice", action: "set-volume", userId, volume }); + res.json({ ok: true, result }); + }) + ); + app.post( "/api/voice/join", wrap(async (req, res) => { @@ -526,8 +577,28 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { app.get( "/api/audio/tts/voices", - wrap(async (_req, res) => { - res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) }); + wrap(async (req, res) => { + const engine = req.query.engine ? String(req.query.engine) : undefined; + res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices(engine as TtsProvider | undefined)) }); + }) + ); + + app.post( + "/api/audio/tts/warm", + wrap(async (req, res) => { + const requested = req.body?.engine; + const engine = ["local", "espeak", "command", "api"].includes(requested) + ? (requested as TtsProvider) + : undefined; + + const result = await deps.audio.warmTts(engine); + deps.getPuppet()?.appendAction({ + type: "audio", + action: "warm-engine", + engine: result.engine, + ms: result.ms + }); + res.json({ ok: true, result }); }) ); @@ -568,11 +639,35 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { const text = String(req.body?.text ?? ""); const prefs = deps.preferences.get(); - const provider = - req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider; + const requested = req.body?.provider; + const provider = ["api", "local", "espeak", "command"].includes(requested) + ? requested + : prefs.tts.provider; const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined; + const speed = req.body?.speed !== undefined ? Number(req.body.speed) : prefs.tts.speed ?? undefined; + const pitch = req.body?.pitch !== undefined ? Number(req.body.pitch) : prefs.tts.pitch ?? undefined; + const expression = + req.body?.expression !== undefined ? Number(req.body.expression) : prefs.tts.expression ?? undefined; + const profile = req.body?.profile && typeof req.body.profile === "object" ? req.body.profile : prefs.tts.profile; + const highpass = + req.body?.highpass !== undefined ? Number(req.body.highpass) : prefs.tts.highpass ?? undefined; + const echo = req.body?.echo !== undefined ? Number(req.body.echo) : prefs.tts.echo ?? undefined; + const words = req.body?.words !== undefined ? req.body.words === true : prefs.tts.words; + const wordGapMs = + req.body?.wordGapMs !== undefined ? Number(req.body.wordGapMs) : prefs.tts.wordGapMs ?? undefined; - const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice }); + const result = await deps.audio.speak(text, { + provider: provider ?? undefined, + voice, + ...(Number.isFinite(speed) ? { speed } : {}), + ...(Number.isFinite(pitch) ? { pitch } : {}), + ...(Number.isFinite(expression) ? { expression } : {}), + ...(profile && Object.keys(profile).length ? { profile } : {}), + ...(Number.isFinite(highpass) ? { highpass } : {}), + ...(Number.isFinite(echo) ? { echo } : {}), + ...(words ? { words: true } : {}), + ...(Number.isFinite(wordGapMs) ? { wordGapMs } : {}) + }); deps.getPuppet()?.appendAction({ type: "audio", action: "speak", diff --git a/src/main/ipcCommands.ts b/src/main/ipcCommands.ts index 8731b8d..2123480 100644 --- a/src/main/ipcCommands.ts +++ b/src/main/ipcCommands.ts @@ -12,6 +12,9 @@ import { mainWin } from "./mainWindow"; const resolvers = new Map void>>(); +/** Renderer commands that never answer must not leak their resolver or hang the caller. */ +const RENDERER_TIMEOUT_MS = 15_000; + export interface IpcMessage { nonce: string; message: string; @@ -31,18 +34,36 @@ export interface IpcResponse { * You must add a handler for the message in the renderer process. */ export function sendRendererCommand(message: string, data?: any) { - if (mainWin.isDestroyed()) { - console.warn("Main window is destroyed, cannot send IPC command:", message); - return Promise.reject(new Error("Main window is destroyed")); + if (mainWin.isDestroyed() || mainWin.webContents.isDestroyed() || mainWin.webContents.isCrashed()) { + return Promise.reject(new Error("the client renderer is not available")); } const nonce = randomUUID(); const promise = new Promise((resolve, reject) => { - resolvers.set(nonce, { resolve, reject }); + const timer = setTimeout(() => { + resolvers.delete(nonce); + reject(new Error(`the renderer did not answer ${message} in time`)); + }, RENDERER_TIMEOUT_MS); + + resolvers.set(nonce, { + resolve: value => { + clearTimeout(timer); + resolve(value); + }, + reject: error => { + clearTimeout(timer); + reject(error); + } + }); }); - mainWin.webContents.send(IpcEvents.IPC_COMMAND, { nonce, message, data }); + try { + mainWin.webContents.send(IpcEvents.IPC_COMMAND, { nonce, message, data }); + } catch (error) { + resolvers.delete(nonce); + return Promise.reject(error); + } return promise; } diff --git a/src/main/mainWindow.ts b/src/main/mainWindow.ts index 08bfe2e..f12e247 100644 --- a/src/main/mainWindow.ts +++ b/src/main/mainWindow.ts @@ -255,7 +255,14 @@ function initSettingsListeners(win: BrowserWindow) { } async function initSpellCheckLanguages(win: BrowserWindow, languages?: string[]) { - languages ??= await sendRendererCommand(IpcCommands.GET_LANGUAGES); + if (!languages) { + try { + languages = await sendRendererCommand(IpcCommands.GET_LANGUAGES); + } catch { + // The renderer may be loading or gone; spell check keeps its default. + return; + } + } if (!languages) return; const ses = session.defaultSession; diff --git a/src/main/puppetBridge.ts b/src/main/puppetBridge.ts index 27a3832..1f730e6 100644 --- a/src/main/puppetBridge.ts +++ b/src/main/puppetBridge.ts @@ -21,6 +21,7 @@ import { IpcCommands, IpcEvents } from "shared/IpcEvents"; import { PUPPET_LIMITS, PuppetAction, + PuppetActions, PuppetErrorCode, type PuppetEvent, type PuppetRequest, @@ -36,30 +37,13 @@ export const PUPPET_SOCKET_PATH = ? "\\\\?\\pipe\\vesktop-puppet.sock" : join(env.XDG_RUNTIME_DIR || env.TMPDIR || env.TMP || env.TEMP || "/tmp", "vesktop-puppet.sock"); -const KNOWN_ACTIONS = new Set([ - PuppetAction.Ping, - PuppetAction.EventsSubscribe, - PuppetAction.StateGet, - PuppetAction.VoiceGet, - PuppetAction.VoiceOverview, - PuppetAction.VoiceJoin, - PuppetAction.VoiceLeave, - PuppetAction.VoiceMove, - PuppetAction.VoiceUsers, - PuppetAction.UserStatus, - PuppetAction.VoiceDevices, - PuppetAction.VoiceSetDevices, - PuppetAction.UserSearch, - PuppetAction.DmList, - PuppetAction.MessageSend, - PuppetAction.MessageRead, - PuppetAction.MessageFetch, - PuppetAction.ReactionAdd, - PuppetAction.DmSend -]); +// Derived from the shared list so a new action cannot be added in the protocol +// yet rejected by the bridge. +const KNOWN_ACTIONS = new Set(PuppetActions); let server: Server | undefined; const subscribers = new Set(); +let rendererWarned = false; export async function initPuppetBridge() { await rm(PUPPET_SOCKET_PATH, { force: true }); @@ -148,11 +132,24 @@ async function handleLine(socket: Socket, line: string) { try { const result = await sendRendererCommand(IpcCommands.PUPPET_COMMAND, { action, params }); + rendererWarned = false; if (result?.ok === false) respond(socket, { id, ok: false, error: result.error }); else respond(socket, { id, ok: true, result: result?.result }); } catch (error) { - console.error("[puppet] action failed:", action, error); + const message = error instanceof Error ? error.message : String(error); + + // A dead renderer is expected during a restart; the daemon polls every + // second, so log the outage once instead of once per command. + if (message === "the client renderer is not available") { + if (!rendererWarned) { + rendererWarned = true; + console.warn("[puppet] the client renderer is not available; commands fail until it reloads"); + } + } else { + console.error("[puppet] action failed:", action, error); + } + respond(socket, { id, ok: false, diff --git a/src/renderer/puppet/actions.ts b/src/renderer/puppet/actions.ts index ce6bc6d..ccc9843 100644 --- a/src/renderer/puppet/actions.ts +++ b/src/renderer/puppet/actions.ts @@ -14,8 +14,9 @@ */ import { openPrivateChannel, sendMessage } from "@vencord/types/utils"; -import { findByPropsLazy } from "@vencord/types/webpack"; +import { findByPropsLazy, findStoreLazy } from "@vencord/types/webpack"; import { + ChannelRTCStore, ChannelStore, GuildMemberStore, GuildStore, @@ -45,6 +46,63 @@ const ChannelActions = findByPropsLazy("selectVoiceChannel"); /** Discord's media engine actions, located lazily at runtime. */ const MediaEngineActions = findByPropsLazy("setSelfMute"); +/** Per-user volume store, located lazily at runtime. */ +const UserVolumeStore = findStoreLazy("UserVolumeStore"); + +/** Discord's per-user volume actions, located lazily at runtime. */ +const UserVolumeActions = findByPropsLazy("setUserVolume"); + +/** Media engine store, for the client's own output volume. */ +const MediaEngineStore = findStoreLazy("MediaEngineStore"); + +function userVolumeOf(userId: string): number | null { + const candidates: any[] = [UserVolumeStore, UserVolumeActions, MediaEngineStore, MediaEngineActions]; + + for (const candidate of candidates) { + try { + const getter = candidate?.getUserVolume; + if (typeof getter !== "function") continue; + + const raw = getter.call(candidate, userId); + const value = typeof raw === "object" && raw !== null ? (raw as any).volume : raw; + const volume = Number(value); + if (Number.isFinite(volume)) return volume; + } catch { + // The lazy proxy throws when the module is absent. + } + } + + return null; +} + +function userVolumeSetter(): ((userId: string, volume: number) => void) | undefined { + const candidates: any[] = [UserVolumeStore, UserVolumeActions, MediaEngineStore, MediaEngineActions]; + + for (const candidate of candidates) { + try { + const setter = candidate?.setUserVolume; + if (typeof setter === "function") return setter.bind(candidate); + } catch { + // The lazy proxy throws when the module is absent. + } + } + + return undefined; +} + +function outputVolume(): number | null { + try { + const store = MediaEngineStore as any; + if (typeof store?.getLocalVolume === "function") { + const volume = Number(store.getLocalVolume()); + return Number.isFinite(volume) ? volume : null; + } + } catch { + // Not available in this client build. + } + return null; +} + function compareIds(a: string, b: string): number { try { const left = BigInt(a); @@ -193,7 +251,8 @@ function voiceMember(state: any, guildId?: string) { selfDeaf: !!state.selfDeaf, video: !!state.selfVideo, stream: !!state.selfStream || !!state.stream, - suppress: !!state.suppress + suppress: !!state.suppress, + volume: userVolumeOf(state.userId) }; } @@ -373,6 +432,7 @@ export async function handlePuppetAction(action: string, params: Record voiceMember(state, (channel?.guild_id as string | undefined) ?? (guildId as string | undefined)) ) @@ -436,6 +496,72 @@ export async function handlePuppetAction(action: string, params: Record participant.speaking) + .map(participant => participant.user?.id ?? participant.id) + .filter(Boolean); + + return ok({ channelId: voiceState.channelId, userIds }); + } + + case PuppetAction.UserVolume: { + const { userIds } = params; + assert( + Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50, + PuppetErrorCode.InvalidParam, + "userIds must be an array of 1 to 50 user IDs" + ); + assert( + (userIds as string[]).every(id => isSnowflake(id)), + PuppetErrorCode.InvalidSnowflake, + "every user ID must be a snowflake" + ); + + return ok({ + outputVolume: outputVolume(), + users: (userIds as string[]).map(id => ({ id, volume: userVolumeOf(id) })) + }); + } + + case PuppetAction.UserSetVolume: { + const { userId, volume } = params; + assert(isSnowflake(userId), PuppetErrorCode.InvalidSnowflake, "userId must be a snowflake"); + assert( + typeof volume === "number" && Number.isFinite(volume) && volume >= 0 && volume <= 200, + PuppetErrorCode.InvalidParam, + "volume must be a number from 0 to 200" + ); + + const setUserVolume = userVolumeSetter(); + assert( + typeof setUserVolume === "function", + PuppetErrorCode.NotSupported, + "per-user volume is not available in this client build" + ); + + enforceOutboundRateLimit(); + + try { + setUserVolume(userId, volume); + } catch (error) { + throw new ActionError( + PuppetErrorCode.VoiceFailed, + `setting the user volume failed: ${String(error)}` + ); + } + + return ok({ userId, volume, outputVolume: outputVolume() }); + } + case PuppetAction.VoiceOverview: { const { guildId } = params; assert(isSnowflake(guildId), PuppetErrorCode.InvalidSnowflake, "guildId must be a snowflake"); @@ -489,6 +615,18 @@ export async function handlePuppetAction(action: string, params: Record { try { - if (!event?.userId) return; + const userId = event?.userId ?? event?.user_id; + if (!userId) return; + + // Discord sends a number bitfield (1 = microphone). A missing + // flag is treated as a start pulse: some dispatcher payloads + // carry no flag, and the UI decays on its own after silence. + const raw = event.speaking ?? event.speakingFlag; + const speaking = raw === undefined ? true : typeof raw === "boolean" ? raw : Number(raw) !== 0; + VesktopNative.puppet.emit({ type: "VOICE_SPEAKING", - userId: event.userId, - speaking: event.speaking !== false, + userId, + speaking, channelId: event.channelId ?? event.channel_id ?? null }); } catch (error) { diff --git a/src/shared/puppetProtocol.ts b/src/shared/puppetProtocol.ts index 8965177..a29e1c4 100644 --- a/src/shared/puppetProtocol.ts +++ b/src/shared/puppetProtocol.ts @@ -22,7 +22,10 @@ export const enum PuppetAction { VoiceLeave = "voice.leave", VoiceMove = "voice.move", VoiceUsers = "voice.users", + VoiceSpeaking = "voice.speaking", UserStatus = "user.status", + UserVolume = "user.volume", + UserSetVolume = "user.setVolume", VoiceDevices = "voice.devices", VoiceSetDevices = "voice.setDevices", UserSearch = "user.search", @@ -44,7 +47,10 @@ export const PuppetActions: readonly PuppetAction[] = [ PuppetAction.VoiceLeave, PuppetAction.VoiceMove, PuppetAction.VoiceUsers, + PuppetAction.VoiceSpeaking, PuppetAction.UserStatus, + PuppetAction.UserVolume, + PuppetAction.UserSetVolume, PuppetAction.VoiceDevices, PuppetAction.VoiceSetDevices, PuppetAction.UserSearch,