Add MicroslopSam TTS, Flextalk voice profiles, effects, and engine controls

This commit is contained in:
Fisto
2026-09-15 15:09:23 +02:00
parent eab94132bb
commit 302436f95f
42 changed files with 4675 additions and 180 deletions
+2 -1
View File
@@ -15,7 +15,8 @@
"test:trigger": "tsx src/test-trigger.ts",
"cdp:dispatch": "tsx src/cdp-dispatch.ts",
"test:cache": "tsx src/test-cache.ts",
"test:audio": "tsx src/test-audio.ts"
"test:audio": "tsx src/test-audio.ts",
"audio:doctor": "bash scripts/audio-doctor.sh"
},
"dependencies": {
"@modelcontextprotocol/sdk": "^1.30.0",
+636 -61
View File
@@ -26,8 +26,11 @@ const state = {
visibleGuildsSaved: null,
timeZoneOverride: localStorage.getItem("veskslop-timezone") ?? "",
speech: { provider: null, language: null, voice: null },
profileDefaults: {},
expressionDefault: null,
scope: "guild",
speaking: new Set(),
speakingFade: new Map(),
people: new Map(),
favourites: { users: [], channels: [] },
favouriteUsers: [],
@@ -661,16 +664,19 @@ function renderVisibleGuilds() {
function voiceMarks(member) {
const marks = el("span", { class: "vk-marks" });
const speaking = state.speaking.has(member.id);
const muted = member.suppress || member.mute || member.selfMute;
// Fixed order: audio → video → share.
if (speaking) {
const levels = el("span", { class: "vk-levels", title: "speaking" });
for (let index = 0; index < 4; index++) levels.append(el("span"));
marks.append(levels);
}
// The bars are always in the layout; they scale up while speaking and
// decay back to nothing when the voice stops.
const classes = ["vk-levels"];
if (!muted && state.speaking.has(member.id)) classes.push("speaking");
if (!muted && state.speakingFade.has(member.id)) classes.push("stopping");
if (member.suppress || member.mute || member.selfMute) {
const levels = el("span", { class: classes.join(" "), title: "speaking" });
for (let index = 0; index < 4; index++) levels.append(el("span"));
marks.append(levels);
if (muted) {
const title = member.suppress ? "muted by server" : member.mute ? "server-muted" : "self-muted";
marks.append(el("span", { class: "neutral", title, html: VK.icon("micSlash") }));
}
@@ -702,6 +708,53 @@ function memberNameNode(member, className = "vk-name") {
});
}
/* Speaking marks. Bars stay on until an explicit stop, a mute, or a channel
change: Discord's speaking events are not reliable enough for an idle decay.
The fade runs the equalizer animation inside a shrinking container, so the
amplitude drops instead of the whole block collapsing at full height. */
const SPEAKING_FADE_MS = 750;
function renderSpeakingViews() {
renderVoiceRail();
renderPeople();
renderRightVoice();
renderRightFavourites();
renderCallStrip();
}
function scheduleSpeakingRender() {
const now = Date.now();
if (now - state.speakingRenderAt < 250) return;
state.speakingRenderAt = now;
renderSpeakingViews();
}
function beginSpeaking(userId) {
const fade = state.speakingFade.get(userId);
if (fade) {
clearTimeout(fade);
state.speakingFade.delete(userId);
}
const wasSpeaking = state.speaking.has(userId);
state.speaking.add(userId);
if (!wasSpeaking) scheduleSpeakingRender();
}
function endSpeaking(userId) {
if (!state.speaking.delete(userId)) return;
const fadeTimer = setTimeout(() => {
state.speakingFade.delete(userId);
renderSpeakingViews();
}, SPEAKING_FADE_MS);
state.speakingFade.set(userId, fadeTimer);
scheduleSpeakingRender();
}
function trackPeople(members) {
for (const member of members ?? []) {
if (!member?.id || member.id === state.user?.id) continue;
@@ -1208,6 +1261,7 @@ function toggleTts() {
const panel = $("#tts-panel");
panel.hidden = !panel.hidden;
$("#tts-target").textContent = `as ${state.user?.username ?? "account"} → ${channelLabel()}`;
if (!panel.hidden) void refreshSpeech();
}
/* Right column */
@@ -1664,10 +1718,10 @@ function fillDeviceSelect(select, devices, defaultLabel) {
async function refreshAudio() {
try {
const status = await VK.api.get("/api/audio");
state.audioRecording = !!status.recording;
state.audioRecording = !!status.capturing;
$("#audio-status").textContent = status.available
? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.recording ? " · recording" : ""}`
? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.capturing ? " · capturing" : ""}`
: (status.error ?? "unavailable");
const hintRow = $("#audio-hint-row");
@@ -1678,7 +1732,15 @@ async function refreshAudio() {
$("#audio-tts").textContent = `local ${tts.local ? "yes" : "no"} · api ${
tts.api ? "yes" : "no"
} · player ${status.player ? "yes" : "no"} · queue ${status.queue ?? 0}`;
$("#audio-capture").textContent = status.recording ? "Stop capture" : "Start capture";
$("#audio-capture").textContent = status.capturing ? "Stop capture" : "Start capture";
const duck = $("#tts-duck");
if (duck) {
const on = status.duckEnabled !== false;
duck.classList.toggle("vk-chip-on", on);
duck.classList.toggle("vk-chip-off", !on);
duck.textContent = on ? "duck on voice" : "duck off";
}
} catch (error) {
$("#audio-status").textContent = error.message;
}
@@ -1690,11 +1752,17 @@ async function refreshAudio() {
/* Speech settings (server-side preferences). */
async function refreshSpeech() {
let engines = [];
let defaultProvider = "local";
try {
const status = await VK.api.get("/api/audio");
const tts = status.tts ?? {};
const stt = status.stt ?? {};
engines = tts.engines ?? [];
defaultProvider = tts.defaultProvider ?? "local";
$("#speech-status").textContent = `${tts.engine ?? "no local engine"}${tts.api ? " + api" : ""}`;
$("#speech-stt").textContent = stt.local
? `local ready${stt.engine ? ` (${stt.engine})` : ""}`
@@ -1705,36 +1773,77 @@ async function refreshSpeech() {
// the daemon status is optional here
}
let groups = [];
try {
groups = await VK.api.get("/api/audio/tts/voices");
} catch {
// voices need a local engine
}
try {
const prefs = await VK.api.get("/api/preferences");
state.speech = prefs.tts ?? { provider: null, language: null, voice: null };
const languageSelect = $("#speech-language");
languageSelect.replaceChildren(el("option", { value: "", text: "Auto" }));
for (const group of groups) {
languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` }));
}
languageSelect.value = state.speech.language ?? "";
fillVoiceOptions(groups, state.speech.language, state.speech.voice);
$("#speech-provider").value = state.speech.provider ?? "";
$("#tts-provider").value = state.speech.provider ?? "";
state.speech = prefs.tts ?? { provider: null, language: null, voice: null, speed: null, pitch: null };
} catch {
// preferences are optional
}
const selected = resolveEngine(state.speech.provider, engines, defaultProvider);
const selectedEngine = engines.find(engine => engine.id === selected);
const hasLanguage = selectedEngine?.language !== false;
fillEngineOptions($("#speech-provider"), engines, state.speech.provider ?? selected);
fillEngineOptions($("#tts-provider"), engines, state.speech.provider ?? selected);
let groups = [];
try {
groups = await VK.api.get(`/api/audio/tts/voices?engine=${encodeURIComponent(selected)}`);
} catch {
// voices need an engine
}
const languageSelect = $("#speech-language");
languageSelect.replaceChildren(el("option", { value: "", text: "Auto" }));
for (const group of groups) {
languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` }));
}
languageSelect.value = state.speech.language ?? "";
$("#speech-language-row").hidden = !hasLanguage;
const voices = voiceList(groups, hasLanguage ? state.speech.language : null, hasLanguage);
fillVoiceSelect($("#speech-voice"), voices, state.speech.voice);
fillVoiceSelect($("#tts-voice"), voices, state.speech.voice);
syncVoiceControls(voices, state.speech.voice);
fillSpeechSliders(state.speech);
}
function fillVoiceOptions(groups, language, selected) {
const select = $("#speech-voice");
const voices = groups.find(group => group.id === language)?.voices ?? [];
/** Resolve the engine that will actually run: provider override first, then the auto default. */
function resolveEngine(provider, engines, defaultProvider) {
if (provider === "espeak" || provider === "command" || provider === "api") return provider;
if (defaultProvider === "espeak" || defaultProvider === "command" || defaultProvider === "api") return defaultProvider;
const available = id => engines.some(engine => engine.id === id && engine.available);
if (available("command")) return "command";
if (available("espeak")) return "espeak";
return "api";
}
function fillEngineOptions(select, engines, value) {
if (!select) return;
select.replaceChildren(el("option", { value: "", text: "Auto" }));
for (const engine of engines) {
const option = el("option", {
value: engine.id,
text: engine.available ? engine.name : `${engine.name} (unavailable)`
});
option.disabled = !engine.available;
select.append(option);
}
select.value = value;
}
function voiceList(groups, language, hasLanguage) {
if (!hasLanguage) return (groups ?? []).flatMap(group => group.voices ?? []);
if (!language) return [];
return groups?.find(group => group.id === language)?.voices ?? [];
}
function fillVoiceSelect(select, voices, selected) {
if (!select) return;
select.replaceChildren(el("option", { value: "", text: "Default" }));
for (const voice of voices) {
@@ -1745,15 +1854,38 @@ function fillVoiceOptions(groups, language, selected) {
select.value = selected && voices.some(voice => voice.id === selected) ? selected : "";
}
async function saveSpeech() {
function fillVoiceOptions(groups, language, selected) {
fillVoiceSelect($("#speech-voice"), voiceList(groups, language, !!language), selected);
}
async function saveSpeech(extra = {}) {
const provider = $("#speech-provider").value || null;
const language = $("#speech-language").value || null;
const language = $("#speech-language-row").hidden
? (state.speech.language ?? null)
: $("#speech-language").value || null;
const voice = $("#speech-voice").value || null;
const speed = Number($("#tts-speed").value);
const pitch = Number($("#tts-pitch").value);
const expressionEl = $("#tts-expression");
const expression =
expressionEl.dataset.dirty === "1" || typeof state.speech.expression === "number"
? Number(expressionEl.value)
: null;
try {
const prefs = await VK.api.post("/api/preferences", { tts: { provider, language, voice } });
state.speech = prefs.tts ?? { provider: null, language: null, voice: null };
const prefs = await VK.api.post("/api/preferences", {
tts: { provider, language, voice, speed, pitch, expression, ...effectOptions(), ...extra }
});
state.speech = prefs.tts ?? {
provider: null,
language: null,
voice: null,
speed: null,
pitch: null,
expression: null
};
$("#tts-provider").value = state.speech.provider ?? "";
fillSpeechSliders(state.speech);
await refreshSpeech();
toast("Speech settings saved", "ok");
} catch (error) {
@@ -1761,6 +1893,205 @@ async function saveSpeech() {
}
}
async function warmEngine() {
const button = $("#speech-warm");
const engine = $("#speech-provider").value || null;
const label = $("#speech-provider").selectedOptions[0]?.textContent ?? "engine";
button.disabled = true;
button.textContent = "Starting…";
try {
const result = await VK.api.post("/api/audio/tts/warm", engine ? { engine } : {});
toast(result.warmed ? `${label} ready in ${result.ms} ms` : `${label} needs no warm-up`, "ok");
$("#speech-status").textContent = result.warmed ? `${label} ready (${result.ms} ms)` : label;
} catch (error) {
toast(error.message, "error");
} finally {
button.disabled = false;
button.textContent = "Start engine";
}
}
function fillSpeechSliders(tts) {
const speed = typeof tts?.speed === "number" ? tts.speed : 1;
const pitch = typeof tts?.pitch === "number" ? tts.pitch : 0;
$("#tts-speed").value = Number.isFinite(speed) ? speed : 1;
$("#tts-pitch").value = Number.isFinite(pitch) ? pitch : 0;
syncSpeechSliderLabels();
}
function syncSpeechSliderLabels() {
const speed = Number($("#tts-speed").value);
const pitch = Number($("#tts-pitch").value);
const pitchSlider = $("#tts-pitch");
$("#tts-speed-value").textContent = `${speed.toFixed(2)}x`;
$("#tts-pitch-value").textContent = pitchSlider.disabled ? "n/a" : `${pitch > 0 ? "+" : ""}${pitch} st`;
const expressionSlider = $("#tts-expression");
if (expressionSlider) {
const explicit = expressionSlider.dataset.dirty === "1" || expressionSlider.dataset.stored === "1";
$("#tts-expression-value").textContent = explicit ? expressionSlider.value : "default";
}
for (const button of $$("#tts-speed-presets .vk-chip")) {
const active = Math.abs(Number(button.dataset.speed) - speed) < 0.001;
button.classList.toggle("vk-chip-on", active);
button.classList.toggle("vk-chip-off", !active);
}
}
/* Flextalk profile controls (VOX/HEV kit): profile values, effects, word mode. */
const PROFILE_SPEC = [
["pitch", 0, 120],
["rate", 0, 100],
["prominences", 0, 100],
["head_size", 0, 100],
["lowering", 0, 100]
];
function buildProfileSliders() {
const root = $("#tts-profile-sliders");
if (!root) return;
for (const [key, min, max] of PROFILE_SPEC) {
const slider = el("input", { type: "range", min: String(min), max: String(max), step: "1" });
slider.id = `tts-profile-${key}`;
slider.dataset.key = key;
const value = el("span", { class: "vk-mono vk-dim", text: "default" });
root.append(
el(
"div",
{ class: "vk-tts-profile-row" },
el("span", { class: "vk-micro-label", text: key.replace("_", " ") }),
slider,
value
)
);
slider.addEventListener("input", () => {
const engineDefault = state.profileDefaults?.[key];
const snapped = typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2;
if (snapped) {
slider.value = engineDefault;
slider.dataset.dirty = "";
value.textContent = "default";
} else {
slider.dataset.dirty = "1";
value.textContent = slider.value;
}
});
slider.addEventListener("change", () => void saveSpeech());
}
}
function profileFromControls() {
const profile = {};
for (const [key] of PROFILE_SPEC) {
const slider = $(`#tts-profile-${key}`);
if (slider?.dataset.dirty === "1") profile[key] = Number(slider.value);
}
return profile;
}
function fillProfileControls() {
const stored = state.speech.profile ?? {};
fillProfileSlots();
for (const [key] of PROFILE_SPEC) {
const slider = $(`#tts-profile-${key}`);
if (!slider) continue;
if (typeof stored[key] === "number") {
slider.value = stored[key];
slider.dataset.dirty = "1";
} else {
const engineDefault = state.profileDefaults?.[key];
if (typeof engineDefault === "number") slider.value = engineDefault;
slider.dataset.dirty = "";
}
const value = slider.parentElement?.querySelector(".vk-mono");
if (value) value.textContent = slider.dataset.dirty === "1" ? slider.value : "default";
}
const highpass = Number(state.speech.highpass);
$("#tts-highpass").value = Number.isFinite(highpass) && state.speech.highpass !== null ? highpass : 40;
$("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`;
const echo = Number(state.speech.echo);
$("#tts-echo").value = Number.isFinite(echo) && state.speech.echo !== null ? echo : 0;
$("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off";
$("#tts-words").checked = state.speech.words === true;
const gap = Number(state.speech.wordGapMs);
$("#tts-word-gap").value = Number.isFinite(gap) && state.speech.wordGapMs !== null ? gap : 120;
$("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`;
}
function fillProfileSlots() {
const select = $("#tts-profile-slot");
if (!select) return;
const presets = state.speech.profilePresets ?? [];
const current = select.value || "0";
select.replaceChildren(
...Array.from({ length: 5 }, (_, index) => {
const preset = presets[index];
const label = preset ? `${index + 1} · ${preset.name ?? "profile"}` : `${index + 1} · empty`;
return el("option", { value: String(index), text: label });
})
);
select.value = current;
const preset = presets[Number(select.value)];
const name = $("#tts-profile-name");
if (name) name.value = preset?.name ?? "";
}
function effectOptions() {
return {
profile: profileFromControls(),
highpass: Number($("#tts-highpass").value),
echo: Number($("#tts-echo").value),
words: $("#tts-words").checked,
wordGapMs: Number($("#tts-word-gap").value)
};
}
function syncVoiceControls(voices, selected) {
const voice = (voices ?? []).find(entry => entry.id === selected);
const pitchable = !voice || voice.supportsPitch !== false;
$("#tts-pitch-control").hidden = !pitchable;
$("#tts-pitch").disabled = !pitchable;
const expressive = !!voice?.supportsExpression;
state.profileDefaults = voice?.profileDefaults ?? {};
state.expressionDefault = typeof voice?.expressionDefault === "number" ? voice.expressionDefault : null;
if (expressive) fillProfileControls();
{
const slider = $("#tts-expression");
const pref = state.speech.expression;
if (typeof pref === "number") {
slider.value = pref;
} else if (typeof voice.expressionDefault === "number") {
slider.value = voice.expressionDefault;
} else {
slider.value = 50;
}
slider.dataset.stored = typeof pref === "number" ? "1" : "0";
delete slider.dataset.dirty;
}
syncSpeechSliderLabels();
}
/* Browser audio (puppeting) — separate from the bot's own audio devices. */
async function refreshBrowserAudio() {
@@ -1793,19 +2124,38 @@ async function refreshBrowserAudio() {
}
async function grantBrowserAudio() {
if (await ensureBrowserAudio()) toast("Browser audio enabled", "ok");
}
/** Get the browser ready for playback: permission, device labels, and a live
* AudioContext. Must run inside the click so the context is not suspended. */
async function ensureBrowserAudio() {
if (!navigator.mediaDevices?.getUserMedia) {
toast("Browser audio needs HTTPS or localhost", "error");
return;
return false;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
for (const track of stream.getTracks()) track.stop();
toast("Browser audio enabled", "ok");
await refreshBrowserAudio();
} catch (error) {
toast(`Could not enable browser audio: ${error.message}`, "error");
return false;
}
if (!state.clientAudio.ctx) {
state.clientAudio.ctx = new AudioContext({ sampleRate: 48000 });
applyBrowserSink();
}
if (state.clientAudio.ctx.state === "suspended") {
await state.clientAudio.ctx.resume().catch(error => {
console.warn("[client-audio] could not resume the audio context:", error);
});
}
await refreshBrowserAudio();
return true;
}
function applyBrowserSink() {
@@ -1824,6 +2174,8 @@ function playPcm(arrayBuffer) {
}
const context = state.clientAudio.ctx;
if (context.state === "suspended") void context.resume();
const samples = new Int16Array(arrayBuffer);
const buffer = context.createBuffer(1, samples.length, 48000);
const data = buffer.getChannelData(0);
@@ -1846,7 +2198,10 @@ function updateClientAudioButton() {
if (!button) return;
button.classList.toggle("active", state.clientAudio.on);
button.textContent = state.clientAudio.on ? "Client audio on" : "Client audio";
const label = button.querySelector(".vk-btn-text");
const text = state.clientAudio.on ? "Client audio on" : "Client audio";
if (label) label.textContent = text;
else button.textContent = text;
const status = $("#browser-audio-status");
if (status) status.textContent = state.clientAudio.on ? "on" : "off";
@@ -1911,9 +2266,17 @@ function stopClientAudio() {
updateClientAudioButton();
}
function toggleClientAudio() {
if (state.clientAudio.on || state.clientAudio.ws) stopClientAudio();
else startClientAudio();
async function toggleClientAudio() {
if (state.clientAudio.on || state.clientAudio.ws) {
stopClientAudio();
return;
}
// Clicking for audio is the consent: unlock the browser in the same gesture
// so playback is not blocked by a suspended AudioContext.
if (!(await ensureBrowserAudio())) return;
startClientAudio();
}
function syncTtsControls() {
@@ -1921,6 +2284,17 @@ function syncTtsControls() {
if (speak) speak.disabled = !state.takeover;
}
async function setCapture(start) {
try {
await VK.api.post("/api/audio/capture", { start });
toast(start ? "Capture started" : "Capture stopped", "ok");
await refreshTranscript();
void refreshAudio();
} catch (error) {
toast(error.message, "error");
}
}
async function speakText() {
if (!state.takeover) {
toast("Composer is read-only — turn Takeover on to speak", "error");
@@ -1932,7 +2306,20 @@ async function speakText() {
try {
const provider = $("#tts-provider").value;
await VK.api.post("/api/audio/speak", { text, provider });
const speed = Number($("#tts-speed").value);
const pitch = Number($("#tts-pitch").value);
const expressionEl = $("#tts-expression");
const expression =
expressionEl.dataset.dirty === "1" ? Number(expressionEl.value) : state.speech.expression ?? undefined;
await VK.api.post("/api/audio/speak", {
text,
provider,
speed,
pitch,
...(typeof expression === "number" ? { expression } : {}),
...effectOptions()
});
$("#tts-status").textContent = "queued";
toast("Queued for speech", "ok");
void refreshAudio();
@@ -1961,6 +2348,7 @@ function transcriptLine(segment) {
async function refreshTranscript() {
try {
const result = await VK.api.get("/api/transcript?limit=200");
const capture = await VK.api.get("/api/audio").catch(() => null);
const list = $("#transcript-list");
list.replaceChildren();
@@ -1969,7 +2357,7 @@ async function refreshTranscript() {
list.append(
el("div", {
class: "vk-empty",
text: "No captured segments yet. Start capture in Settings → Client → Bot audio."
text: "No captured segments yet. Press Start above."
})
);
} else {
@@ -1978,9 +2366,10 @@ async function refreshTranscript() {
}
const active = [...(result.sessions ?? [])].reverse().find(session => !session.endedAt);
const captureState = capture?.capturing ? "capturing" : "capture off";
$("#transcript-status").textContent = active
? `session ${active.id.slice(-8)} · ${segments.length} segments`
: `${segments.length} segments`;
? `${captureState} · session ${active.id.slice(-8)} · ${segments.length} segments`
: `${captureState} · ${segments.length} segments`;
} catch (error) {
toast(error.message, "error");
}
@@ -2139,22 +2528,16 @@ function handleLiveEvent(event) {
}
if (event.type === "VOICE_SPEAKING") {
if (event.speaking) state.speaking.add(event.userId);
else state.speaking.delete(event.userId);
const now = Date.now();
if (now - state.speakingRenderAt > 500) {
state.speakingRenderAt = now;
renderVoiceRail();
renderPeople();
renderRightVoice();
renderRightFavourites();
renderCallStrip();
}
if (event.speaking) beginSpeaking(event.userId);
else endSpeaking(event.userId);
return;
}
if (event.type === "VOICE_STATE_UPDATE") {
// A mute or a channel change ends speaking immediately: a muted member
// never transmits, and the stop event may not arrive.
if (!event.channelId || event.mute || event.selfMute) endSpeaking(event.userId);
const now = Date.now();
if (now - state.voiceReloadAt > 1000) {
state.voiceReloadAt = now;
@@ -2282,6 +2665,18 @@ function init() {
if (isMobileLayout()) setMobileTab("activity");
});
$("#transcript-refresh").addEventListener("click", () => void refreshTranscript());
$("#transcript-start").addEventListener("click", () => void setCapture(true));
$("#transcript-stop").addEventListener("click", () => void setCapture(false));
$("#tts-duck").addEventListener("click", async () => {
const on = !$("#tts-duck").classList.contains("vk-chip-on");
try {
await VK.api.post("/api/preferences", { audio: { duck: on } });
toast(on ? "TTS ducking on" : "TTS ducking off", "ok");
void refreshAudio();
} catch (error) {
toast(error.message, "error");
}
});
$("#channel-refresh").addEventListener("click", () => void loadMessages({ refresh: true }));
$("#guilds-show-all").addEventListener("click", () => {
@@ -2310,13 +2705,193 @@ function init() {
void saveSpeech();
});
for (const id of ["tts-speed", "tts-pitch"]) {
$(`#${id}`).addEventListener("input", syncSpeechSliderLabels);
$(`#${id}`).addEventListener("change", () => void saveSpeech());
}
$("#tts-expression").addEventListener("input", () => {
const slider = $("#tts-expression");
const engineDefault = state.expressionDefault;
const snapped =
typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2;
if (snapped) {
slider.value = engineDefault;
slider.dataset.dirty = "";
} else {
slider.dataset.dirty = "1";
}
syncSpeechSliderLabels();
});
$("#tts-expression").addEventListener("change", () => void saveSpeech());
buildProfileSliders();
for (const id of ["tts-highpass", "tts-echo", "tts-word-gap"]) {
$(`#${id}`).addEventListener("input", () => {
$("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`;
$("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off";
$("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`;
});
$(`#${id}`).addEventListener("change", () => void saveSpeech());
}
$("#tts-words").addEventListener("change", () => void saveSpeech());
$("#profile-open").addEventListener("click", () => {
const panel = document.querySelector(".vk-profile-panel");
if (panel) {
panel.style.left = "";
panel.style.top = "";
panel.style.transform = "";
}
$("#profile-overlay").hidden = false;
fillProfileControls();
});
$("#profile-close").addEventListener("click", () => {
$("#profile-overlay").hidden = true;
});
$("#profile-overlay").addEventListener("click", event => {
if (window.__profileDragged) {
window.__profileDragged = false;
return;
}
if (event.target === $("#profile-overlay")) $("#profile-overlay").hidden = true;
});
(() => {
const panel = document.querySelector(".vk-profile-panel");
const handle = $("#profile-drag");
if (!panel || !handle) return;
let dragging = false;
let startX = 0;
let startY = 0;
let originX = 0;
let originY = 0;
handle.addEventListener("mousedown", event => {
const rect = panel.getBoundingClientRect();
dragging = true;
startX = event.clientX;
startY = event.clientY;
originX = rect.left;
originY = rect.top;
panel.style.transform = "none";
panel.style.left = `${originX}px`;
panel.style.top = `${originY}px`;
event.preventDefault();
});
window.addEventListener("mousemove", event => {
if (!dragging) return;
if (Math.abs(event.clientX - startX) + Math.abs(event.clientY - startY) > 3) {
window.__profileDragged = true;
}
panel.style.left = `${originX + event.clientX - startX}px`;
panel.style.top = `${originY + event.clientY - startY}px`;
});
window.addEventListener("mouseup", () => {
dragging = false;
});
})();
for (const slot of ["1", "2"]) {
const input = $(`#tts-test-custom-${slot}`);
input.value = localStorage.getItem(`veskslop-test-line-${slot}`) ?? "";
input.addEventListener("input", () =>
localStorage.setItem(`veskslop-test-line-${slot}`, input.value)
);
}
$$(".vk-profile-tests button").forEach(button => {
button.addEventListener("click", async () => {
let text = button.dataset.test ?? "";
if (button.dataset.testCustom) {
text = $(`#tts-test-custom-${button.dataset.testCustom}`).value.trim();
if (!text) {
toast("Custom line is empty", "error");
return;
}
}
try {
const expressionEl = $("#tts-expression");
const expression =
expressionEl.dataset.dirty === "1"
? Number(expressionEl.value)
: state.speech.expression ?? undefined;
await VK.api.post("/api/audio/speak", {
text,
provider: $("#tts-provider").value,
...effectOptions(),
...(typeof expression === "number" ? { expression } : {})
});
toast(`Playing "${text}"`, "ok");
} catch (error) {
toast(error.message, "error");
}
});
});
$("#tts-profile-slot").addEventListener("change", () => {
const preset = (state.speech.profilePresets ?? [])[Number($("#tts-profile-slot").value)];
$("#tts-profile-name").value = preset?.name ?? "";
});
$("#tts-profile-save").addEventListener("click", async () => {
const slot = Number($("#tts-profile-slot").value);
const presets = (state.speech.profilePresets ?? [null, null, null, null, null]).slice(0, 5);
const name = $("#tts-profile-name").value.trim() || `Profile ${slot + 1}`;
presets[slot] = {
name,
...profileFromControls(),
...(typeof state.speech.expression === "number" ? { expression: state.speech.expression } : {})
};
await saveSpeech({ profilePresets: presets });
toast(`Saved "${name}" to slot ${slot + 1}`, "ok");
});
$("#tts-profile-load").addEventListener("click", async () => {
const slot = Number($("#tts-profile-slot").value);
const preset = (state.speech.profilePresets ?? [])[slot];
if (!preset) {
toast(`Slot ${slot + 1} is empty`, "error");
return;
}
for (const [key] of PROFILE_SPEC) {
const slider = $(`#tts-profile-${key}`);
if (!slider || typeof preset[key] !== "number") continue;
slider.value = preset[key];
slider.dataset.dirty = "1";
}
if (typeof preset.expression === "number") {
$("#tts-expression").value = preset.expression;
$("#tts-expression").dataset.dirty = "1";
}
fillProfileControls();
syncSpeechSliderLabels();
await saveSpeech();
toast(`Loaded "${preset.name ?? `Profile ${slot + 1}`}"`, "ok");
});
$("#tts-speed-presets").addEventListener("click", event => {
const button = event.target.closest("button[data-speed]");
if (!button) return;
$("#tts-speed").value = button.dataset.speed;
syncSpeechSliderLabels();
void saveSpeech();
});
syncSpeechSliderLabels();
$("#speech-provider").addEventListener("change", () => void saveSpeech());
$("#speech-warm").addEventListener("click", () => void warmEngine());
$("#speech-language").addEventListener("change", () => {
$("#speech-voice").value = "";
void saveSpeech();
});
$("#speech-voice").addEventListener("change", () => void saveSpeech());
$("#tts-voice").addEventListener("change", () => {
$("#speech-voice").value = $("#tts-voice").value;
void saveSpeech();
});
$("#scope-guild").addEventListener("click", () => void applyScope("guild"));
$("#scope-dm").addEventListener("click", () => void applyScope("dm"));
+113 -11
View File
@@ -129,12 +129,34 @@
</div>
<div class="vk-tts-settings">
<span class="vk-micro-label">Voice settings</span>
<select id="tts-provider" class="vk-select">
<option value="local">local TTS</option>
<option value="api">API TTS</option>
</select>
<span class="vk-chip vk-chip-on">duck on voice</span>
<select id="tts-provider" class="vk-select"></select>
<select id="tts-voice" class="vk-select" title="Voice"></select>
<button id="tts-duck" class="vk-chip vk-chip-on" type="button" title="Lower the TTS while someone else speaks">
duck on voice
</button>
<span class="vk-chip vk-chip-on">queue lines</span>
<div class="vk-tts-speed">
<label class="vk-tts-slider" title="Speech rate; 1.0x is the engine's natural rate">
<span class="vk-micro-label">speed</span>
<input id="tts-speed" type="range" min="0.5" max="2" step="0.05" value="1" />
<span id="tts-speed-value" class="vk-mono vk-dim">1.0x</span>
</label>
<div id="tts-speed-presets" class="vk-tts-presets">
<button class="vk-chip vk-chip-off" type="button" data-speed="0.5">0.5x</button>
<button class="vk-chip vk-chip-off" type="button" data-speed="0.75">0.75x</button>
<button class="vk-chip vk-chip-on" type="button" data-speed="1">1x</button>
<button class="vk-chip vk-chip-off" type="button" data-speed="1.25">1.25x</button>
<button class="vk-chip vk-chip-off" type="button" data-speed="1.5">1.5x</button>
</div>
</div>
<label id="tts-pitch-control" class="vk-tts-slider" title="Pitch offset; 0 is the engine's natural pitch">
<span class="vk-micro-label">pitch</span>
<input id="tts-pitch" type="range" min="-12" max="12" step="1" value="0" />
<span id="tts-pitch-value" class="vk-mono vk-dim">0 st</span>
</label>
<button id="profile-open" class="vk-chip vk-chip-off" type="button" title="Flextalk voice profile">
voice profile…
</button>
</div>
<div class="vk-tts-body">
<textarea id="tts-input" class="vk-tts-field" rows="3" placeholder="Speak text into the call…"></textarea>
@@ -216,6 +238,8 @@
<div class="vk-transcript">
<div class="vk-transcript-toolbar">
<span id="transcript-status" class="vk-mono vk-dim">no capture session</span>
<button id="transcript-start" class="btn btn-secondary vk-small-btn" type="button">Start</button>
<button id="transcript-stop" class="btn btn-secondary vk-small-btn" type="button">Stop</button>
<button id="transcript-refresh" class="btn btn-secondary vk-small-btn" type="button">Refresh</button>
</div>
<div id="transcript-list" class="vk-transcript-list"></div>
@@ -307,13 +331,14 @@
<div class="vk-settings-card">
<div class="vk-status-row">
<span class="vk-status-label">TTS engine</span>
<select class="vk-select" id="speech-provider">
<option value="">Auto</option>
<option value="local">Local</option>
<option value="api">API</option>
</select>
<span class="vk-engine-row">
<select class="vk-select" id="speech-provider"></select>
<button id="speech-warm" class="btn btn-secondary vk-small-btn" type="button" title="Start the engine so the first line is fast">
Start engine
</button>
</span>
</div>
<div class="vk-status-row">
<div class="vk-status-row" id="speech-language-row">
<span class="vk-status-label">Language</span>
<select class="vk-select" id="speech-language">
<option value="">Auto</option>
@@ -427,6 +452,83 @@
</div>
</div>
<div id="profile-overlay" class="vk-overlay" hidden>
<div class="vk-profile-panel elev-lg" role="dialog" aria-label="Flextalk voice profile">
<div class="vk-settings-head" id="profile-drag">
<span class="vk-settings-title">Flextalk voice profile</span>
<span class="vk-mono vk-dim" id="profile-voice-label"></span>
<button id="profile-close" class="btn btn-ghost vk-icon-btn" type="button" aria-label="Close">
<span class="vk-icon" data-icon="x"></span>
</button>
</div>
<div class="vk-profile-body">
<section class="vk-profile-section">
<div class="vk-section-head">
<span class="vk-rail-label vk-rail-label-flat">Voice character</span>
<span class="vk-mono vk-dim">within 2 of the engine default snaps to default</span>
</div>
<div id="tts-profile-sliders" class="vk-profile-grid"></div>
<div class="vk-profile-row">
<span class="vk-micro-label">expression</span>
<input id="tts-expression" type="range" min="0" max="100" step="1" value="59" />
<span id="tts-expression-value" class="vk-mono vk-dim">default</span>
</div>
</section>
<section class="vk-profile-section">
<div class="vk-section-head">
<span class="vk-rail-label vk-rail-label-flat">Presets</span>
</div>
<div class="vk-profile-row">
<select id="tts-profile-slot" class="vk-select"></select>
<input id="tts-profile-name" class="input" placeholder="Profile name" maxlength="40" />
<button id="tts-profile-load" class="btn btn-secondary vk-small-btn" type="button">Load</button>
<button id="tts-profile-save" class="btn btn-primary vk-small-btn" type="button">Save</button>
</div>
</section>
<section class="vk-profile-section">
<div class="vk-section-head">
<span class="vk-rail-label vk-rail-label-flat">Radio kit</span>
<span class="vk-mono vk-dim">VOX / HEV</span>
</div>
<div class="vk-profile-row">
<span class="vk-micro-label">highpass</span>
<input id="tts-highpass" type="range" min="40" max="400" step="10" value="40" />
<span id="tts-highpass-value" class="vk-mono vk-dim">40Hz</span>
</div>
<div class="vk-profile-row">
<span class="vk-micro-label">echo</span>
<input id="tts-echo" type="range" min="0" max="100" step="5" value="0" />
<span id="tts-echo-value" class="vk-mono vk-dim">off</span>
</div>
<div class="vk-profile-row">
<label class="vk-tts-check"><input id="tts-words" type="checkbox" /> word mode</label>
<input id="tts-word-gap" type="range" min="0" max="500" step="10" value="120" />
<span id="tts-word-gap-value" class="vk-mono vk-dim">120ms</span>
</div>
</section>
<section class="vk-profile-section">
<div class="vk-section-head">
<span class="vk-rail-label vk-rail-label-flat">Test lines</span>
</div>
<div class="vk-profile-tests">
<button class="vk-chip vk-chip-off" data-test="Attention" type="button">Attention</button>
<button class="vk-chip vk-chip-off" data-test="Laboratory" type="button">Laboratory</button>
<button class="vk-chip vk-chip-off" data-test="Critical" type="button">Critical</button>
<button class="vk-chip vk-chip-off" data-test="Health" type="button">Health</button>
<button class="vk-chip vk-chip-off" data-test="Radiation" type="button">Radiation</button>
<input class="input vk-test-custom" id="tts-test-custom-1" placeholder="custom 1" />
<button class="vk-chip vk-chip-off" data-test-custom="1" type="button">play</button>
<input class="input vk-test-custom" id="tts-test-custom-2" placeholder="custom 2" />
<button class="vk-chip vk-chip-off" data-test-custom="2" type="button">play</button>
</div>
</section>
</div>
</div>
</div>
<div id="toasts" class="vk-toasts"></div>
<script src="icons.js"></script>
+224 -5
View File
@@ -950,8 +950,8 @@ a.vk-attachment:hover {
.vk-composer-input {
flex: 1;
min-height: 38px;
max-height: 160px;
resize: none;
max-height: 50vh;
resize: vertical;
padding: 8px 10px;
border-radius: var(--radius-sm);
background: var(--color-bg);
@@ -1016,6 +1016,46 @@ a.vk-attachment:hover {
padding: 9px 11px 8px;
}
.vk-tts-slider {
display: flex;
align-items: center;
gap: 6px;
flex: none;
}
.vk-tts-slider input[type="range"] {
width: 84px;
accent-color: var(--color-accent);
}
.vk-tts-slider .vk-mono {
min-width: 34px;
text-align: right;
}
.vk-tts-speed {
display: flex;
flex-direction: column;
gap: 4px;
}
.vk-tts-presets {
display: flex;
gap: 4px;
padding-left: 2px;
}
.vk-tts-presets .vk-chip {
padding: 3px 7px;
font-size: 10px;
}
.vk-engine-row {
display: flex;
align-items: center;
gap: 6px;
}
.vk-chip {
display: flex;
align-items: center;
@@ -1027,6 +1067,14 @@ a.vk-attachment:hover {
box-shadow: inset 0 0 0 1px var(--color-accent-800);
color: var(--color-neutral-300);
font: 400 11px/1 ui-monospace, Menlo, monospace;
border: none;
cursor: pointer;
}
.vk-chip-off {
background: var(--color-bg);
box-shadow: inset 0 0 0 1px var(--color-neutral-800);
color: var(--color-neutral-500);
}
.vk-chip-on {
@@ -1047,8 +1095,9 @@ a.vk-attachment:hover {
.vk-tts-field {
width: 100%;
max-height: 150px;
resize: none;
min-height: 64px;
max-height: 40vh;
resize: vertical;
border: none;
outline: none;
background: transparent;
@@ -1203,7 +1252,11 @@ a.vk-attachment:hover {
}
.vk-transcript-toolbar .vk-small-btn {
margin-left: auto;
margin-left: 0;
}
.vk-transcript-toolbar #transcript-status {
margin-right: auto;
}
.vk-transcript-list {
@@ -1266,6 +1319,9 @@ a.vk-attachment:hover {
align-items: flex-end;
gap: 2px;
height: 11px;
transform-origin: bottom;
transform: scaleY(0);
opacity: 0;
}
.vk-levels span {
@@ -1274,9 +1330,49 @@ a.vk-attachment:hover {
border-radius: 1px;
background: var(--color-accent);
transform-origin: bottom;
}
.vk-levels.speaking {
opacity: 1;
transform: scaleY(1);
animation: vk-levels-in 200ms ease-out;
}
.vk-levels.speaking span,
.vk-levels.stopping span {
animation: vk-lvl 0.7s ease-in-out infinite;
}
.vk-levels.stopping {
animation: vk-levels-out 700ms ease-in forwards;
}
@keyframes vk-levels-in {
from {
opacity: 0;
transform: scaleY(0.05);
}
to {
opacity: 1;
transform: scaleY(1);
}
}
@keyframes vk-levels-out {
0% {
opacity: 1;
transform: scaleY(1);
}
60% {
opacity: 0.8;
transform: scaleY(0.35);
}
100% {
opacity: 0;
transform: scaleY(0.02);
}
}
.vk-levels span:nth-child(2) {
animation-delay: 0.1s;
}
@@ -1308,6 +1404,19 @@ a.vk-attachment:hover {
}
@media (prefers-reduced-motion: reduce) {
.vk-levels,
.vk-levels.speaking,
.vk-levels.stopping {
animation: none;
}
.vk-levels.speaking {
opacity: 1;
transform: scaleY(1);
}
.vk-levels.stopping {
opacity: 0;
transform: scaleY(0);
}
.vk-levels span {
animation: none;
transform: scaleY(0.6);
@@ -1482,6 +1591,116 @@ a.vk-attachment:hover {
display: none;
}
/* Flextalk voice profile: a bottom drawer over the chat, separate from the TTS row. */
.vk-profile-panel {
position: fixed;
left: 50%;
top: 10vh;
transform: translateX(-50%);
width: min(760px, 100%);
max-height: 76vh;
display: flex;
flex-direction: column;
background: var(--color-surface);
box-shadow: var(--shadow-lg);
border-radius: var(--radius-lg) var(--radius-lg) 0 0;
animation: vk-drawer-in 180ms ease-out;
}
.vk-profile-body {
flex: 1;
min-height: 0;
overflow-y: auto;
display: flex;
flex-direction: column;
gap: 16px;
padding: 6px 16px 18px;
}
.vk-profile-section {
display: flex;
flex-direction: column;
gap: 8px;
}
.vk-profile-section + .vk-profile-section {
padding-top: 14px;
box-shadow: inset 0 1px 0 var(--color-neutral-800);
}
.vk-profile-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 6px 20px;
}
.vk-profile-row {
display: flex;
align-items: center;
gap: 8px;
}
.vk-profile-row input[type="range"] {
flex: 1;
min-width: 80px;
accent-color: var(--color-accent);
}
.vk-profile-row .vk-mono {
min-width: 44px;
text-align: right;
}
.vk-profile-row .input {
flex: 1;
min-height: 28px;
}
#profile-drag {
cursor: move;
user-select: none;
}
.vk-profile-tests {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 6px;
}
.vk-test-custom {
width: 110px;
min-height: 26px;
}
@keyframes vk-drawer-in {
from {
transform: translateY(26px);
opacity: 0;
}
to {
transform: translateY(0);
opacity: 1;
}
}
@media (max-width: 768px) {
.vk-profile-panel {
max-height: 82vh;
}
.vk-profile-grid {
grid-template-columns: minmax(0, 1fr);
}
}
@media (prefers-reduced-motion: reduce) {
.vk-profile-panel {
animation: none;
}
}
.vk-settings-panel {
width: min(680px, 100%);
height: 100%;
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env bash
# Veskslop audio doctor. Run it on the server while the bot sits in a call and
# people are talking. It shows where client audio goes and what the daemon sees.
set -uo pipefail
PORT="${WEBUI_PORT:-8789}"
section() { printf '\n== %s ==\n' "$1"; }
section "host addresses"
hostname -I 2>/dev/null || true
section "default sink"
pactl get-default-sink 2>&1 || true
section "sinks"
pactl list short sinks 2>&1 || true
section "sink inputs: who is playing, and into which sink"
if command -v pactl >/dev/null; then
pactl list short sink-inputs 2>&1 || true
echo
echo "-- vesktop stream state (Corked / Mute / Volume) --"
pactl list sink-inputs 2>&1 | grep -E "Sink Input|Sink:|Corked:|Mute:|Volume:|application\.(name|process\.binary)|media\.name" || true
fi
section "microphone sources the client reads"
if command -v pactl >/dev/null; then
pactl list short source-outputs 2>&1 || true
echo
echo "-- source output names --"
pactl list source-outputs 2>&1 | grep -E "Source Output #|Source:|application\.(name|process\.binary)|media\.name" || true
fi
section "client processes"
ps -eo pid,args 2>/dev/null | grep -iE "vesktop|electron" | grep -v grep || echo "no veskstop/electron process found"
section "pulse clients"
if command -v pactl >/dev/null; then
pactl list short clients 2>&1 || true
fi
section "daemon audio status"
if command -v curl >/dev/null; then
curl -sk "https://127.0.0.1:${PORT}/api/audio" || curl -s "http://127.0.0.1:${PORT}/api/audio"
echo
else
echo "curl is missing"
fi
section "bot identity"
if command -v curl >/dev/null; then
curl -sk "https://127.0.0.1:${PORT}/health" || curl -s "http://127.0.0.1:${PORT}/health"
echo
fi
section "bot voice state"
if command -v curl >/dev/null; then
curl -sk "https://127.0.0.1:${PORT}/api/voice" || curl -s "http://127.0.0.1:${PORT}/api/voice"
echo
fi
section "volumes: client output and per-user"
if command -v curl >/dev/null; then
curl -sk "https://127.0.0.1:${PORT}/api/voice/volume" || curl -s "http://127.0.0.1:${PORT}/api/voice/volume"
echo
fi
section "capture sink level for 5 seconds (talk during this)"
if command -v ffmpeg >/dev/null; then
ffmpeg -hide_banner -loglevel info -f pulse -i "veskslop_capture.monitor" -t 5 \
-af volumedetect -f null - 2>&1 | grep -E "mean_volume|max_volume" || true
else
echo "ffmpeg is missing"
fi
section "live events for 8 seconds (talk during this)"
if command -v curl >/dev/null; then
curl -skN --max-time 8 "https://127.0.0.1:${PORT}/api/events" 2>/dev/null \
| grep -E "VOICE_SPEAKING|VOICE_STATE_UPDATE" | head -10 || true
fi
printf '\nDaemon terminal lines to look for: [audio] moved stream, [capture] auto-start, ffmpeg errors.\n'
+39 -9
View File
@@ -10,7 +10,7 @@ import { AudioManager } from "./audio.js";
import type { CaptureSegment } from "./capture.js";
import { CapturePipeline } from "./capture.js";
import type { Stt } from "./stt.js";
import { Tts, type TtsOptions, type TtsVoice } from "./tts.js";
import { Tts, type TtsOptions, type TtsProvider, type TtsVoice } from "./tts.js";
import type { TranscriptSegment, TranscriptSession, TranscriptStore } from "./transcript.js";
import { DEFAULT_VAD, type VadConfig } from "./vad.js";
@@ -50,10 +50,12 @@ export class AudioService {
private readonly speakerIntervals: SpeakerInterval[] = [];
private readonly speakerNames = new Map<string, string>();
private playing = false;
private duckEnabled = true;
private nextId = 1;
private ducked = false;
private selfId?: string;
private capture?: CapturePipeline;
private captureAdoptTimer?: ReturnType<typeof setInterval>;
private session?: TranscriptSession;
private segmentCount = 0;
private initialized = false;
@@ -94,6 +96,7 @@ export class AudioService {
stt: this.stt.status(),
queue: this.queue.length,
speaking: this.playing,
duckEnabled: this.duckEnabled,
capturing: !!this.capture,
sessionId: this.session?.id ?? null,
segments: this.segmentCount,
@@ -101,6 +104,16 @@ export class AudioService {
};
}
/** Enable or disable ducking the TTS while others speak. */
setDuckEnabled(on: boolean): void {
this.duckEnabled = on;
void this.applyDuck(this.shouldDuck());
}
private shouldDuck(): boolean {
return this.duckEnabled && this.playing && this.speakers.size > 0;
}
async speak(text: string, options: TtsOptions = {}): Promise<SpeakResult> {
await this.init();
@@ -109,12 +122,18 @@ export class AudioService {
const tts = this.tts.status();
const provider = options.provider ?? tts.defaultProvider;
if (provider === "local" && !tts.local) {
throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)");
}
if (provider === "api" && !tts.api) {
throw new Error("API TTS is not configured (VESKSLOP_TTS_API_URL)");
}
if (provider === "command" && !tts.command) {
throw new Error("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)");
}
if (provider === "espeak" && !tts.espeak) {
throw new Error("espeak-ng is not installed");
}
if (provider === "local" && !tts.local) {
throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)");
}
const trimmed = text.trim();
if (!trimmed) throw new Error("text is required");
@@ -156,7 +175,7 @@ export class AudioService {
}
this.pruneIntervals();
await this.applyDuck(this.playing && this.speakers.size > 0);
await this.applyDuck(this.shouldDuck());
}
async startCapture(): Promise<{ sessionId: string }> {
@@ -185,6 +204,8 @@ export class AudioService {
});
this.capture.start();
void this.manager.adoptClientStreams();
this.captureAdoptTimer = setInterval(() => void this.manager.adoptClientStreams(), 5000);
this.session = session;
this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION", session });
@@ -192,6 +213,8 @@ export class AudioService {
}
stopCapture(): void {
clearInterval(this.captureAdoptTimer);
this.captureAdoptTimer = undefined;
this.capture?.stop();
this.capture = undefined;
@@ -209,8 +232,12 @@ export class AudioService {
return this.manager.startMonitor(onData, onError);
}
ttsVoices(): Promise<TtsVoice[]> {
return this.tts.voices();
ttsVoices(engine?: TtsProvider): Promise<TtsVoice[]> {
return this.tts.voices(engine);
}
warmTts(engine?: TtsProvider) {
return this.tts.warm(engine);
}
async dispose(): Promise<void> {
@@ -259,7 +286,10 @@ export class AudioService {
try {
const text = await this.stt.transcribe(record.wav);
if (!text) return;
if (!text) {
console.warn(`[capture] transcription empty for ${record.wav}`);
return;
}
this.transcripts.setSegmentText(record.id, text);
this.options.onTranscript?.({ type: "TRANSCRIPT_TEXT", id: record.id, text });
@@ -321,7 +351,7 @@ export class AudioService {
this.playing = true;
try {
const file = await this.tts.synthesize(item.text, item.options);
await this.applyDuck(this.speakers.size > 0);
await this.applyDuck(this.shouldDuck());
await this.manager.play(file);
} catch (error) {
console.error("[audio] speak failed:", (error as Error).message);
+104 -2
View File
@@ -13,6 +13,7 @@ const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts";
const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic";
const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture";
const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30);
const LEAD_MS = Math.min(Math.max(Number(process.env.VESKSLOP_TTS_LEAD_MS ?? 150) || 0, 0), 2000);
export class AudioUnavailableError extends Error {}
@@ -26,10 +27,15 @@ export interface AudioManagerStatus {
captureSink: string;
ducked: boolean;
duckPercent: number;
/** Lead-in silence in milliseconds before each TTS line. */
ttsLeadMs: number;
playing: boolean;
recording: boolean;
player: boolean;
ffmpeg: boolean;
clientMatch: string[];
adopted: number;
lastAdoptError?: string;
}
interface CommandResult {
@@ -69,6 +75,8 @@ export class AudioManager {
private backend: "pulse" | null = null;
private playerAvailable = false;
private ffmpegAvailable = false;
private adoptedTotal = 0;
private lastAdoptError?: string;
async detect(): Promise<void> {
const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]);
@@ -128,13 +136,24 @@ export class AudioManager {
captureSink: CAPTURE_SINK,
ducked: this.ducked,
duckPercent: DUCK_PERCENT,
ttsLeadMs: LEAD_MS,
playing: !!this.player,
recording: !!this.recorder,
player: this.playerAvailable,
ffmpeg: this.ffmpegAvailable
ffmpeg: this.ffmpegAvailable,
clientMatch: this.clientPatterns(),
adopted: this.adoptedTotal,
lastAdoptError: this.lastAdoptError
};
}
private clientPatterns(): string[] {
return (process.env.VESKSLOP_AUDIO_CLIENT_MATCH ?? "vesktop,electron,chromium,discord")
.split(",")
.map(part => part.trim().toLowerCase())
.filter(Boolean);
}
/** Create the virtual devices when they are missing. Safe to call twice. */
async ensureDevices(): Promise<void> {
if (!this.available) await this.detect();
@@ -173,6 +192,76 @@ export class AudioManager {
]);
this.modules.push(Number(id));
}
await this.adoptClientStreams();
}
/**
* Move the client's playback streams into the capture sink. Without this the
* capture sink stays silent and the browser monitor streams zero samples.
*/
async adoptClientStreams(): Promise<number> {
if (!this.available) return 0;
const patterns = this.clientPatterns();
if (!patterns.length) return 0;
const sinks = await run("pactl", ["list", "short", "sinks"]);
if (sinks.code !== 0) {
this.lastAdoptError = sinks.stderr.trim() || "pactl list sinks failed";
return 0;
}
const captureIndex = sinks.stdout
.split("\n")
.map(line => line.split(/\s+/))
.find(fields => fields[1] === CAPTURE_SINK)?.[0];
if (captureIndex === undefined) return 0;
const list = await run("pactl", ["list", "sink-inputs"]);
if (list.code !== 0) {
this.lastAdoptError = list.stderr.trim() || "pactl list sink-inputs failed";
return 0;
}
let moved = 0;
let index: string | undefined;
let sinkIndex: string | undefined;
let matchText = "";
const tryMove = async () => {
if (!index || sinkIndex === captureIndex) return;
if (!patterns.some(pattern => matchText.includes(pattern))) return;
const result = await run("pactl", ["move-sink-input", index, CAPTURE_SINK]);
if (result.code === 0) {
moved++;
this.adoptedTotal++;
console.log(`[audio] moved stream ${index} to ${CAPTURE_SINK}`);
} else {
this.lastAdoptError = result.stderr.trim() || `move-sink-input ${index} failed`;
}
};
for (const line of list.stdout.split("\n")) {
const header = line.match(/^Sink Input #(\d+)/);
if (header) {
await tryMove();
index = header[1];
sinkIndex = undefined;
matchText = "";
continue;
}
const sink = line.match(/^\s*Sink:\s*(\d+)/);
if (sink) sinkIndex = sink[1];
const property = line.match(/^\s*(application\.name|application\.process\.binary)\s*=\s*"?([^"]+)"?/);
if (property) matchText += ` ${property[2].toLowerCase()}`;
}
await tryMove();
return moved;
}
/** Stop children and unload the modules this daemon created. */
@@ -191,8 +280,15 @@ export class AudioManager {
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay";
// Lead-in silence: give Discord's voice pipeline a moment to settle
// before the first syllable, so the attack is not swallowed.
const args = ["-nodisp", "-autoexit", "-loglevel", "error"];
if (LEAD_MS > 0) args.push("-af", `adelay=${LEAD_MS}:all=1`);
args.push(file);
await new Promise<void>((resolve, reject) => {
const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], {
const child = spawn(binary, args, {
env: { ...process.env, PULSE_SINK: SINK },
stdio: ["ignore", "ignore", "pipe"]
});
@@ -278,6 +374,11 @@ export class AudioManager {
}
const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
// The client may start or restart its stream while we listen.
void this.adoptClientStreams();
const adoptTimer = setInterval(() => void this.adoptClientStreams(), 5000);
const child = spawn(
binary,
[
@@ -308,6 +409,7 @@ export class AudioManager {
});
return () => {
clearInterval(adoptTimer);
try {
child.kill("SIGTERM");
} catch {
+187 -9
View File
@@ -22,7 +22,7 @@ import { Stt } from "./stt.js";
import { TranscriptStore } from "./transcript.js";
import { DEFAULT_VAD } from "./vad.js";
import { CoreClient, coreSocketPath } from "./core-client.js";
import { CoreProcess } from "./core-process.js";
import { CoreProcess, staleBundleWarning } from "./core-process.js";
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js";
import { PreferencesStore } from "./preferences.js";
import { Puppet } from "./puppet.js";
@@ -49,6 +49,13 @@ async function connectLoop(): Promise<void> {
puppet = new Puppet(client, audio);
client.onEvent(event => {
// Speaking hints add to the reconciled set so the poller can
// clear users the store no longer reports as speaking.
if (event?.type === "VOICE_SPEAKING" && event.userId) {
if (event.speaking) speakingUsers.add(String(event.userId));
else speakingUsers.delete(String(event.userId));
}
void puppet!.enqueueEvent(event);
void audio.onEvent(event);
webui.broadcast(event);
@@ -56,6 +63,7 @@ async function connectLoop(): Promise<void> {
client.onClose(() => {
console.warn("[daemon] core bridge disconnected");
stopSpeakingPoll();
coreConnected = false;
core = undefined;
puppet = undefined;
@@ -66,7 +74,8 @@ async function connectLoop(): Promise<void> {
coreConnected = true;
console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`);
webui.broadcast({ type: "CORE_STATUS", coreConnected: true });
void autoConnectAudio(client);
startSpeakingPoll();
void autoConnectAudioWithRetry(client);
return;
} catch (error) {
@@ -89,6 +98,69 @@ function requirePuppet(): Puppet {
return puppet;
}
/*
* Discord's dispatcher only sends speaking-start events, so the UI and the
* transcript attribution need the store's authoritative set. Poll it once a
* second and synthesize the missing stop/start events. Dispatcher events feed
* the same set so a user the store no longer reports is cleared.
*/
let speakingTimer: ReturnType<typeof setInterval> | undefined;
const speakingUsers = new Set<string>();
function emitCoreEvent(event: any): void {
void puppet?.enqueueEvent(event);
void audio.onEvent(event);
webui.broadcast(event);
}
async function pollSpeaking(): Promise<void> {
if (!core) return;
try {
const result = (await core.voiceSpeaking()) as { channelId?: string | null; userIds?: string[] };
const next = new Set<string>(result?.channelId ? (result.userIds ?? []) : []);
for (const userId of next) {
if (!speakingUsers.has(userId)) {
emitCoreEvent({
type: "VOICE_SPEAKING",
userId,
speaking: true,
channelId: result.channelId ?? null
});
}
}
for (const userId of speakingUsers) {
if (!next.has(userId)) {
emitCoreEvent({
type: "VOICE_SPEAKING",
userId,
speaking: false,
channelId: result?.channelId ?? null
});
}
}
speakingUsers.clear();
for (const userId of next) speakingUsers.add(userId);
} catch {
// The close handler stops the poller.
}
}
function startSpeakingPoll(): void {
stopSpeakingPoll();
speakingTimer = setInterval(() => void pollSpeaking(), 1000);
}
function stopSpeakingPoll(): void {
if (speakingTimer) clearInterval(speakingTimer);
speakingTimer = undefined;
speakingUsers.clear();
}
const coreProcess = new CoreProcess({ tokenFile });
const transcripts = new TranscriptStore(
@@ -129,6 +201,7 @@ const audio = new AudioService({
onTranscript: event => webui.broadcast(event)
});
audio.setDuckEnabled(preferences.get().audio?.duck !== false);
void audio.init();
/** Point the client at the virtual microphone and capture sink when present. */
@@ -161,11 +234,25 @@ async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolea
console.log("[audio] client devices set:", JSON.stringify(result));
return { connected: true, message: "the client now uses the virtual devices" };
} catch (error) {
console.warn("[audio] device auto-connect failed:", (error as Error).message);
return { connected: false, message: (error as Error).message };
}
}
/**
* The bridge socket is up before the renderer finishes loading, so the first
* device calls can fail with "core is not ready" style errors. Retry.
*/
async function autoConnectAudioWithRetry(client: CoreClient): Promise<void> {
for (let attempt = 1; attempt <= 10; attempt++) {
if (attempt > 1) await sleep(2500);
const result = await autoConnectAudio(client);
if (result.connected) return;
console.warn(`[audio] device auto-connect attempt ${attempt}: ${result.message}`);
}
}
const auth = new AuthService(tokenFile, update => {
webui.broadcast({ type: "AUTH_QR", update });
@@ -385,12 +472,36 @@ function buildServer(): McpServer {
{
description: "Move the client to another voice channel.",
inputSchema: {
channelId: z.string().describe("Voice channel ID")
channelId: z.string().describe("Channel ID")
}
},
async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId)))
);
server.registerTool(
"get_user_volume",
{
description:
"Per-user playback volume (0 to 200, 100 is default) and the client's own output volume. A volume of 0 silences that user.",
inputSchema: {
userIds: z.array(z.string()).min(1).max(50).describe("User IDs to look up")
}
},
async ({ userIds }) => jsonText(await requireCore().userVolume(userIds))
);
server.registerTool(
"set_user_volume",
{
description: "Set a user's playback volume. 0 silences the user; 100 is default; 200 is the maximum.",
inputSchema: {
userId: z.string().describe("User ID"),
volume: z.number().int().min(0).max(200).describe("Volume from 0 to 200")
}
},
async ({ userId, volume }) => jsonText(await pacedVoice(() => requireCore().userSetVolume(userId, volume)))
);
server.registerTool(
"get_audio_status",
{
@@ -405,11 +516,58 @@ function buildServer(): McpServer {
description: "Queue text to speech into the bot's virtual microphone.",
inputSchema: {
text: z.string().min(1).max(1000).describe("Text to speak"),
provider: z.enum(["local", "api"]).optional().describe("TTS provider"),
voice: z.string().optional().describe("Voice name for the provider")
provider: z
.enum(["local", "espeak", "command", "api"])
.optional()
.describe("TTS engine: command is the configured VESKSLOP_TTS_CMD engine"),
voice: z.string().optional().describe("Voice name for the provider"),
speed: z.number().min(0.5).max(2).optional().describe("Rate multiplier, 1.0 is natural"),
pitch: z.number().min(-12).max(12).optional().describe("Pitch offset in semitones"),
expression: z
.number()
.min(0)
.max(100)
.optional()
.describe("Flextalk Expression profile value (Keith Bell, Kerry Watson)"),
profile: z
.record(z.string(), z.number())
.optional()
.describe("Flextalk profile overrides (pitch, rate, prominences, head_size, lowering)"),
highpass: z.number().min(40).max(400).optional().describe("High-pass filter Hz"),
echo: z.number().min(0).max(100).optional().describe("Mono echo amount"),
words: z.boolean().optional().describe("Synthesize each word and merge with gaps"),
wordGapMs: z.number().min(0).max(500).optional().describe("Silence between words in ms")
}
},
async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice }))
async ({ text, provider, voice, speed, pitch, expression, profile, highpass, echo, words, wordGapMs }) =>
jsonText(
await audio.speak(text, {
provider,
voice,
speed,
pitch,
expression,
profile,
highpass,
echo,
words,
wordGapMs
})
)
);
server.registerTool(
"start_tts_engine",
{
description: "Start (warm up) the TTS engine so the first spoken line is fast.",
inputSchema: {
engine: z
.enum(["local", "espeak", "command", "api"])
.optional()
.describe("Engine to start. Defaults to the configured one.")
}
},
async ({ engine }) => jsonText(await audio.warmTts(engine))
);
server.registerTool(
@@ -620,8 +778,24 @@ app.delete("/mcp", (_req, res) => {
);
});
app.get("/health", (_req, res) => {
res.json({ ok: true, coreConnected, socketPath: coreSocketPath, ownerId: puppet?.owner, queries: getQueryCount() });
app.get("/health", async (_req, res) => {
let selfId = puppet?.self;
if (!selfId && core) {
try {
selfId = await core.getSelfId();
} catch {
// The core may not be ready yet.
}
}
res.json({
ok: true,
coreConnected,
socketPath: coreSocketPath,
ownerId: puppet?.owner,
selfId,
queries: getQueryCount()
});
});
app.listen(PORT, HOST, (error?: Error) => {
@@ -630,9 +804,13 @@ app.listen(PORT, HOST, (error?: Error) => {
process.exit(1);
}
console.log(`[daemon] MCP listening on http://${HOST}:${PORT}/mcp`);
const stale = staleBundleWarning();
if (stale) console.warn(`[daemon] ${stale}`);
});
function shutdown() {
stopSpeakingPoll();
core?.close();
void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0));
}
+12
View File
@@ -196,10 +196,22 @@ export class CoreClient {
return this.request("voice.users", { userIds });
}
async voiceSpeaking() {
return this.request("voice.speaking", {});
}
async userStatus(userIds: string[]) {
return this.request("user.status", { userIds });
}
async userVolume(userIds: string[]) {
return this.request("user.volume", { userIds });
}
async userSetVolume(userId: string, volume: number) {
return this.request("user.setVolume", { userId, volume });
}
async voiceDevices() {
return this.request("voice.devices");
}
+32 -2
View File
@@ -9,7 +9,7 @@
*/
import { spawn, type ChildProcess } from "node:child_process";
import { existsSync } from "node:fs";
import { existsSync, statSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
@@ -17,7 +17,37 @@ const ROOT = fileURLToPath(new URL("../..", import.meta.url));
const DEFAULT_ARGS = "--no-sandbox --disable-gpu --disable-dev-shm-usage";
const XVFB_ARGS = "-screen 0 1280x720x24 -nolisten tcp";
const MAX_OUTPUT_LINES = 20;
const MAX_OUTPUT_LINES = 200;
/**
* Warn when the built client bundle predates the core sources. Without this, a
* new daemon paired with a stale client reports UNKNOWN_ACTION for new actions.
*/
export function staleBundleWarning(): string | null {
const sources = [
"src/renderer/puppet/actions.ts",
"src/renderer/puppet/events.ts",
"src/shared/puppetProtocol.ts",
"src/main/puppetBridge.ts"
];
const bundles = ["dist/js/renderer.js", "dist/js/main.js"];
try {
let newest = 0;
for (const source of sources) newest = Math.max(newest, statSync(join(ROOT, source)).mtimeMs);
let oldest = Number.POSITIVE_INFINITY;
for (const bundle of bundles) oldest = Math.min(oldest, statSync(join(ROOT, bundle)).mtimeMs);
if (oldest < newest) {
return "the built client bundle is older than the core sources; run `pnpm build` and restart the client";
}
} catch {
// A missing bundle is reported by the client start itself.
}
return null;
}
export interface CoreExit {
code: number | null;
+18
View File
@@ -100,6 +100,8 @@ const FAKE_HISTORY = Array.from({ length: 8 }, (_, index) => ({
const MOCK_VOICE_CHANNEL = "200000000000000002";
let botVoiceChannel: string | null = MOCK_VOICE_CHANNEL;
let ownerVoiceChannel: string | null = MOCK_VOICE_CHANNEL;
let outputVolume: number | null = 100;
const userVolumes: Record<string, number> = {};
const STATE_FAIL_MS = Number(process.env.VESKSLOP_MOCK_STATE_FAIL_MS ?? 0);
const startedAt = Date.now();
@@ -165,6 +167,8 @@ function handle(action: string | undefined, params: any) {
members: id === "999" ? mockVoiceMembers() : []
}))
};
case "voice.speaking":
return { channelId: botVoiceChannel, userIds: [] };
case "user.status":
return {
users: (params?.userIds ?? []).map((id: string) => ({
@@ -172,6 +176,20 @@ function handle(action: string | undefined, params: any) {
status: id === "999" ? "online" : id === "177117296140091393" ? "idle" : "offline"
}))
};
case "user.volume":
return {
outputVolume: outputVolume,
users: (params?.userIds ?? []).map((id: string) => ({
id,
volume: userVolumes[id] ?? 100
}))
};
case "user.setVolume": {
const userId = String(params?.userId);
const volume = Number(params?.volume);
userVolumes[userId] = volume;
return { userId, volume, outputVolume };
}
case "voice.setDevices":
return { inputDeviceId: params?.inputDeviceId ?? null, outputDeviceId: params?.outputDeviceId ?? null };
case "user.search": {
+145 -6
View File
@@ -9,9 +9,61 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
export interface TtsPreferences {
provider: "local" | "api" | null;
provider: "local" | "espeak" | "command" | "api" | null;
language: string | null;
voice: string | null;
/** Speech-rate multiplier, 1.0 is the engine's natural rate. Null uses the engine default. */
speed: number | null;
/** Pitch offset in semitones, 0 is the engine's natural pitch. Null uses the engine default. */
pitch: number | null;
/** Flextalk Expression profile value, 0 to 100. Null keeps the engine's registry default. */
expression: number | null;
/** Flextalk profile overrides (only the keys the user changed). */
profile: Record<string, number>;
/** Five saved profile slots; null means empty. `name` labels the slot. */
profilePresets: Array<Record<string, number | string> | null>;
/** High-pass filter Hz, 40-400. Null uses the engine default (40). */
highpass: number | null;
/** Mono echo amount, 0-100. Null or 0 disables. */
echo: number | null;
/** Synthesize each word and merge with silence gaps (Flextalk only). */
words: boolean;
/** Silence between words in milliseconds, 0-500. */
wordGapMs: number | null;
}
const PROFILE_RANGES: Record<string, [number, number]> = {
pitch: [0, 255],
rate: [0, 100],
volume: [0, 65535],
expression: [0, 100],
prominences: [0, 100],
head_size: [0, 100],
lowering: [0, 100]
};
function normalizeProfile(value: unknown): Record<string, number> {
const profile: Record<string, number> = {};
if (!value || typeof value !== "object") return profile;
for (const [key, range] of Object.entries(PROFILE_RANGES)) {
const raw = (value as Record<string, unknown>)[key];
if (isUnset(raw)) continue;
const number = Number(raw);
if (!Number.isFinite(number) || number < range[0] || number > range[1]) continue;
profile[key] = Math.round(number);
}
return profile;
}
function normalizePreset(value: unknown): Record<string, number | string> | null {
if (!value || typeof value !== "object") return null;
const preset: Record<string, number | string> = normalizeProfile(value);
const name = (value as Record<string, unknown>).name;
if (typeof name === "string" && name.trim()) preset.name = name.trim().slice(0, 40);
return preset;
}
export interface FavouritesPreferences {
@@ -19,16 +71,36 @@ export interface FavouritesPreferences {
channels: string[];
}
export interface AudioPreferences {
/** Duck the TTS while someone else speaks. */
duck: boolean;
}
export interface WebPreferences {
/** null means every guild is visible. An empty array hides all. */
visibleGuilds: string[] | null;
tts: TtsPreferences;
audio: AudioPreferences;
favourites: FavouritesPreferences;
}
const DEFAULTS: WebPreferences = {
visibleGuilds: null,
tts: { provider: null, language: null, voice: null },
tts: {
provider: null,
language: null,
voice: null,
speed: null,
pitch: null,
expression: null,
profile: {},
profilePresets: [null, null, null, null, null],
highpass: null,
echo: null,
words: false,
wordGapMs: null
},
audio: { duck: true },
favourites: { users: [], channels: [] }
};
@@ -37,6 +109,42 @@ export interface PreferencesOptions {
legacyFile?: string;
}
function normalizeProvider(value: unknown): TtsPreferences["provider"] {
return value === "api" || value === "local" || value === "espeak" || value === "command" ? value : null;
}
function isUnset(value: unknown): boolean {
return value === null || value === undefined || value === "";
}
function normalizeSpeed(value: unknown): number | null {
if (isUnset(value)) return null;
const speed = Number(value);
if (!Number.isFinite(speed) || speed < 0.5 || speed > 2) return null;
return Math.round(speed * 100) / 100;
}
function normalizePitch(value: unknown): number | null {
if (isUnset(value)) return null;
const pitch = Number(value);
if (!Number.isFinite(pitch) || pitch < -12 || pitch > 12) return null;
return Math.round(pitch * 10) / 10;
}
function normalizeExpression(value: unknown): number | null {
if (isUnset(value)) return null;
const expression = Number(value);
if (!Number.isFinite(expression) || expression < 0 || expression > 100) return null;
return Math.round(expression);
}
function normalizeRange(value: unknown, min: number, max: number): number | null {
if (isUnset(value)) return null;
const number = Number(value);
if (!Number.isFinite(number) || number < min || number > max) return null;
return Math.round(number);
}
export class PreferencesStore {
private preferences: WebPreferences = structuredClone(DEFAULTS);
@@ -68,9 +176,26 @@ export class PreferencesStore {
if (patch.tts !== undefined) {
this.preferences.tts = {
provider: patch.tts?.provider === "api" ? "api" : patch.tts?.provider === "local" ? "local" : null,
provider: normalizeProvider(patch.tts?.provider),
language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null,
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null,
speed: normalizeSpeed(patch.tts?.speed),
pitch: normalizePitch(patch.tts?.pitch),
expression: normalizeExpression(patch.tts?.expression),
profile: normalizeProfile(patch.tts?.profile),
profilePresets: Array.isArray(patch.tts?.profilePresets)
? Array.from({ length: 5 }, (_, index) => normalizePreset(patch.tts?.profilePresets?.[index]))
: this.preferences.tts.profilePresets,
highpass: normalizeRange(patch.tts?.highpass, 40, 400),
echo: normalizeRange(patch.tts?.echo, 0, 100),
words: patch.tts?.words === true,
wordGapMs: normalizeRange(patch.tts?.wordGapMs, 0, 500)
};
}
if (patch.audio !== undefined) {
this.preferences.audio = {
duck: patch.audio?.duck !== false
};
}
@@ -112,9 +237,23 @@ export class PreferencesStore {
? [...new Set<string>(raw.visibleGuilds.map((id: unknown) => String(id)))]
: null,
tts: {
provider: provider === "api" || provider === "local" ? provider : null,
provider: normalizeProvider(provider),
language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null,
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null,
speed: normalizeSpeed(raw?.tts?.speed),
pitch: normalizePitch(raw?.tts?.pitch),
expression: normalizeExpression(raw?.tts?.expression),
profile: normalizeProfile(raw?.tts?.profile),
profilePresets: Array.isArray(raw?.tts?.profilePresets)
? Array.from({ length: 5 }, (_, index) => normalizePreset(raw.tts.profilePresets[index]))
: [null, null, null, null, null],
highpass: normalizeRange(raw?.tts?.highpass, 40, 400),
echo: normalizeRange(raw?.tts?.echo, 0, 100),
words: raw?.tts?.words === true,
wordGapMs: normalizeRange(raw?.tts?.wordGapMs, 0, 500)
},
audio: {
duck: raw?.audio?.duck !== false
},
favourites: {
users: Array.isArray(raw?.favourites?.users)
+5
View File
@@ -157,6 +157,11 @@ export class Puppet {
private selfId?: string;
private queue: Promise<void> = Promise.resolve();
/** The logged-in account's user ID, once the core has been asked for it. */
get self(): string | undefined {
return this.selfId;
}
constructor(
private readonly core: PuppetTransport,
private readonly audio?: PuppetAudio
+229 -15
View File
@@ -7,24 +7,51 @@
*/
import { spawn } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { mkdirSync, unlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
export class TtsError extends Error {}
export type TtsProvider = "local" | "api";
export type TtsProvider = "local" | "espeak" | "command" | "api";
export interface TtsOptions {
provider?: TtsProvider;
voice?: string;
/** Speech-rate multiplier; the engine maps it to its own units. */
speed?: number;
/** Pitch offset in semitones; the engine maps it to its own units. */
pitch?: number;
/** Flextalk Expression profile value, 0 to 100. */
expression?: number;
/** Flextalk profile overrides: pitch, rate, volume, expression, prominences, head_size, lowering. */
profile?: Record<string, number>;
/** High-pass filter Hz, 40 to 400. */
highpass?: number;
/** Mono echo amount, 0 to 100. */
echo?: number;
/** Synthesize each word and merge with silence gaps. */
words?: boolean;
/** Silence between words in milliseconds, 0 to 500. */
wordGapMs?: number;
}
export interface TtsEngine {
id: "espeak" | "command" | "api";
name: string;
available: boolean;
/** false when the engine has no language dimension (single-language command engines). */
language: boolean;
}
export interface TtsStatus {
local: boolean;
api: boolean;
espeak: boolean;
command: boolean;
defaultProvider: TtsProvider;
engine: string | null;
engines: TtsEngine[];
}
export interface TtsVoice {
@@ -32,8 +59,19 @@ export interface TtsVoice {
name: string;
language: string;
gender: "male" | "female" | null;
/** false when the engine ignores pitch for this voice (for example Flextalk). */
supportsPitch?: boolean;
/** true when the voice has a Flextalk-style Expression profile parameter. */
supportsExpression?: boolean;
/** Registry default for the expression parameter, when known. */
expressionDefault?: number;
/** Per-voice Flextalk profile defaults (pitch, rate, prominences, head_size, lowering, expression). */
profileDefaults?: Record<string, number>;
}
/** espeak-ng voice variants, appended to every base voice. */
const ESPEAK_VARIANTS = ["m1", "m2", "m3", "m4", "m5", "m6", "m7", "f1", "f2", "f3", "f4", "f5", "croak", "whisper"];
interface CommandResult {
code: number | null;
stdout: string;
@@ -81,11 +119,62 @@ function parseVoices(output: string): TtsVoice[] {
return voices;
}
/**
* Command-mode voice list. Entries are separated by commas or newlines. Each entry is
* `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or
* `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch`
* (the engine ignores pitch) and `expression` / `expression=NN` (Flextalk profile).
*/
function parseEngineVoices(raw: string): TtsVoice[] {
return raw
.split(/[,\n]+/)
.map(part => part.trim())
.filter(Boolean)
.map((entry): TtsVoice => {
const [id, name, gender, language, flags] = entry.split("|").map(part => part?.trim());
const flagList = flags?.split("+").map(flag => flag.trim()) ?? [];
const expression = flagList.find(flag => flag === "expression" || flag.startsWith("expression="));
const defaults = flagList.find(flag => flag.startsWith("defaults="));
let profileDefaults: Record<string, number> | undefined;
if (defaults) {
const parts = defaults.slice("defaults=".length).split(":");
profileDefaults = {};
for (let index = 0; index + 1 < parts.length; index += 2) {
const value = Number(parts[index + 1]);
if (Number.isFinite(value)) profileDefaults[parts[index]] = value;
}
}
const expressionDefault = profileDefaults?.expression ??
(expression?.includes("=") ? Number(expression.split("=")[1]) : undefined);
return {
id,
name: name || id,
language: language || "en",
gender: gender === "male" || gender === "female" ? gender : null,
supportsPitch: !flagList.includes("no-pitch"),
supportsExpression: !!expression || !!profileDefaults,
...(Number.isFinite(expressionDefault) ? { expressionDefault } : {}),
...(profileDefaults ? { profileDefaults } : {})
};
});
}
/** Serialize a profile object to the wrapper's `key=value,key=value` form. */
function serializeProfile(profile?: Record<string, number>): string {
if (!profile) return "";
return Object.entries(profile)
.map(([key, value]) => `${key}=${value}`)
.join(",");
}
export class Tts {
private readonly dir: string;
private counter = 0;
private espeak = false;
private voiceCache?: TtsVoice[];
private readonly voiceCache = new Map<string, TtsVoice[]>();
constructor(cacheDir?: string) {
const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache");
@@ -109,39 +198,128 @@ export class Tts {
return this.dir;
}
/** Display name for the configured command engine. */
get commandName(): string {
return process.env.VESKSLOP_TTS_NAME ?? "custom engine";
}
async detect(): Promise<void> {
const result = await run("espeak-ng", ["--version"]);
this.espeak = result.code === 0;
}
engines(): TtsEngine[] {
return [
{ id: "espeak", name: "espeak-ng", available: this.espeak, language: true },
{ id: "command", name: this.commandName, available: !!this.localCommand, language: false },
{ id: "api", name: "HTTP API", available: !!this.apiUrl, language: true }
];
}
private autoProvider(): TtsProvider {
if (this.localCommand) return "command";
if (this.espeak) return "espeak";
return this.apiUrl ? "api" : "local";
}
status(): TtsStatus {
const local = !!this.localCommand || this.espeak;
const override = process.env.VESKSLOP_TTS_PROVIDER;
const allowed = new Set<TtsProvider>(["local", "espeak", "command", "api"]);
return {
local,
api: !!this.apiUrl,
defaultProvider: override === "local" || override === "api" ? override : local ? "local" : "api",
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null
espeak: this.espeak,
command: !!this.localCommand,
defaultProvider: allowed.has(override as TtsProvider) ? (override as TtsProvider) : this.autoProvider(),
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null,
engines: this.engines()
};
}
async voices(): Promise<TtsVoice[]> {
if (this.voiceCache) return this.voiceCache;
if (!this.espeak) return (this.voiceCache = []);
async voices(engine?: TtsProvider): Promise<TtsVoice[]> {
const key = engine ?? this.autoProvider();
const cached = this.voiceCache.get(key);
if (cached) return cached;
let voices: TtsVoice[] = [];
if (key === "command" || (key === "local" && this.localCommand)) {
const declared = process.env.VESKSLOP_TTS_VOICES;
voices = declared ? parseEngineVoices(declared) : [];
} else if (key === "espeak" || key === "local") {
voices = await this.espeakVoices();
}
this.voiceCache.set(key, voices);
return voices;
}
private async espeakVoices(): Promise<TtsVoice[]> {
if (!this.espeak) return [];
const result = await run("espeak-ng", ["--voices"]);
this.voiceCache = result.code === 0 ? parseVoices(result.stdout) : [];
return this.voiceCache;
const base = result.code === 0 ? parseVoices(result.stdout) : [];
return base.flatMap((voice): TtsVoice[] => [
voice,
...ESPEAK_VARIANTS.map(
(variant): TtsVoice => ({
id: `${voice.id}+${variant}`,
name: `${voice.name} (${variant})`,
language: voice.language,
gender: variant.startsWith("m") ? "male" : variant.startsWith("f") ? "female" : null
})
)
]);
}
/**
* Start the engine so the first real line is fast: run one throwaway synthesis
* (the command wrapper starts Xvfb, Wine, and the host), then delete the file.
*/
async warm(engine?: TtsProvider): Promise<{ engine: TtsProvider; ms: number; warmed: boolean }> {
const key = engine ?? this.autoProvider();
const start = Date.now();
if (key === "command" && !this.localCommand) {
throw new TtsError("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)");
}
const usesCommand = key === "command" || (key === "local" && !!this.localCommand);
if (!usesCommand) {
// espeak starts instantly and the HTTP provider is expected to stay warm.
return { engine: key, ms: 0, warmed: false };
}
const voice = process.env.VESKSLOP_TTS_VOICE || undefined;
const file = await this.viaLocalCommand("The quick brown fox jumps over the lazy dog.", {
provider: key,
voice
});
try {
unlinkSync(file);
} catch {
// best effort
}
return { engine: key, ms: Date.now() - start, warmed: true };
}
async synthesize(text: string, options: TtsOptions = {}): Promise<string> {
const provider = options.provider ?? this.status().defaultProvider;
return provider === "api" ? this.viaApi(text, options) : this.viaLocalCommand(text, options);
if (provider === "api") return this.viaApi(text, options);
if (provider === "espeak") return this.viaLocalCommand(text, options, true);
if (provider === "command") return this.viaLocalCommand(text, options, false);
return this.viaLocalCommand(text, options);
}
private async viaLocalCommand(text: string, options: TtsOptions): Promise<string> {
const command = this.localCommand;
private async viaLocalCommand(text: string, options: TtsOptions, forceEspeak = false): Promise<string> {
const command = forceEspeak ? undefined : this.localCommand;
if (!command && !this.espeak) {
throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)");
}
@@ -162,6 +340,14 @@ export class Tts {
.replace("{out}", out)
.replace("{text_file}", textFile)
.replace("{voice}", options.voice ?? "")
.replace("{speed}", options.speed !== undefined ? String(options.speed) : "")
.replace("{pitch}", options.pitch !== undefined ? String(options.pitch) : "")
.replace("{expression}", options.expression !== undefined ? String(options.expression) : "")
.replace("{profile}", serializeProfile(options.profile))
.replace("{highpass}", options.highpass !== undefined ? String(options.highpass) : "")
.replace("{echo}", options.echo !== undefined ? String(options.echo) : "")
.replace("{words}", options.words ? "1" : "")
.replace("{word_gap_ms}", options.wordGapMs !== undefined ? String(options.wordGapMs) : "")
)
: ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile];
@@ -169,7 +355,24 @@ export class Tts {
await new Promise<void>((resolve, reject) => {
const child = spawn(args[0], args.slice(1), {
env: { ...process.env, VESKSLOP_TTS_VOICE: options.voice ?? "" },
env: {
...process.env,
VESKSLOP_TTS_VOICE: options.voice ?? "",
...(options.speed !== undefined ? { VESKSLOP_TTS_SPEED: String(options.speed) } : {}),
...(options.pitch !== undefined ? { VESKSLOP_TTS_PITCH: String(options.pitch) } : {}),
...(options.expression !== undefined
? { VESKSLOP_TTS_EXPRESSION: String(options.expression) }
: {}),
...(options.profile && Object.keys(options.profile).length
? { VESKSLOP_TTS_PROFILE: serializeProfile(options.profile) }
: {}),
...(options.highpass !== undefined ? { VESKSLOP_TTS_HIGHPASS: String(options.highpass) } : {}),
...(options.echo !== undefined ? { VESKSLOP_TTS_ECHO: String(options.echo) } : {}),
...(options.words ? { VESKSLOP_TTS_WORDS: "1" } : {}),
...(options.wordGapMs !== undefined
? { VESKSLOP_TTS_WORD_GAP_MS: String(options.wordGapMs) }
: {})
},
stdio: ["ignore", "ignore", "pipe"]
});
let stderr = "";
@@ -197,7 +400,18 @@ export class Tts {
? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` }
: {})
},
body: JSON.stringify({ text, ...(options.voice ? { voice: options.voice } : {}) })
body: JSON.stringify({
text,
...(options.voice ? { voice: options.voice } : {}),
...(options.speed !== undefined ? { speed: options.speed } : {}),
...(options.pitch !== undefined ? { pitch: options.pitch } : {}),
...(options.expression !== undefined ? { expression: options.expression } : {}),
...(options.profile && Object.keys(options.profile).length ? { profile: options.profile } : {}),
...(options.highpass !== undefined ? { highpass: options.highpass } : {}),
...(options.echo !== undefined ? { echo: options.echo } : {}),
...(options.words ? { words: true } : {}),
...(options.wordGapMs !== undefined ? { wordGapMs: options.wordGapMs } : {})
})
});
if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`);
+103 -8
View File
@@ -25,7 +25,7 @@ import type { PreferencesStore, WebPreferences } from "./preferences.js";
import type { Puppet } from "./puppet.js";
import { ensureTls } from "./tls.js";
import type { TranscriptStore } from "./transcript.js";
import type { TtsVoice } from "./tts.js";
import type { TtsProvider, TtsVoice } from "./tts.js";
const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"];
const LANGUAGE_NAMES: Record<string, string> = {
@@ -401,7 +401,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
app.post(
"/api/preferences",
wrap(async (req, res) => {
const { visibleGuilds, tts, favourites } = req.body ?? {};
const { visibleGuilds, tts, audio, favourites } = req.body ?? {};
if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
@@ -414,7 +414,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
patch.tts = {
provider: tts?.provider ?? null,
language: tts?.language ?? null,
voice: tts?.voice ?? null
voice: tts?.voice ?? null,
speed: tts?.speed ?? null,
pitch: tts?.pitch ?? null,
expression: tts?.expression ?? null,
profile: tts?.profile ?? {},
profilePresets: Array.isArray(tts?.profilePresets)
? tts.profilePresets.slice(0, 5)
: deps.preferences.get().tts.profilePresets,
highpass: tts?.highpass ?? null,
echo: tts?.echo ?? null,
words: tts?.words === true,
wordGapMs: tts?.wordGapMs ?? null
};
}
@@ -425,6 +436,11 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
};
}
if (audio !== undefined) {
patch.audio = { duck: audio?.duck !== false };
deps.audio.setDuckEnabled(patch.audio.duck);
}
res.json({ ok: true, result: deps.preferences.update(patch) });
})
);
@@ -486,6 +502,41 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
})
);
app.get(
"/api/voice/volume",
wrap(async (req, res) => {
let userIds = req.query.userIds
? String(req.query.userIds)
.split(",")
.map(id => id.trim())
.filter(Boolean)
: [];
if (!userIds.length) {
const voice = (await requireCore().voiceGet()) as { members?: Array<{ id: string }> } | undefined;
userIds = (voice?.members ?? []).map(member => member.id);
}
if (!userIds.length) {
res.json({ ok: true, result: { outputVolume: null, users: [] } });
return;
}
res.json({ ok: true, result: await requireCore().userVolume(userIds) });
})
);
app.post(
"/api/voice/volume",
wrap(async (req, res) => {
const userId = String(req.body?.userId ?? "");
const volume = Number(req.body?.volume);
const result = await pacedVoice(() => requireCore().userSetVolume(userId, volume));
deps.getPuppet()?.appendAction({ type: "voice", action: "set-volume", userId, volume });
res.json({ ok: true, result });
})
);
app.post(
"/api/voice/join",
wrap(async (req, res) => {
@@ -526,8 +577,28 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
app.get(
"/api/audio/tts/voices",
wrap(async (_req, res) => {
res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) });
wrap(async (req, res) => {
const engine = req.query.engine ? String(req.query.engine) : undefined;
res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices(engine as TtsProvider | undefined)) });
})
);
app.post(
"/api/audio/tts/warm",
wrap(async (req, res) => {
const requested = req.body?.engine;
const engine = ["local", "espeak", "command", "api"].includes(requested)
? (requested as TtsProvider)
: undefined;
const result = await deps.audio.warmTts(engine);
deps.getPuppet()?.appendAction({
type: "audio",
action: "warm-engine",
engine: result.engine,
ms: result.ms
});
res.json({ ok: true, result });
})
);
@@ -568,11 +639,35 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
const text = String(req.body?.text ?? "");
const prefs = deps.preferences.get();
const provider =
req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider;
const requested = req.body?.provider;
const provider = ["api", "local", "espeak", "command"].includes(requested)
? requested
: prefs.tts.provider;
const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
const speed = req.body?.speed !== undefined ? Number(req.body.speed) : prefs.tts.speed ?? undefined;
const pitch = req.body?.pitch !== undefined ? Number(req.body.pitch) : prefs.tts.pitch ?? undefined;
const expression =
req.body?.expression !== undefined ? Number(req.body.expression) : prefs.tts.expression ?? undefined;
const profile = req.body?.profile && typeof req.body.profile === "object" ? req.body.profile : prefs.tts.profile;
const highpass =
req.body?.highpass !== undefined ? Number(req.body.highpass) : prefs.tts.highpass ?? undefined;
const echo = req.body?.echo !== undefined ? Number(req.body.echo) : prefs.tts.echo ?? undefined;
const words = req.body?.words !== undefined ? req.body.words === true : prefs.tts.words;
const wordGapMs =
req.body?.wordGapMs !== undefined ? Number(req.body.wordGapMs) : prefs.tts.wordGapMs ?? undefined;
const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice });
const result = await deps.audio.speak(text, {
provider: provider ?? undefined,
voice,
...(Number.isFinite(speed) ? { speed } : {}),
...(Number.isFinite(pitch) ? { pitch } : {}),
...(Number.isFinite(expression) ? { expression } : {}),
...(profile && Object.keys(profile).length ? { profile } : {}),
...(Number.isFinite(highpass) ? { highpass } : {}),
...(Number.isFinite(echo) ? { echo } : {}),
...(words ? { words: true } : {}),
...(Number.isFinite(wordGapMs) ? { wordGapMs } : {})
});
deps.getPuppet()?.appendAction({
type: "audio",
action: "speak",