Add MicroslopSam TTS, Flextalk voice profiles, effects, and engine controls
This commit is contained in:
+2
-1
@@ -15,7 +15,8 @@
|
||||
"test:trigger": "tsx src/test-trigger.ts",
|
||||
"cdp:dispatch": "tsx src/cdp-dispatch.ts",
|
||||
"test:cache": "tsx src/test-cache.ts",
|
||||
"test:audio": "tsx src/test-audio.ts"
|
||||
"test:audio": "tsx src/test-audio.ts",
|
||||
"audio:doctor": "bash scripts/audio-doctor.sh"
|
||||
},
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.30.0",
|
||||
|
||||
+636
-61
@@ -26,8 +26,11 @@ const state = {
|
||||
visibleGuildsSaved: null,
|
||||
timeZoneOverride: localStorage.getItem("veskslop-timezone") ?? "",
|
||||
speech: { provider: null, language: null, voice: null },
|
||||
profileDefaults: {},
|
||||
expressionDefault: null,
|
||||
scope: "guild",
|
||||
speaking: new Set(),
|
||||
speakingFade: new Map(),
|
||||
people: new Map(),
|
||||
favourites: { users: [], channels: [] },
|
||||
favouriteUsers: [],
|
||||
@@ -661,16 +664,19 @@ function renderVisibleGuilds() {
|
||||
|
||||
function voiceMarks(member) {
|
||||
const marks = el("span", { class: "vk-marks" });
|
||||
const speaking = state.speaking.has(member.id);
|
||||
const muted = member.suppress || member.mute || member.selfMute;
|
||||
|
||||
// Fixed order: audio → video → share.
|
||||
if (speaking) {
|
||||
const levels = el("span", { class: "vk-levels", title: "speaking" });
|
||||
for (let index = 0; index < 4; index++) levels.append(el("span"));
|
||||
marks.append(levels);
|
||||
}
|
||||
// The bars are always in the layout; they scale up while speaking and
|
||||
// decay back to nothing when the voice stops.
|
||||
const classes = ["vk-levels"];
|
||||
if (!muted && state.speaking.has(member.id)) classes.push("speaking");
|
||||
if (!muted && state.speakingFade.has(member.id)) classes.push("stopping");
|
||||
|
||||
if (member.suppress || member.mute || member.selfMute) {
|
||||
const levels = el("span", { class: classes.join(" "), title: "speaking" });
|
||||
for (let index = 0; index < 4; index++) levels.append(el("span"));
|
||||
marks.append(levels);
|
||||
|
||||
if (muted) {
|
||||
const title = member.suppress ? "muted by server" : member.mute ? "server-muted" : "self-muted";
|
||||
marks.append(el("span", { class: "neutral", title, html: VK.icon("micSlash") }));
|
||||
}
|
||||
@@ -702,6 +708,53 @@ function memberNameNode(member, className = "vk-name") {
|
||||
});
|
||||
}
|
||||
|
||||
/* Speaking marks. Bars stay on until an explicit stop, a mute, or a channel
|
||||
change: Discord's speaking events are not reliable enough for an idle decay.
|
||||
The fade runs the equalizer animation inside a shrinking container, so the
|
||||
amplitude drops instead of the whole block collapsing at full height. */
|
||||
|
||||
const SPEAKING_FADE_MS = 750;
|
||||
|
||||
function renderSpeakingViews() {
|
||||
renderVoiceRail();
|
||||
renderPeople();
|
||||
renderRightVoice();
|
||||
renderRightFavourites();
|
||||
renderCallStrip();
|
||||
}
|
||||
|
||||
function scheduleSpeakingRender() {
|
||||
const now = Date.now();
|
||||
if (now - state.speakingRenderAt < 250) return;
|
||||
state.speakingRenderAt = now;
|
||||
renderSpeakingViews();
|
||||
}
|
||||
|
||||
function beginSpeaking(userId) {
|
||||
const fade = state.speakingFade.get(userId);
|
||||
if (fade) {
|
||||
clearTimeout(fade);
|
||||
state.speakingFade.delete(userId);
|
||||
}
|
||||
|
||||
const wasSpeaking = state.speaking.has(userId);
|
||||
state.speaking.add(userId);
|
||||
|
||||
if (!wasSpeaking) scheduleSpeakingRender();
|
||||
}
|
||||
|
||||
function endSpeaking(userId) {
|
||||
if (!state.speaking.delete(userId)) return;
|
||||
|
||||
const fadeTimer = setTimeout(() => {
|
||||
state.speakingFade.delete(userId);
|
||||
renderSpeakingViews();
|
||||
}, SPEAKING_FADE_MS);
|
||||
state.speakingFade.set(userId, fadeTimer);
|
||||
|
||||
scheduleSpeakingRender();
|
||||
}
|
||||
|
||||
function trackPeople(members) {
|
||||
for (const member of members ?? []) {
|
||||
if (!member?.id || member.id === state.user?.id) continue;
|
||||
@@ -1208,6 +1261,7 @@ function toggleTts() {
|
||||
const panel = $("#tts-panel");
|
||||
panel.hidden = !panel.hidden;
|
||||
$("#tts-target").textContent = `as ${state.user?.username ?? "account"} → ${channelLabel()}`;
|
||||
if (!panel.hidden) void refreshSpeech();
|
||||
}
|
||||
|
||||
/* Right column */
|
||||
@@ -1664,10 +1718,10 @@ function fillDeviceSelect(select, devices, defaultLabel) {
|
||||
async function refreshAudio() {
|
||||
try {
|
||||
const status = await VK.api.get("/api/audio");
|
||||
state.audioRecording = !!status.recording;
|
||||
state.audioRecording = !!status.capturing;
|
||||
|
||||
$("#audio-status").textContent = status.available
|
||||
? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.recording ? " · recording" : ""}`
|
||||
? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.capturing ? " · capturing" : ""}`
|
||||
: (status.error ?? "unavailable");
|
||||
|
||||
const hintRow = $("#audio-hint-row");
|
||||
@@ -1678,7 +1732,15 @@ async function refreshAudio() {
|
||||
$("#audio-tts").textContent = `local ${tts.local ? "yes" : "no"} · api ${
|
||||
tts.api ? "yes" : "no"
|
||||
} · player ${status.player ? "yes" : "no"} · queue ${status.queue ?? 0}`;
|
||||
$("#audio-capture").textContent = status.recording ? "Stop capture" : "Start capture";
|
||||
$("#audio-capture").textContent = status.capturing ? "Stop capture" : "Start capture";
|
||||
|
||||
const duck = $("#tts-duck");
|
||||
if (duck) {
|
||||
const on = status.duckEnabled !== false;
|
||||
duck.classList.toggle("vk-chip-on", on);
|
||||
duck.classList.toggle("vk-chip-off", !on);
|
||||
duck.textContent = on ? "duck on voice" : "duck off";
|
||||
}
|
||||
} catch (error) {
|
||||
$("#audio-status").textContent = error.message;
|
||||
}
|
||||
@@ -1690,11 +1752,17 @@ async function refreshAudio() {
|
||||
/* Speech settings (server-side preferences). */
|
||||
|
||||
async function refreshSpeech() {
|
||||
let engines = [];
|
||||
let defaultProvider = "local";
|
||||
|
||||
try {
|
||||
const status = await VK.api.get("/api/audio");
|
||||
const tts = status.tts ?? {};
|
||||
const stt = status.stt ?? {};
|
||||
|
||||
engines = tts.engines ?? [];
|
||||
defaultProvider = tts.defaultProvider ?? "local";
|
||||
|
||||
$("#speech-status").textContent = `${tts.engine ?? "no local engine"}${tts.api ? " + api" : ""}`;
|
||||
$("#speech-stt").textContent = stt.local
|
||||
? `local ready${stt.engine ? ` (${stt.engine})` : ""}`
|
||||
@@ -1705,36 +1773,77 @@ async function refreshSpeech() {
|
||||
// the daemon status is optional here
|
||||
}
|
||||
|
||||
let groups = [];
|
||||
try {
|
||||
groups = await VK.api.get("/api/audio/tts/voices");
|
||||
} catch {
|
||||
// voices need a local engine
|
||||
}
|
||||
|
||||
try {
|
||||
const prefs = await VK.api.get("/api/preferences");
|
||||
state.speech = prefs.tts ?? { provider: null, language: null, voice: null };
|
||||
|
||||
const languageSelect = $("#speech-language");
|
||||
languageSelect.replaceChildren(el("option", { value: "", text: "Auto" }));
|
||||
for (const group of groups) {
|
||||
languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` }));
|
||||
}
|
||||
languageSelect.value = state.speech.language ?? "";
|
||||
|
||||
fillVoiceOptions(groups, state.speech.language, state.speech.voice);
|
||||
|
||||
$("#speech-provider").value = state.speech.provider ?? "";
|
||||
$("#tts-provider").value = state.speech.provider ?? "";
|
||||
state.speech = prefs.tts ?? { provider: null, language: null, voice: null, speed: null, pitch: null };
|
||||
} catch {
|
||||
// preferences are optional
|
||||
}
|
||||
|
||||
const selected = resolveEngine(state.speech.provider, engines, defaultProvider);
|
||||
const selectedEngine = engines.find(engine => engine.id === selected);
|
||||
const hasLanguage = selectedEngine?.language !== false;
|
||||
|
||||
fillEngineOptions($("#speech-provider"), engines, state.speech.provider ?? selected);
|
||||
fillEngineOptions($("#tts-provider"), engines, state.speech.provider ?? selected);
|
||||
|
||||
let groups = [];
|
||||
try {
|
||||
groups = await VK.api.get(`/api/audio/tts/voices?engine=${encodeURIComponent(selected)}`);
|
||||
} catch {
|
||||
// voices need an engine
|
||||
}
|
||||
|
||||
const languageSelect = $("#speech-language");
|
||||
languageSelect.replaceChildren(el("option", { value: "", text: "Auto" }));
|
||||
for (const group of groups) {
|
||||
languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` }));
|
||||
}
|
||||
languageSelect.value = state.speech.language ?? "";
|
||||
$("#speech-language-row").hidden = !hasLanguage;
|
||||
|
||||
const voices = voiceList(groups, hasLanguage ? state.speech.language : null, hasLanguage);
|
||||
fillVoiceSelect($("#speech-voice"), voices, state.speech.voice);
|
||||
fillVoiceSelect($("#tts-voice"), voices, state.speech.voice);
|
||||
syncVoiceControls(voices, state.speech.voice);
|
||||
fillSpeechSliders(state.speech);
|
||||
}
|
||||
|
||||
function fillVoiceOptions(groups, language, selected) {
|
||||
const select = $("#speech-voice");
|
||||
const voices = groups.find(group => group.id === language)?.voices ?? [];
|
||||
/** Resolve the engine that will actually run: provider override first, then the auto default. */
|
||||
function resolveEngine(provider, engines, defaultProvider) {
|
||||
if (provider === "espeak" || provider === "command" || provider === "api") return provider;
|
||||
if (defaultProvider === "espeak" || defaultProvider === "command" || defaultProvider === "api") return defaultProvider;
|
||||
|
||||
const available = id => engines.some(engine => engine.id === id && engine.available);
|
||||
if (available("command")) return "command";
|
||||
if (available("espeak")) return "espeak";
|
||||
return "api";
|
||||
}
|
||||
|
||||
function fillEngineOptions(select, engines, value) {
|
||||
if (!select) return;
|
||||
|
||||
select.replaceChildren(el("option", { value: "", text: "Auto" }));
|
||||
for (const engine of engines) {
|
||||
const option = el("option", {
|
||||
value: engine.id,
|
||||
text: engine.available ? engine.name : `${engine.name} (unavailable)`
|
||||
});
|
||||
option.disabled = !engine.available;
|
||||
select.append(option);
|
||||
}
|
||||
|
||||
select.value = value;
|
||||
}
|
||||
|
||||
function voiceList(groups, language, hasLanguage) {
|
||||
if (!hasLanguage) return (groups ?? []).flatMap(group => group.voices ?? []);
|
||||
if (!language) return [];
|
||||
return groups?.find(group => group.id === language)?.voices ?? [];
|
||||
}
|
||||
|
||||
function fillVoiceSelect(select, voices, selected) {
|
||||
if (!select) return;
|
||||
|
||||
select.replaceChildren(el("option", { value: "", text: "Default" }));
|
||||
for (const voice of voices) {
|
||||
@@ -1745,15 +1854,38 @@ function fillVoiceOptions(groups, language, selected) {
|
||||
select.value = selected && voices.some(voice => voice.id === selected) ? selected : "";
|
||||
}
|
||||
|
||||
async function saveSpeech() {
|
||||
function fillVoiceOptions(groups, language, selected) {
|
||||
fillVoiceSelect($("#speech-voice"), voiceList(groups, language, !!language), selected);
|
||||
}
|
||||
|
||||
async function saveSpeech(extra = {}) {
|
||||
const provider = $("#speech-provider").value || null;
|
||||
const language = $("#speech-language").value || null;
|
||||
const language = $("#speech-language-row").hidden
|
||||
? (state.speech.language ?? null)
|
||||
: $("#speech-language").value || null;
|
||||
const voice = $("#speech-voice").value || null;
|
||||
const speed = Number($("#tts-speed").value);
|
||||
const pitch = Number($("#tts-pitch").value);
|
||||
const expressionEl = $("#tts-expression");
|
||||
const expression =
|
||||
expressionEl.dataset.dirty === "1" || typeof state.speech.expression === "number"
|
||||
? Number(expressionEl.value)
|
||||
: null;
|
||||
|
||||
try {
|
||||
const prefs = await VK.api.post("/api/preferences", { tts: { provider, language, voice } });
|
||||
state.speech = prefs.tts ?? { provider: null, language: null, voice: null };
|
||||
const prefs = await VK.api.post("/api/preferences", {
|
||||
tts: { provider, language, voice, speed, pitch, expression, ...effectOptions(), ...extra }
|
||||
});
|
||||
state.speech = prefs.tts ?? {
|
||||
provider: null,
|
||||
language: null,
|
||||
voice: null,
|
||||
speed: null,
|
||||
pitch: null,
|
||||
expression: null
|
||||
};
|
||||
$("#tts-provider").value = state.speech.provider ?? "";
|
||||
fillSpeechSliders(state.speech);
|
||||
await refreshSpeech();
|
||||
toast("Speech settings saved", "ok");
|
||||
} catch (error) {
|
||||
@@ -1761,6 +1893,205 @@ async function saveSpeech() {
|
||||
}
|
||||
}
|
||||
|
||||
async function warmEngine() {
|
||||
const button = $("#speech-warm");
|
||||
const engine = $("#speech-provider").value || null;
|
||||
const label = $("#speech-provider").selectedOptions[0]?.textContent ?? "engine";
|
||||
|
||||
button.disabled = true;
|
||||
button.textContent = "Starting…";
|
||||
|
||||
try {
|
||||
const result = await VK.api.post("/api/audio/tts/warm", engine ? { engine } : {});
|
||||
toast(result.warmed ? `${label} ready in ${result.ms} ms` : `${label} needs no warm-up`, "ok");
|
||||
$("#speech-status").textContent = result.warmed ? `${label} ready (${result.ms} ms)` : label;
|
||||
} catch (error) {
|
||||
toast(error.message, "error");
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
button.textContent = "Start engine";
|
||||
}
|
||||
}
|
||||
|
||||
function fillSpeechSliders(tts) {
|
||||
const speed = typeof tts?.speed === "number" ? tts.speed : 1;
|
||||
const pitch = typeof tts?.pitch === "number" ? tts.pitch : 0;
|
||||
|
||||
$("#tts-speed").value = Number.isFinite(speed) ? speed : 1;
|
||||
$("#tts-pitch").value = Number.isFinite(pitch) ? pitch : 0;
|
||||
syncSpeechSliderLabels();
|
||||
}
|
||||
|
||||
function syncSpeechSliderLabels() {
|
||||
const speed = Number($("#tts-speed").value);
|
||||
const pitch = Number($("#tts-pitch").value);
|
||||
const pitchSlider = $("#tts-pitch");
|
||||
|
||||
$("#tts-speed-value").textContent = `${speed.toFixed(2)}x`;
|
||||
$("#tts-pitch-value").textContent = pitchSlider.disabled ? "n/a" : `${pitch > 0 ? "+" : ""}${pitch} st`;
|
||||
|
||||
const expressionSlider = $("#tts-expression");
|
||||
if (expressionSlider) {
|
||||
const explicit = expressionSlider.dataset.dirty === "1" || expressionSlider.dataset.stored === "1";
|
||||
$("#tts-expression-value").textContent = explicit ? expressionSlider.value : "default";
|
||||
}
|
||||
|
||||
for (const button of $$("#tts-speed-presets .vk-chip")) {
|
||||
const active = Math.abs(Number(button.dataset.speed) - speed) < 0.001;
|
||||
button.classList.toggle("vk-chip-on", active);
|
||||
button.classList.toggle("vk-chip-off", !active);
|
||||
}
|
||||
}
|
||||
|
||||
/* Flextalk profile controls (VOX/HEV kit): profile values, effects, word mode. */
|
||||
|
||||
const PROFILE_SPEC = [
|
||||
["pitch", 0, 120],
|
||||
["rate", 0, 100],
|
||||
["prominences", 0, 100],
|
||||
["head_size", 0, 100],
|
||||
["lowering", 0, 100]
|
||||
];
|
||||
|
||||
function buildProfileSliders() {
|
||||
const root = $("#tts-profile-sliders");
|
||||
if (!root) return;
|
||||
|
||||
for (const [key, min, max] of PROFILE_SPEC) {
|
||||
const slider = el("input", { type: "range", min: String(min), max: String(max), step: "1" });
|
||||
slider.id = `tts-profile-${key}`;
|
||||
slider.dataset.key = key;
|
||||
const value = el("span", { class: "vk-mono vk-dim", text: "default" });
|
||||
|
||||
root.append(
|
||||
el(
|
||||
"div",
|
||||
{ class: "vk-tts-profile-row" },
|
||||
el("span", { class: "vk-micro-label", text: key.replace("_", " ") }),
|
||||
slider,
|
||||
value
|
||||
)
|
||||
);
|
||||
|
||||
slider.addEventListener("input", () => {
|
||||
const engineDefault = state.profileDefaults?.[key];
|
||||
const snapped = typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2;
|
||||
|
||||
if (snapped) {
|
||||
slider.value = engineDefault;
|
||||
slider.dataset.dirty = "";
|
||||
value.textContent = "default";
|
||||
} else {
|
||||
slider.dataset.dirty = "1";
|
||||
value.textContent = slider.value;
|
||||
}
|
||||
});
|
||||
slider.addEventListener("change", () => void saveSpeech());
|
||||
}
|
||||
}
|
||||
|
||||
function profileFromControls() {
|
||||
const profile = {};
|
||||
for (const [key] of PROFILE_SPEC) {
|
||||
const slider = $(`#tts-profile-${key}`);
|
||||
if (slider?.dataset.dirty === "1") profile[key] = Number(slider.value);
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
||||
function fillProfileControls() {
|
||||
const stored = state.speech.profile ?? {};
|
||||
fillProfileSlots();
|
||||
for (const [key] of PROFILE_SPEC) {
|
||||
const slider = $(`#tts-profile-${key}`);
|
||||
if (!slider) continue;
|
||||
|
||||
if (typeof stored[key] === "number") {
|
||||
slider.value = stored[key];
|
||||
slider.dataset.dirty = "1";
|
||||
} else {
|
||||
const engineDefault = state.profileDefaults?.[key];
|
||||
if (typeof engineDefault === "number") slider.value = engineDefault;
|
||||
slider.dataset.dirty = "";
|
||||
}
|
||||
const value = slider.parentElement?.querySelector(".vk-mono");
|
||||
if (value) value.textContent = slider.dataset.dirty === "1" ? slider.value : "default";
|
||||
}
|
||||
|
||||
const highpass = Number(state.speech.highpass);
|
||||
$("#tts-highpass").value = Number.isFinite(highpass) && state.speech.highpass !== null ? highpass : 40;
|
||||
$("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`;
|
||||
|
||||
const echo = Number(state.speech.echo);
|
||||
$("#tts-echo").value = Number.isFinite(echo) && state.speech.echo !== null ? echo : 0;
|
||||
$("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off";
|
||||
|
||||
$("#tts-words").checked = state.speech.words === true;
|
||||
const gap = Number(state.speech.wordGapMs);
|
||||
$("#tts-word-gap").value = Number.isFinite(gap) && state.speech.wordGapMs !== null ? gap : 120;
|
||||
$("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`;
|
||||
}
|
||||
|
||||
function fillProfileSlots() {
|
||||
const select = $("#tts-profile-slot");
|
||||
if (!select) return;
|
||||
|
||||
const presets = state.speech.profilePresets ?? [];
|
||||
const current = select.value || "0";
|
||||
select.replaceChildren(
|
||||
...Array.from({ length: 5 }, (_, index) => {
|
||||
const preset = presets[index];
|
||||
const label = preset ? `${index + 1} · ${preset.name ?? "profile"}` : `${index + 1} · empty`;
|
||||
return el("option", { value: String(index), text: label });
|
||||
})
|
||||
);
|
||||
select.value = current;
|
||||
|
||||
const preset = presets[Number(select.value)];
|
||||
const name = $("#tts-profile-name");
|
||||
if (name) name.value = preset?.name ?? "";
|
||||
}
|
||||
|
||||
function effectOptions() {
|
||||
return {
|
||||
profile: profileFromControls(),
|
||||
highpass: Number($("#tts-highpass").value),
|
||||
echo: Number($("#tts-echo").value),
|
||||
words: $("#tts-words").checked,
|
||||
wordGapMs: Number($("#tts-word-gap").value)
|
||||
};
|
||||
}
|
||||
function syncVoiceControls(voices, selected) {
|
||||
const voice = (voices ?? []).find(entry => entry.id === selected);
|
||||
const pitchable = !voice || voice.supportsPitch !== false;
|
||||
|
||||
$("#tts-pitch-control").hidden = !pitchable;
|
||||
$("#tts-pitch").disabled = !pitchable;
|
||||
|
||||
const expressive = !!voice?.supportsExpression;
|
||||
state.profileDefaults = voice?.profileDefaults ?? {};
|
||||
state.expressionDefault = typeof voice?.expressionDefault === "number" ? voice.expressionDefault : null;
|
||||
if (expressive) fillProfileControls();
|
||||
|
||||
{
|
||||
const slider = $("#tts-expression");
|
||||
const pref = state.speech.expression;
|
||||
|
||||
if (typeof pref === "number") {
|
||||
slider.value = pref;
|
||||
} else if (typeof voice.expressionDefault === "number") {
|
||||
slider.value = voice.expressionDefault;
|
||||
} else {
|
||||
slider.value = 50;
|
||||
}
|
||||
|
||||
slider.dataset.stored = typeof pref === "number" ? "1" : "0";
|
||||
delete slider.dataset.dirty;
|
||||
}
|
||||
|
||||
syncSpeechSliderLabels();
|
||||
}
|
||||
|
||||
/* Browser audio (puppeting) — separate from the bot's own audio devices. */
|
||||
|
||||
async function refreshBrowserAudio() {
|
||||
@@ -1793,19 +2124,38 @@ async function refreshBrowserAudio() {
|
||||
}
|
||||
|
||||
async function grantBrowserAudio() {
|
||||
if (await ensureBrowserAudio()) toast("Browser audio enabled", "ok");
|
||||
}
|
||||
|
||||
/** Get the browser ready for playback: permission, device labels, and a live
|
||||
* AudioContext. Must run inside the click so the context is not suspended. */
|
||||
async function ensureBrowserAudio() {
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
toast("Browser audio needs HTTPS or localhost", "error");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
toast("Browser audio enabled", "ok");
|
||||
await refreshBrowserAudio();
|
||||
} catch (error) {
|
||||
toast(`Could not enable browser audio: ${error.message}`, "error");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!state.clientAudio.ctx) {
|
||||
state.clientAudio.ctx = new AudioContext({ sampleRate: 48000 });
|
||||
applyBrowserSink();
|
||||
}
|
||||
|
||||
if (state.clientAudio.ctx.state === "suspended") {
|
||||
await state.clientAudio.ctx.resume().catch(error => {
|
||||
console.warn("[client-audio] could not resume the audio context:", error);
|
||||
});
|
||||
}
|
||||
|
||||
await refreshBrowserAudio();
|
||||
return true;
|
||||
}
|
||||
|
||||
function applyBrowserSink() {
|
||||
@@ -1824,6 +2174,8 @@ function playPcm(arrayBuffer) {
|
||||
}
|
||||
|
||||
const context = state.clientAudio.ctx;
|
||||
if (context.state === "suspended") void context.resume();
|
||||
|
||||
const samples = new Int16Array(arrayBuffer);
|
||||
const buffer = context.createBuffer(1, samples.length, 48000);
|
||||
const data = buffer.getChannelData(0);
|
||||
@@ -1846,7 +2198,10 @@ function updateClientAudioButton() {
|
||||
if (!button) return;
|
||||
|
||||
button.classList.toggle("active", state.clientAudio.on);
|
||||
button.textContent = state.clientAudio.on ? "Client audio on" : "Client audio";
|
||||
const label = button.querySelector(".vk-btn-text");
|
||||
const text = state.clientAudio.on ? "Client audio on" : "Client audio";
|
||||
if (label) label.textContent = text;
|
||||
else button.textContent = text;
|
||||
|
||||
const status = $("#browser-audio-status");
|
||||
if (status) status.textContent = state.clientAudio.on ? "on" : "off";
|
||||
@@ -1911,9 +2266,17 @@ function stopClientAudio() {
|
||||
updateClientAudioButton();
|
||||
}
|
||||
|
||||
function toggleClientAudio() {
|
||||
if (state.clientAudio.on || state.clientAudio.ws) stopClientAudio();
|
||||
else startClientAudio();
|
||||
async function toggleClientAudio() {
|
||||
if (state.clientAudio.on || state.clientAudio.ws) {
|
||||
stopClientAudio();
|
||||
return;
|
||||
}
|
||||
|
||||
// Clicking for audio is the consent: unlock the browser in the same gesture
|
||||
// so playback is not blocked by a suspended AudioContext.
|
||||
if (!(await ensureBrowserAudio())) return;
|
||||
|
||||
startClientAudio();
|
||||
}
|
||||
|
||||
function syncTtsControls() {
|
||||
@@ -1921,6 +2284,17 @@ function syncTtsControls() {
|
||||
if (speak) speak.disabled = !state.takeover;
|
||||
}
|
||||
|
||||
async function setCapture(start) {
|
||||
try {
|
||||
await VK.api.post("/api/audio/capture", { start });
|
||||
toast(start ? "Capture started" : "Capture stopped", "ok");
|
||||
await refreshTranscript();
|
||||
void refreshAudio();
|
||||
} catch (error) {
|
||||
toast(error.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function speakText() {
|
||||
if (!state.takeover) {
|
||||
toast("Composer is read-only — turn Takeover on to speak", "error");
|
||||
@@ -1932,7 +2306,20 @@ async function speakText() {
|
||||
|
||||
try {
|
||||
const provider = $("#tts-provider").value;
|
||||
await VK.api.post("/api/audio/speak", { text, provider });
|
||||
const speed = Number($("#tts-speed").value);
|
||||
const pitch = Number($("#tts-pitch").value);
|
||||
const expressionEl = $("#tts-expression");
|
||||
const expression =
|
||||
expressionEl.dataset.dirty === "1" ? Number(expressionEl.value) : state.speech.expression ?? undefined;
|
||||
|
||||
await VK.api.post("/api/audio/speak", {
|
||||
text,
|
||||
provider,
|
||||
speed,
|
||||
pitch,
|
||||
...(typeof expression === "number" ? { expression } : {}),
|
||||
...effectOptions()
|
||||
});
|
||||
$("#tts-status").textContent = "queued";
|
||||
toast("Queued for speech", "ok");
|
||||
void refreshAudio();
|
||||
@@ -1961,6 +2348,7 @@ function transcriptLine(segment) {
|
||||
async function refreshTranscript() {
|
||||
try {
|
||||
const result = await VK.api.get("/api/transcript?limit=200");
|
||||
const capture = await VK.api.get("/api/audio").catch(() => null);
|
||||
const list = $("#transcript-list");
|
||||
list.replaceChildren();
|
||||
|
||||
@@ -1969,7 +2357,7 @@ async function refreshTranscript() {
|
||||
list.append(
|
||||
el("div", {
|
||||
class: "vk-empty",
|
||||
text: "No captured segments yet. Start capture in Settings → Client → Bot audio."
|
||||
text: "No captured segments yet. Press Start above."
|
||||
})
|
||||
);
|
||||
} else {
|
||||
@@ -1978,9 +2366,10 @@ async function refreshTranscript() {
|
||||
}
|
||||
|
||||
const active = [...(result.sessions ?? [])].reverse().find(session => !session.endedAt);
|
||||
const captureState = capture?.capturing ? "capturing" : "capture off";
|
||||
$("#transcript-status").textContent = active
|
||||
? `session ${active.id.slice(-8)} · ${segments.length} segments`
|
||||
: `${segments.length} segments`;
|
||||
? `${captureState} · session ${active.id.slice(-8)} · ${segments.length} segments`
|
||||
: `${captureState} · ${segments.length} segments`;
|
||||
} catch (error) {
|
||||
toast(error.message, "error");
|
||||
}
|
||||
@@ -2139,22 +2528,16 @@ function handleLiveEvent(event) {
|
||||
}
|
||||
|
||||
if (event.type === "VOICE_SPEAKING") {
|
||||
if (event.speaking) state.speaking.add(event.userId);
|
||||
else state.speaking.delete(event.userId);
|
||||
|
||||
const now = Date.now();
|
||||
if (now - state.speakingRenderAt > 500) {
|
||||
state.speakingRenderAt = now;
|
||||
renderVoiceRail();
|
||||
renderPeople();
|
||||
renderRightVoice();
|
||||
renderRightFavourites();
|
||||
renderCallStrip();
|
||||
}
|
||||
if (event.speaking) beginSpeaking(event.userId);
|
||||
else endSpeaking(event.userId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "VOICE_STATE_UPDATE") {
|
||||
// A mute or a channel change ends speaking immediately: a muted member
|
||||
// never transmits, and the stop event may not arrive.
|
||||
if (!event.channelId || event.mute || event.selfMute) endSpeaking(event.userId);
|
||||
|
||||
const now = Date.now();
|
||||
if (now - state.voiceReloadAt > 1000) {
|
||||
state.voiceReloadAt = now;
|
||||
@@ -2282,6 +2665,18 @@ function init() {
|
||||
if (isMobileLayout()) setMobileTab("activity");
|
||||
});
|
||||
$("#transcript-refresh").addEventListener("click", () => void refreshTranscript());
|
||||
$("#transcript-start").addEventListener("click", () => void setCapture(true));
|
||||
$("#transcript-stop").addEventListener("click", () => void setCapture(false));
|
||||
$("#tts-duck").addEventListener("click", async () => {
|
||||
const on = !$("#tts-duck").classList.contains("vk-chip-on");
|
||||
try {
|
||||
await VK.api.post("/api/preferences", { audio: { duck: on } });
|
||||
toast(on ? "TTS ducking on" : "TTS ducking off", "ok");
|
||||
void refreshAudio();
|
||||
} catch (error) {
|
||||
toast(error.message, "error");
|
||||
}
|
||||
});
|
||||
$("#channel-refresh").addEventListener("click", () => void loadMessages({ refresh: true }));
|
||||
|
||||
$("#guilds-show-all").addEventListener("click", () => {
|
||||
@@ -2310,13 +2705,193 @@ function init() {
|
||||
void saveSpeech();
|
||||
});
|
||||
|
||||
for (const id of ["tts-speed", "tts-pitch"]) {
|
||||
$(`#${id}`).addEventListener("input", syncSpeechSliderLabels);
|
||||
$(`#${id}`).addEventListener("change", () => void saveSpeech());
|
||||
}
|
||||
$("#tts-expression").addEventListener("input", () => {
|
||||
const slider = $("#tts-expression");
|
||||
const engineDefault = state.expressionDefault;
|
||||
const snapped =
|
||||
typeof engineDefault === "number" && Math.abs(Number(slider.value) - engineDefault) <= 2;
|
||||
|
||||
if (snapped) {
|
||||
slider.value = engineDefault;
|
||||
slider.dataset.dirty = "";
|
||||
} else {
|
||||
slider.dataset.dirty = "1";
|
||||
}
|
||||
syncSpeechSliderLabels();
|
||||
});
|
||||
$("#tts-expression").addEventListener("change", () => void saveSpeech());
|
||||
buildProfileSliders();
|
||||
for (const id of ["tts-highpass", "tts-echo", "tts-word-gap"]) {
|
||||
$(`#${id}`).addEventListener("input", () => {
|
||||
$("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`;
|
||||
$("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off";
|
||||
$("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`;
|
||||
});
|
||||
$(`#${id}`).addEventListener("change", () => void saveSpeech());
|
||||
}
|
||||
$("#tts-words").addEventListener("change", () => void saveSpeech());
|
||||
$("#profile-open").addEventListener("click", () => {
|
||||
const panel = document.querySelector(".vk-profile-panel");
|
||||
if (panel) {
|
||||
panel.style.left = "";
|
||||
panel.style.top = "";
|
||||
panel.style.transform = "";
|
||||
}
|
||||
$("#profile-overlay").hidden = false;
|
||||
fillProfileControls();
|
||||
});
|
||||
$("#profile-close").addEventListener("click", () => {
|
||||
$("#profile-overlay").hidden = true;
|
||||
});
|
||||
$("#profile-overlay").addEventListener("click", event => {
|
||||
if (window.__profileDragged) {
|
||||
window.__profileDragged = false;
|
||||
return;
|
||||
}
|
||||
if (event.target === $("#profile-overlay")) $("#profile-overlay").hidden = true;
|
||||
});
|
||||
|
||||
(() => {
|
||||
const panel = document.querySelector(".vk-profile-panel");
|
||||
const handle = $("#profile-drag");
|
||||
if (!panel || !handle) return;
|
||||
|
||||
let dragging = false;
|
||||
let startX = 0;
|
||||
let startY = 0;
|
||||
let originX = 0;
|
||||
let originY = 0;
|
||||
|
||||
handle.addEventListener("mousedown", event => {
|
||||
const rect = panel.getBoundingClientRect();
|
||||
dragging = true;
|
||||
startX = event.clientX;
|
||||
startY = event.clientY;
|
||||
originX = rect.left;
|
||||
originY = rect.top;
|
||||
panel.style.transform = "none";
|
||||
panel.style.left = `${originX}px`;
|
||||
panel.style.top = `${originY}px`;
|
||||
event.preventDefault();
|
||||
});
|
||||
|
||||
window.addEventListener("mousemove", event => {
|
||||
if (!dragging) return;
|
||||
if (Math.abs(event.clientX - startX) + Math.abs(event.clientY - startY) > 3) {
|
||||
window.__profileDragged = true;
|
||||
}
|
||||
panel.style.left = `${originX + event.clientX - startX}px`;
|
||||
panel.style.top = `${originY + event.clientY - startY}px`;
|
||||
});
|
||||
|
||||
window.addEventListener("mouseup", () => {
|
||||
dragging = false;
|
||||
});
|
||||
})();
|
||||
|
||||
for (const slot of ["1", "2"]) {
|
||||
const input = $(`#tts-test-custom-${slot}`);
|
||||
input.value = localStorage.getItem(`veskslop-test-line-${slot}`) ?? "";
|
||||
input.addEventListener("input", () =>
|
||||
localStorage.setItem(`veskslop-test-line-${slot}`, input.value)
|
||||
);
|
||||
}
|
||||
|
||||
$$(".vk-profile-tests button").forEach(button => {
|
||||
button.addEventListener("click", async () => {
|
||||
let text = button.dataset.test ?? "";
|
||||
if (button.dataset.testCustom) {
|
||||
text = $(`#tts-test-custom-${button.dataset.testCustom}`).value.trim();
|
||||
if (!text) {
|
||||
toast("Custom line is empty", "error");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const expressionEl = $("#tts-expression");
|
||||
const expression =
|
||||
expressionEl.dataset.dirty === "1"
|
||||
? Number(expressionEl.value)
|
||||
: state.speech.expression ?? undefined;
|
||||
|
||||
await VK.api.post("/api/audio/speak", {
|
||||
text,
|
||||
provider: $("#tts-provider").value,
|
||||
...effectOptions(),
|
||||
...(typeof expression === "number" ? { expression } : {})
|
||||
});
|
||||
toast(`Playing "${text}"`, "ok");
|
||||
} catch (error) {
|
||||
toast(error.message, "error");
|
||||
}
|
||||
});
|
||||
});
|
||||
$("#tts-profile-slot").addEventListener("change", () => {
|
||||
const preset = (state.speech.profilePresets ?? [])[Number($("#tts-profile-slot").value)];
|
||||
$("#tts-profile-name").value = preset?.name ?? "";
|
||||
});
|
||||
$("#tts-profile-save").addEventListener("click", async () => {
|
||||
const slot = Number($("#tts-profile-slot").value);
|
||||
const presets = (state.speech.profilePresets ?? [null, null, null, null, null]).slice(0, 5);
|
||||
const name = $("#tts-profile-name").value.trim() || `Profile ${slot + 1}`;
|
||||
presets[slot] = {
|
||||
name,
|
||||
...profileFromControls(),
|
||||
...(typeof state.speech.expression === "number" ? { expression: state.speech.expression } : {})
|
||||
};
|
||||
await saveSpeech({ profilePresets: presets });
|
||||
toast(`Saved "${name}" to slot ${slot + 1}`, "ok");
|
||||
});
|
||||
$("#tts-profile-load").addEventListener("click", async () => {
|
||||
const slot = Number($("#tts-profile-slot").value);
|
||||
const preset = (state.speech.profilePresets ?? [])[slot];
|
||||
if (!preset) {
|
||||
toast(`Slot ${slot + 1} is empty`, "error");
|
||||
return;
|
||||
}
|
||||
for (const [key] of PROFILE_SPEC) {
|
||||
const slider = $(`#tts-profile-${key}`);
|
||||
if (!slider || typeof preset[key] !== "number") continue;
|
||||
slider.value = preset[key];
|
||||
slider.dataset.dirty = "1";
|
||||
}
|
||||
if (typeof preset.expression === "number") {
|
||||
$("#tts-expression").value = preset.expression;
|
||||
$("#tts-expression").dataset.dirty = "1";
|
||||
}
|
||||
fillProfileControls();
|
||||
syncSpeechSliderLabels();
|
||||
await saveSpeech();
|
||||
toast(`Loaded "${preset.name ?? `Profile ${slot + 1}`}"`, "ok");
|
||||
});
|
||||
$("#tts-speed-presets").addEventListener("click", event => {
|
||||
const button = event.target.closest("button[data-speed]");
|
||||
if (!button) return;
|
||||
|
||||
$("#tts-speed").value = button.dataset.speed;
|
||||
syncSpeechSliderLabels();
|
||||
void saveSpeech();
|
||||
});
|
||||
syncSpeechSliderLabels();
|
||||
|
||||
$("#speech-provider").addEventListener("change", () => void saveSpeech());
|
||||
$("#speech-warm").addEventListener("click", () => void warmEngine());
|
||||
$("#speech-language").addEventListener("change", () => {
|
||||
$("#speech-voice").value = "";
|
||||
void saveSpeech();
|
||||
});
|
||||
$("#speech-voice").addEventListener("change", () => void saveSpeech());
|
||||
|
||||
$("#tts-voice").addEventListener("change", () => {
|
||||
$("#speech-voice").value = $("#tts-voice").value;
|
||||
void saveSpeech();
|
||||
});
|
||||
|
||||
$("#scope-guild").addEventListener("click", () => void applyScope("guild"));
|
||||
$("#scope-dm").addEventListener("click", () => void applyScope("dm"));
|
||||
|
||||
|
||||
+113
-11
@@ -129,12 +129,34 @@
|
||||
</div>
|
||||
<div class="vk-tts-settings">
|
||||
<span class="vk-micro-label">Voice settings</span>
|
||||
<select id="tts-provider" class="vk-select">
|
||||
<option value="local">local TTS</option>
|
||||
<option value="api">API TTS</option>
|
||||
</select>
|
||||
<span class="vk-chip vk-chip-on">duck on voice</span>
|
||||
<select id="tts-provider" class="vk-select"></select>
|
||||
<select id="tts-voice" class="vk-select" title="Voice"></select>
|
||||
<button id="tts-duck" class="vk-chip vk-chip-on" type="button" title="Lower the TTS while someone else speaks">
|
||||
duck on voice
|
||||
</button>
|
||||
<span class="vk-chip vk-chip-on">queue lines</span>
|
||||
<div class="vk-tts-speed">
|
||||
<label class="vk-tts-slider" title="Speech rate; 1.0x is the engine's natural rate">
|
||||
<span class="vk-micro-label">speed</span>
|
||||
<input id="tts-speed" type="range" min="0.5" max="2" step="0.05" value="1" />
|
||||
<span id="tts-speed-value" class="vk-mono vk-dim">1.0x</span>
|
||||
</label>
|
||||
<div id="tts-speed-presets" class="vk-tts-presets">
|
||||
<button class="vk-chip vk-chip-off" type="button" data-speed="0.5">0.5x</button>
|
||||
<button class="vk-chip vk-chip-off" type="button" data-speed="0.75">0.75x</button>
|
||||
<button class="vk-chip vk-chip-on" type="button" data-speed="1">1x</button>
|
||||
<button class="vk-chip vk-chip-off" type="button" data-speed="1.25">1.25x</button>
|
||||
<button class="vk-chip vk-chip-off" type="button" data-speed="1.5">1.5x</button>
|
||||
</div>
|
||||
</div>
|
||||
<label id="tts-pitch-control" class="vk-tts-slider" title="Pitch offset; 0 is the engine's natural pitch">
|
||||
<span class="vk-micro-label">pitch</span>
|
||||
<input id="tts-pitch" type="range" min="-12" max="12" step="1" value="0" />
|
||||
<span id="tts-pitch-value" class="vk-mono vk-dim">0 st</span>
|
||||
</label>
|
||||
<button id="profile-open" class="vk-chip vk-chip-off" type="button" title="Flextalk voice profile">
|
||||
voice profile…
|
||||
</button>
|
||||
</div>
|
||||
<div class="vk-tts-body">
|
||||
<textarea id="tts-input" class="vk-tts-field" rows="3" placeholder="Speak text into the call…"></textarea>
|
||||
@@ -216,6 +238,8 @@
|
||||
<div class="vk-transcript">
|
||||
<div class="vk-transcript-toolbar">
|
||||
<span id="transcript-status" class="vk-mono vk-dim">no capture session</span>
|
||||
<button id="transcript-start" class="btn btn-secondary vk-small-btn" type="button">Start</button>
|
||||
<button id="transcript-stop" class="btn btn-secondary vk-small-btn" type="button">Stop</button>
|
||||
<button id="transcript-refresh" class="btn btn-secondary vk-small-btn" type="button">Refresh</button>
|
||||
</div>
|
||||
<div id="transcript-list" class="vk-transcript-list"></div>
|
||||
@@ -307,13 +331,14 @@
|
||||
<div class="vk-settings-card">
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">TTS engine</span>
|
||||
<select class="vk-select" id="speech-provider">
|
||||
<option value="">Auto</option>
|
||||
<option value="local">Local</option>
|
||||
<option value="api">API</option>
|
||||
</select>
|
||||
<span class="vk-engine-row">
|
||||
<select class="vk-select" id="speech-provider"></select>
|
||||
<button id="speech-warm" class="btn btn-secondary vk-small-btn" type="button" title="Start the engine so the first line is fast">
|
||||
Start engine
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<div class="vk-status-row" id="speech-language-row">
|
||||
<span class="vk-status-label">Language</span>
|
||||
<select class="vk-select" id="speech-language">
|
||||
<option value="">Auto</option>
|
||||
@@ -427,6 +452,83 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="profile-overlay" class="vk-overlay" hidden>
|
||||
<div class="vk-profile-panel elev-lg" role="dialog" aria-label="Flextalk voice profile">
|
||||
<div class="vk-settings-head" id="profile-drag">
|
||||
<span class="vk-settings-title">Flextalk voice profile</span>
|
||||
<span class="vk-mono vk-dim" id="profile-voice-label"></span>
|
||||
<button id="profile-close" class="btn btn-ghost vk-icon-btn" type="button" aria-label="Close">
|
||||
<span class="vk-icon" data-icon="x"></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="vk-profile-body">
|
||||
<section class="vk-profile-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Voice character</span>
|
||||
<span class="vk-mono vk-dim">within 2 of the engine default snaps to default</span>
|
||||
</div>
|
||||
<div id="tts-profile-sliders" class="vk-profile-grid"></div>
|
||||
<div class="vk-profile-row">
|
||||
<span class="vk-micro-label">expression</span>
|
||||
<input id="tts-expression" type="range" min="0" max="100" step="1" value="59" />
|
||||
<span id="tts-expression-value" class="vk-mono vk-dim">default</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="vk-profile-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Presets</span>
|
||||
</div>
|
||||
<div class="vk-profile-row">
|
||||
<select id="tts-profile-slot" class="vk-select"></select>
|
||||
<input id="tts-profile-name" class="input" placeholder="Profile name" maxlength="40" />
|
||||
<button id="tts-profile-load" class="btn btn-secondary vk-small-btn" type="button">Load</button>
|
||||
<button id="tts-profile-save" class="btn btn-primary vk-small-btn" type="button">Save</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="vk-profile-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Radio kit</span>
|
||||
<span class="vk-mono vk-dim">VOX / HEV</span>
|
||||
</div>
|
||||
<div class="vk-profile-row">
|
||||
<span class="vk-micro-label">highpass</span>
|
||||
<input id="tts-highpass" type="range" min="40" max="400" step="10" value="40" />
|
||||
<span id="tts-highpass-value" class="vk-mono vk-dim">40Hz</span>
|
||||
</div>
|
||||
<div class="vk-profile-row">
|
||||
<span class="vk-micro-label">echo</span>
|
||||
<input id="tts-echo" type="range" min="0" max="100" step="5" value="0" />
|
||||
<span id="tts-echo-value" class="vk-mono vk-dim">off</span>
|
||||
</div>
|
||||
<div class="vk-profile-row">
|
||||
<label class="vk-tts-check"><input id="tts-words" type="checkbox" /> word mode</label>
|
||||
<input id="tts-word-gap" type="range" min="0" max="500" step="10" value="120" />
|
||||
<span id="tts-word-gap-value" class="vk-mono vk-dim">120ms</span>
|
||||
</div>
|
||||
</section>
|
||||
<section class="vk-profile-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Test lines</span>
|
||||
</div>
|
||||
<div class="vk-profile-tests">
|
||||
<button class="vk-chip vk-chip-off" data-test="Attention" type="button">Attention</button>
|
||||
<button class="vk-chip vk-chip-off" data-test="Laboratory" type="button">Laboratory</button>
|
||||
<button class="vk-chip vk-chip-off" data-test="Critical" type="button">Critical</button>
|
||||
<button class="vk-chip vk-chip-off" data-test="Health" type="button">Health</button>
|
||||
<button class="vk-chip vk-chip-off" data-test="Radiation" type="button">Radiation</button>
|
||||
<input class="input vk-test-custom" id="tts-test-custom-1" placeholder="custom 1" />
|
||||
<button class="vk-chip vk-chip-off" data-test-custom="1" type="button">play</button>
|
||||
<input class="input vk-test-custom" id="tts-test-custom-2" placeholder="custom 2" />
|
||||
<button class="vk-chip vk-chip-off" data-test-custom="2" type="button">play</button>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="toasts" class="vk-toasts"></div>
|
||||
|
||||
<script src="icons.js"></script>
|
||||
|
||||
+224
-5
@@ -950,8 +950,8 @@ a.vk-attachment:hover {
|
||||
.vk-composer-input {
|
||||
flex: 1;
|
||||
min-height: 38px;
|
||||
max-height: 160px;
|
||||
resize: none;
|
||||
max-height: 50vh;
|
||||
resize: vertical;
|
||||
padding: 8px 10px;
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--color-bg);
|
||||
@@ -1016,6 +1016,46 @@ a.vk-attachment:hover {
|
||||
padding: 9px 11px 8px;
|
||||
}
|
||||
|
||||
.vk-tts-slider {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.vk-tts-slider input[type="range"] {
|
||||
width: 84px;
|
||||
accent-color: var(--color-accent);
|
||||
}
|
||||
|
||||
.vk-tts-slider .vk-mono {
|
||||
min-width: 34px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.vk-tts-speed {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.vk-tts-presets {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
padding-left: 2px;
|
||||
}
|
||||
|
||||
.vk-tts-presets .vk-chip {
|
||||
padding: 3px 7px;
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.vk-engine-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-chip {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -1027,6 +1067,14 @@ a.vk-attachment:hover {
|
||||
box-shadow: inset 0 0 0 1px var(--color-accent-800);
|
||||
color: var(--color-neutral-300);
|
||||
font: 400 11px/1 ui-monospace, Menlo, monospace;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.vk-chip-off {
|
||||
background: var(--color-bg);
|
||||
box-shadow: inset 0 0 0 1px var(--color-neutral-800);
|
||||
color: var(--color-neutral-500);
|
||||
}
|
||||
|
||||
.vk-chip-on {
|
||||
@@ -1047,8 +1095,9 @@ a.vk-attachment:hover {
|
||||
|
||||
.vk-tts-field {
|
||||
width: 100%;
|
||||
max-height: 150px;
|
||||
resize: none;
|
||||
min-height: 64px;
|
||||
max-height: 40vh;
|
||||
resize: vertical;
|
||||
border: none;
|
||||
outline: none;
|
||||
background: transparent;
|
||||
@@ -1203,7 +1252,11 @@ a.vk-attachment:hover {
|
||||
}
|
||||
|
||||
.vk-transcript-toolbar .vk-small-btn {
|
||||
margin-left: auto;
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.vk-transcript-toolbar #transcript-status {
|
||||
margin-right: auto;
|
||||
}
|
||||
|
||||
.vk-transcript-list {
|
||||
@@ -1266,6 +1319,9 @@ a.vk-attachment:hover {
|
||||
align-items: flex-end;
|
||||
gap: 2px;
|
||||
height: 11px;
|
||||
transform-origin: bottom;
|
||||
transform: scaleY(0);
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.vk-levels span {
|
||||
@@ -1274,9 +1330,49 @@ a.vk-attachment:hover {
|
||||
border-radius: 1px;
|
||||
background: var(--color-accent);
|
||||
transform-origin: bottom;
|
||||
}
|
||||
|
||||
.vk-levels.speaking {
|
||||
opacity: 1;
|
||||
transform: scaleY(1);
|
||||
animation: vk-levels-in 200ms ease-out;
|
||||
}
|
||||
|
||||
.vk-levels.speaking span,
|
||||
.vk-levels.stopping span {
|
||||
animation: vk-lvl 0.7s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.vk-levels.stopping {
|
||||
animation: vk-levels-out 700ms ease-in forwards;
|
||||
}
|
||||
|
||||
@keyframes vk-levels-in {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: scaleY(0.05);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: scaleY(1);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes vk-levels-out {
|
||||
0% {
|
||||
opacity: 1;
|
||||
transform: scaleY(1);
|
||||
}
|
||||
60% {
|
||||
opacity: 0.8;
|
||||
transform: scaleY(0.35);
|
||||
}
|
||||
100% {
|
||||
opacity: 0;
|
||||
transform: scaleY(0.02);
|
||||
}
|
||||
}
|
||||
|
||||
.vk-levels span:nth-child(2) {
|
||||
animation-delay: 0.1s;
|
||||
}
|
||||
@@ -1308,6 +1404,19 @@ a.vk-attachment:hover {
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.vk-levels,
|
||||
.vk-levels.speaking,
|
||||
.vk-levels.stopping {
|
||||
animation: none;
|
||||
}
|
||||
.vk-levels.speaking {
|
||||
opacity: 1;
|
||||
transform: scaleY(1);
|
||||
}
|
||||
.vk-levels.stopping {
|
||||
opacity: 0;
|
||||
transform: scaleY(0);
|
||||
}
|
||||
.vk-levels span {
|
||||
animation: none;
|
||||
transform: scaleY(0.6);
|
||||
@@ -1482,6 +1591,116 @@ a.vk-attachment:hover {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Flextalk voice profile: a bottom drawer over the chat, separate from the TTS row. */
|
||||
|
||||
.vk-profile-panel {
|
||||
position: fixed;
|
||||
left: 50%;
|
||||
top: 10vh;
|
||||
transform: translateX(-50%);
|
||||
width: min(760px, 100%);
|
||||
max-height: 76vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: var(--color-surface);
|
||||
box-shadow: var(--shadow-lg);
|
||||
border-radius: var(--radius-lg) var(--radius-lg) 0 0;
|
||||
animation: vk-drawer-in 180ms ease-out;
|
||||
}
|
||||
|
||||
.vk-profile-body {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
padding: 6px 16px 18px;
|
||||
}
|
||||
|
||||
.vk-profile-section {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.vk-profile-section + .vk-profile-section {
|
||||
padding-top: 14px;
|
||||
box-shadow: inset 0 1px 0 var(--color-neutral-800);
|
||||
}
|
||||
|
||||
.vk-profile-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 6px 20px;
|
||||
}
|
||||
|
||||
.vk-profile-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.vk-profile-row input[type="range"] {
|
||||
flex: 1;
|
||||
min-width: 80px;
|
||||
accent-color: var(--color-accent);
|
||||
}
|
||||
|
||||
.vk-profile-row .vk-mono {
|
||||
min-width: 44px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.vk-profile-row .input {
|
||||
flex: 1;
|
||||
min-height: 28px;
|
||||
}
|
||||
|
||||
#profile-drag {
|
||||
cursor: move;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.vk-profile-tests {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.vk-test-custom {
|
||||
width: 110px;
|
||||
min-height: 26px;
|
||||
}
|
||||
|
||||
@keyframes vk-drawer-in {
|
||||
from {
|
||||
transform: translateY(26px);
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
transform: translateY(0);
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.vk-profile-panel {
|
||||
max-height: 82vh;
|
||||
}
|
||||
|
||||
.vk-profile-grid {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.vk-profile-panel {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
|
||||
.vk-settings-panel {
|
||||
width: min(680px, 100%);
|
||||
height: 100%;
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
# Veskslop audio doctor. Run it on the server while the bot sits in a call and
|
||||
# people are talking. It shows where client audio goes and what the daemon sees.
|
||||
set -uo pipefail
|
||||
|
||||
PORT="${WEBUI_PORT:-8789}"
|
||||
|
||||
section() { printf '\n== %s ==\n' "$1"; }
|
||||
|
||||
section "host addresses"
|
||||
hostname -I 2>/dev/null || true
|
||||
|
||||
section "default sink"
|
||||
pactl get-default-sink 2>&1 || true
|
||||
|
||||
section "sinks"
|
||||
pactl list short sinks 2>&1 || true
|
||||
|
||||
section "sink inputs: who is playing, and into which sink"
|
||||
if command -v pactl >/dev/null; then
|
||||
pactl list short sink-inputs 2>&1 || true
|
||||
echo
|
||||
echo "-- vesktop stream state (Corked / Mute / Volume) --"
|
||||
pactl list sink-inputs 2>&1 | grep -E "Sink Input|Sink:|Corked:|Mute:|Volume:|application\.(name|process\.binary)|media\.name" || true
|
||||
fi
|
||||
|
||||
section "microphone sources the client reads"
|
||||
if command -v pactl >/dev/null; then
|
||||
pactl list short source-outputs 2>&1 || true
|
||||
echo
|
||||
echo "-- source output names --"
|
||||
pactl list source-outputs 2>&1 | grep -E "Source Output #|Source:|application\.(name|process\.binary)|media\.name" || true
|
||||
fi
|
||||
|
||||
section "client processes"
|
||||
ps -eo pid,args 2>/dev/null | grep -iE "vesktop|electron" | grep -v grep || echo "no veskstop/electron process found"
|
||||
|
||||
section "pulse clients"
|
||||
if command -v pactl >/dev/null; then
|
||||
pactl list short clients 2>&1 || true
|
||||
fi
|
||||
|
||||
section "daemon audio status"
|
||||
if command -v curl >/dev/null; then
|
||||
curl -sk "https://127.0.0.1:${PORT}/api/audio" || curl -s "http://127.0.0.1:${PORT}/api/audio"
|
||||
echo
|
||||
else
|
||||
echo "curl is missing"
|
||||
fi
|
||||
|
||||
section "bot identity"
|
||||
if command -v curl >/dev/null; then
|
||||
curl -sk "https://127.0.0.1:${PORT}/health" || curl -s "http://127.0.0.1:${PORT}/health"
|
||||
echo
|
||||
fi
|
||||
|
||||
section "bot voice state"
|
||||
if command -v curl >/dev/null; then
|
||||
curl -sk "https://127.0.0.1:${PORT}/api/voice" || curl -s "http://127.0.0.1:${PORT}/api/voice"
|
||||
echo
|
||||
fi
|
||||
|
||||
section "volumes: client output and per-user"
|
||||
if command -v curl >/dev/null; then
|
||||
curl -sk "https://127.0.0.1:${PORT}/api/voice/volume" || curl -s "http://127.0.0.1:${PORT}/api/voice/volume"
|
||||
echo
|
||||
fi
|
||||
|
||||
section "capture sink level for 5 seconds (talk during this)"
|
||||
if command -v ffmpeg >/dev/null; then
|
||||
ffmpeg -hide_banner -loglevel info -f pulse -i "veskslop_capture.monitor" -t 5 \
|
||||
-af volumedetect -f null - 2>&1 | grep -E "mean_volume|max_volume" || true
|
||||
else
|
||||
echo "ffmpeg is missing"
|
||||
fi
|
||||
|
||||
section "live events for 8 seconds (talk during this)"
|
||||
if command -v curl >/dev/null; then
|
||||
curl -skN --max-time 8 "https://127.0.0.1:${PORT}/api/events" 2>/dev/null \
|
||||
| grep -E "VOICE_SPEAKING|VOICE_STATE_UPDATE" | head -10 || true
|
||||
fi
|
||||
|
||||
printf '\nDaemon terminal lines to look for: [audio] moved stream, [capture] auto-start, ffmpeg errors.\n'
|
||||
@@ -10,7 +10,7 @@ import { AudioManager } from "./audio.js";
|
||||
import type { CaptureSegment } from "./capture.js";
|
||||
import { CapturePipeline } from "./capture.js";
|
||||
import type { Stt } from "./stt.js";
|
||||
import { Tts, type TtsOptions, type TtsVoice } from "./tts.js";
|
||||
import { Tts, type TtsOptions, type TtsProvider, type TtsVoice } from "./tts.js";
|
||||
import type { TranscriptSegment, TranscriptSession, TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD, type VadConfig } from "./vad.js";
|
||||
|
||||
@@ -50,10 +50,12 @@ export class AudioService {
|
||||
private readonly speakerIntervals: SpeakerInterval[] = [];
|
||||
private readonly speakerNames = new Map<string, string>();
|
||||
private playing = false;
|
||||
private duckEnabled = true;
|
||||
private nextId = 1;
|
||||
private ducked = false;
|
||||
private selfId?: string;
|
||||
private capture?: CapturePipeline;
|
||||
private captureAdoptTimer?: ReturnType<typeof setInterval>;
|
||||
private session?: TranscriptSession;
|
||||
private segmentCount = 0;
|
||||
private initialized = false;
|
||||
@@ -94,6 +96,7 @@ export class AudioService {
|
||||
stt: this.stt.status(),
|
||||
queue: this.queue.length,
|
||||
speaking: this.playing,
|
||||
duckEnabled: this.duckEnabled,
|
||||
capturing: !!this.capture,
|
||||
sessionId: this.session?.id ?? null,
|
||||
segments: this.segmentCount,
|
||||
@@ -101,6 +104,16 @@ export class AudioService {
|
||||
};
|
||||
}
|
||||
|
||||
/** Enable or disable ducking the TTS while others speak. */
|
||||
setDuckEnabled(on: boolean): void {
|
||||
this.duckEnabled = on;
|
||||
void this.applyDuck(this.shouldDuck());
|
||||
}
|
||||
|
||||
private shouldDuck(): boolean {
|
||||
return this.duckEnabled && this.playing && this.speakers.size > 0;
|
||||
}
|
||||
|
||||
async speak(text: string, options: TtsOptions = {}): Promise<SpeakResult> {
|
||||
await this.init();
|
||||
|
||||
@@ -109,12 +122,18 @@ export class AudioService {
|
||||
|
||||
const tts = this.tts.status();
|
||||
const provider = options.provider ?? tts.defaultProvider;
|
||||
if (provider === "local" && !tts.local) {
|
||||
throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
if (provider === "api" && !tts.api) {
|
||||
throw new Error("API TTS is not configured (VESKSLOP_TTS_API_URL)");
|
||||
}
|
||||
if (provider === "command" && !tts.command) {
|
||||
throw new Error("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
if (provider === "espeak" && !tts.espeak) {
|
||||
throw new Error("espeak-ng is not installed");
|
||||
}
|
||||
if (provider === "local" && !tts.local) {
|
||||
throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) throw new Error("text is required");
|
||||
@@ -156,7 +175,7 @@ export class AudioService {
|
||||
}
|
||||
|
||||
this.pruneIntervals();
|
||||
await this.applyDuck(this.playing && this.speakers.size > 0);
|
||||
await this.applyDuck(this.shouldDuck());
|
||||
}
|
||||
|
||||
async startCapture(): Promise<{ sessionId: string }> {
|
||||
@@ -185,6 +204,8 @@ export class AudioService {
|
||||
});
|
||||
|
||||
this.capture.start();
|
||||
void this.manager.adoptClientStreams();
|
||||
this.captureAdoptTimer = setInterval(() => void this.manager.adoptClientStreams(), 5000);
|
||||
this.session = session;
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION", session });
|
||||
|
||||
@@ -192,6 +213,8 @@ export class AudioService {
|
||||
}
|
||||
|
||||
stopCapture(): void {
|
||||
clearInterval(this.captureAdoptTimer);
|
||||
this.captureAdoptTimer = undefined;
|
||||
this.capture?.stop();
|
||||
this.capture = undefined;
|
||||
|
||||
@@ -209,8 +232,12 @@ export class AudioService {
|
||||
return this.manager.startMonitor(onData, onError);
|
||||
}
|
||||
|
||||
ttsVoices(): Promise<TtsVoice[]> {
|
||||
return this.tts.voices();
|
||||
ttsVoices(engine?: TtsProvider): Promise<TtsVoice[]> {
|
||||
return this.tts.voices(engine);
|
||||
}
|
||||
|
||||
warmTts(engine?: TtsProvider) {
|
||||
return this.tts.warm(engine);
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
@@ -259,7 +286,10 @@ export class AudioService {
|
||||
|
||||
try {
|
||||
const text = await this.stt.transcribe(record.wav);
|
||||
if (!text) return;
|
||||
if (!text) {
|
||||
console.warn(`[capture] transcription empty for ${record.wav}`);
|
||||
return;
|
||||
}
|
||||
|
||||
this.transcripts.setSegmentText(record.id, text);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_TEXT", id: record.id, text });
|
||||
@@ -321,7 +351,7 @@ export class AudioService {
|
||||
this.playing = true;
|
||||
try {
|
||||
const file = await this.tts.synthesize(item.text, item.options);
|
||||
await this.applyDuck(this.speakers.size > 0);
|
||||
await this.applyDuck(this.shouldDuck());
|
||||
await this.manager.play(file);
|
||||
} catch (error) {
|
||||
console.error("[audio] speak failed:", (error as Error).message);
|
||||
|
||||
+104
-2
@@ -13,6 +13,7 @@ const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts";
|
||||
const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic";
|
||||
const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture";
|
||||
const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30);
|
||||
const LEAD_MS = Math.min(Math.max(Number(process.env.VESKSLOP_TTS_LEAD_MS ?? 150) || 0, 0), 2000);
|
||||
|
||||
export class AudioUnavailableError extends Error {}
|
||||
|
||||
@@ -26,10 +27,15 @@ export interface AudioManagerStatus {
|
||||
captureSink: string;
|
||||
ducked: boolean;
|
||||
duckPercent: number;
|
||||
/** Lead-in silence in milliseconds before each TTS line. */
|
||||
ttsLeadMs: number;
|
||||
playing: boolean;
|
||||
recording: boolean;
|
||||
player: boolean;
|
||||
ffmpeg: boolean;
|
||||
clientMatch: string[];
|
||||
adopted: number;
|
||||
lastAdoptError?: string;
|
||||
}
|
||||
|
||||
interface CommandResult {
|
||||
@@ -69,6 +75,8 @@ export class AudioManager {
|
||||
private backend: "pulse" | null = null;
|
||||
private playerAvailable = false;
|
||||
private ffmpegAvailable = false;
|
||||
private adoptedTotal = 0;
|
||||
private lastAdoptError?: string;
|
||||
|
||||
async detect(): Promise<void> {
|
||||
const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]);
|
||||
@@ -128,13 +136,24 @@ export class AudioManager {
|
||||
captureSink: CAPTURE_SINK,
|
||||
ducked: this.ducked,
|
||||
duckPercent: DUCK_PERCENT,
|
||||
ttsLeadMs: LEAD_MS,
|
||||
playing: !!this.player,
|
||||
recording: !!this.recorder,
|
||||
player: this.playerAvailable,
|
||||
ffmpeg: this.ffmpegAvailable
|
||||
ffmpeg: this.ffmpegAvailable,
|
||||
clientMatch: this.clientPatterns(),
|
||||
adopted: this.adoptedTotal,
|
||||
lastAdoptError: this.lastAdoptError
|
||||
};
|
||||
}
|
||||
|
||||
private clientPatterns(): string[] {
|
||||
return (process.env.VESKSLOP_AUDIO_CLIENT_MATCH ?? "vesktop,electron,chromium,discord")
|
||||
.split(",")
|
||||
.map(part => part.trim().toLowerCase())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** Create the virtual devices when they are missing. Safe to call twice. */
|
||||
async ensureDevices(): Promise<void> {
|
||||
if (!this.available) await this.detect();
|
||||
@@ -173,6 +192,76 @@ export class AudioManager {
|
||||
]);
|
||||
this.modules.push(Number(id));
|
||||
}
|
||||
|
||||
await this.adoptClientStreams();
|
||||
}
|
||||
|
||||
/**
|
||||
* Move the client's playback streams into the capture sink. Without this the
|
||||
* capture sink stays silent and the browser monitor streams zero samples.
|
||||
*/
|
||||
async adoptClientStreams(): Promise<number> {
|
||||
if (!this.available) return 0;
|
||||
|
||||
const patterns = this.clientPatterns();
|
||||
if (!patterns.length) return 0;
|
||||
|
||||
const sinks = await run("pactl", ["list", "short", "sinks"]);
|
||||
if (sinks.code !== 0) {
|
||||
this.lastAdoptError = sinks.stderr.trim() || "pactl list sinks failed";
|
||||
return 0;
|
||||
}
|
||||
|
||||
const captureIndex = sinks.stdout
|
||||
.split("\n")
|
||||
.map(line => line.split(/\s+/))
|
||||
.find(fields => fields[1] === CAPTURE_SINK)?.[0];
|
||||
if (captureIndex === undefined) return 0;
|
||||
|
||||
const list = await run("pactl", ["list", "sink-inputs"]);
|
||||
if (list.code !== 0) {
|
||||
this.lastAdoptError = list.stderr.trim() || "pactl list sink-inputs failed";
|
||||
return 0;
|
||||
}
|
||||
|
||||
let moved = 0;
|
||||
let index: string | undefined;
|
||||
let sinkIndex: string | undefined;
|
||||
let matchText = "";
|
||||
|
||||
const tryMove = async () => {
|
||||
if (!index || sinkIndex === captureIndex) return;
|
||||
if (!patterns.some(pattern => matchText.includes(pattern))) return;
|
||||
|
||||
const result = await run("pactl", ["move-sink-input", index, CAPTURE_SINK]);
|
||||
if (result.code === 0) {
|
||||
moved++;
|
||||
this.adoptedTotal++;
|
||||
console.log(`[audio] moved stream ${index} to ${CAPTURE_SINK}`);
|
||||
} else {
|
||||
this.lastAdoptError = result.stderr.trim() || `move-sink-input ${index} failed`;
|
||||
}
|
||||
};
|
||||
|
||||
for (const line of list.stdout.split("\n")) {
|
||||
const header = line.match(/^Sink Input #(\d+)/);
|
||||
if (header) {
|
||||
await tryMove();
|
||||
index = header[1];
|
||||
sinkIndex = undefined;
|
||||
matchText = "";
|
||||
continue;
|
||||
}
|
||||
|
||||
const sink = line.match(/^\s*Sink:\s*(\d+)/);
|
||||
if (sink) sinkIndex = sink[1];
|
||||
|
||||
const property = line.match(/^\s*(application\.name|application\.process\.binary)\s*=\s*"?([^"]+)"?/);
|
||||
if (property) matchText += ` ${property[2].toLowerCase()}`;
|
||||
}
|
||||
await tryMove();
|
||||
|
||||
return moved;
|
||||
}
|
||||
|
||||
/** Stop children and unload the modules this daemon created. */
|
||||
@@ -191,8 +280,15 @@ export class AudioManager {
|
||||
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
|
||||
|
||||
const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay";
|
||||
|
||||
// Lead-in silence: give Discord's voice pipeline a moment to settle
|
||||
// before the first syllable, so the attack is not swallowed.
|
||||
const args = ["-nodisp", "-autoexit", "-loglevel", "error"];
|
||||
if (LEAD_MS > 0) args.push("-af", `adelay=${LEAD_MS}:all=1`);
|
||||
args.push(file);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], {
|
||||
const child = spawn(binary, args, {
|
||||
env: { ...process.env, PULSE_SINK: SINK },
|
||||
stdio: ["ignore", "ignore", "pipe"]
|
||||
});
|
||||
@@ -278,6 +374,11 @@ export class AudioManager {
|
||||
}
|
||||
|
||||
const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
|
||||
|
||||
// The client may start or restart its stream while we listen.
|
||||
void this.adoptClientStreams();
|
||||
const adoptTimer = setInterval(() => void this.adoptClientStreams(), 5000);
|
||||
|
||||
const child = spawn(
|
||||
binary,
|
||||
[
|
||||
@@ -308,6 +409,7 @@ export class AudioManager {
|
||||
});
|
||||
|
||||
return () => {
|
||||
clearInterval(adoptTimer);
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
|
||||
+187
-9
@@ -22,7 +22,7 @@ import { Stt } from "./stt.js";
|
||||
import { TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD } from "./vad.js";
|
||||
import { CoreClient, coreSocketPath } from "./core-client.js";
|
||||
import { CoreProcess } from "./core-process.js";
|
||||
import { CoreProcess, staleBundleWarning } from "./core-process.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js";
|
||||
import { PreferencesStore } from "./preferences.js";
|
||||
import { Puppet } from "./puppet.js";
|
||||
@@ -49,6 +49,13 @@ async function connectLoop(): Promise<void> {
|
||||
puppet = new Puppet(client, audio);
|
||||
|
||||
client.onEvent(event => {
|
||||
// Speaking hints add to the reconciled set so the poller can
|
||||
// clear users the store no longer reports as speaking.
|
||||
if (event?.type === "VOICE_SPEAKING" && event.userId) {
|
||||
if (event.speaking) speakingUsers.add(String(event.userId));
|
||||
else speakingUsers.delete(String(event.userId));
|
||||
}
|
||||
|
||||
void puppet!.enqueueEvent(event);
|
||||
void audio.onEvent(event);
|
||||
webui.broadcast(event);
|
||||
@@ -56,6 +63,7 @@ async function connectLoop(): Promise<void> {
|
||||
|
||||
client.onClose(() => {
|
||||
console.warn("[daemon] core bridge disconnected");
|
||||
stopSpeakingPoll();
|
||||
coreConnected = false;
|
||||
core = undefined;
|
||||
puppet = undefined;
|
||||
@@ -66,7 +74,8 @@ async function connectLoop(): Promise<void> {
|
||||
coreConnected = true;
|
||||
console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`);
|
||||
webui.broadcast({ type: "CORE_STATUS", coreConnected: true });
|
||||
void autoConnectAudio(client);
|
||||
startSpeakingPoll();
|
||||
void autoConnectAudioWithRetry(client);
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
@@ -89,6 +98,69 @@ function requirePuppet(): Puppet {
|
||||
return puppet;
|
||||
}
|
||||
|
||||
/*
|
||||
* Discord's dispatcher only sends speaking-start events, so the UI and the
|
||||
* transcript attribution need the store's authoritative set. Poll it once a
|
||||
* second and synthesize the missing stop/start events. Dispatcher events feed
|
||||
* the same set so a user the store no longer reports is cleared.
|
||||
*/
|
||||
|
||||
let speakingTimer: ReturnType<typeof setInterval> | undefined;
|
||||
const speakingUsers = new Set<string>();
|
||||
|
||||
function emitCoreEvent(event: any): void {
|
||||
void puppet?.enqueueEvent(event);
|
||||
void audio.onEvent(event);
|
||||
webui.broadcast(event);
|
||||
}
|
||||
|
||||
async function pollSpeaking(): Promise<void> {
|
||||
if (!core) return;
|
||||
|
||||
try {
|
||||
const result = (await core.voiceSpeaking()) as { channelId?: string | null; userIds?: string[] };
|
||||
const next = new Set<string>(result?.channelId ? (result.userIds ?? []) : []);
|
||||
|
||||
for (const userId of next) {
|
||||
if (!speakingUsers.has(userId)) {
|
||||
emitCoreEvent({
|
||||
type: "VOICE_SPEAKING",
|
||||
userId,
|
||||
speaking: true,
|
||||
channelId: result.channelId ?? null
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const userId of speakingUsers) {
|
||||
if (!next.has(userId)) {
|
||||
emitCoreEvent({
|
||||
type: "VOICE_SPEAKING",
|
||||
userId,
|
||||
speaking: false,
|
||||
channelId: result?.channelId ?? null
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
speakingUsers.clear();
|
||||
for (const userId of next) speakingUsers.add(userId);
|
||||
} catch {
|
||||
// The close handler stops the poller.
|
||||
}
|
||||
}
|
||||
|
||||
function startSpeakingPoll(): void {
|
||||
stopSpeakingPoll();
|
||||
speakingTimer = setInterval(() => void pollSpeaking(), 1000);
|
||||
}
|
||||
|
||||
function stopSpeakingPoll(): void {
|
||||
if (speakingTimer) clearInterval(speakingTimer);
|
||||
speakingTimer = undefined;
|
||||
speakingUsers.clear();
|
||||
}
|
||||
|
||||
const coreProcess = new CoreProcess({ tokenFile });
|
||||
|
||||
const transcripts = new TranscriptStore(
|
||||
@@ -129,6 +201,7 @@ const audio = new AudioService({
|
||||
onTranscript: event => webui.broadcast(event)
|
||||
});
|
||||
|
||||
audio.setDuckEnabled(preferences.get().audio?.duck !== false);
|
||||
void audio.init();
|
||||
|
||||
/** Point the client at the virtual microphone and capture sink when present. */
|
||||
@@ -161,11 +234,25 @@ async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolea
|
||||
console.log("[audio] client devices set:", JSON.stringify(result));
|
||||
return { connected: true, message: "the client now uses the virtual devices" };
|
||||
} catch (error) {
|
||||
console.warn("[audio] device auto-connect failed:", (error as Error).message);
|
||||
return { connected: false, message: (error as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The bridge socket is up before the renderer finishes loading, so the first
|
||||
* device calls can fail with "core is not ready" style errors. Retry.
|
||||
*/
|
||||
async function autoConnectAudioWithRetry(client: CoreClient): Promise<void> {
|
||||
for (let attempt = 1; attempt <= 10; attempt++) {
|
||||
if (attempt > 1) await sleep(2500);
|
||||
|
||||
const result = await autoConnectAudio(client);
|
||||
if (result.connected) return;
|
||||
|
||||
console.warn(`[audio] device auto-connect attempt ${attempt}: ${result.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const auth = new AuthService(tokenFile, update => {
|
||||
webui.broadcast({ type: "AUTH_QR", update });
|
||||
|
||||
@@ -385,12 +472,36 @@ function buildServer(): McpServer {
|
||||
{
|
||||
description: "Move the client to another voice channel.",
|
||||
inputSchema: {
|
||||
channelId: z.string().describe("Voice channel ID")
|
||||
channelId: z.string().describe("Channel ID")
|
||||
}
|
||||
},
|
||||
async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_user_volume",
|
||||
{
|
||||
description:
|
||||
"Per-user playback volume (0 to 200, 100 is default) and the client's own output volume. A volume of 0 silences that user.",
|
||||
inputSchema: {
|
||||
userIds: z.array(z.string()).min(1).max(50).describe("User IDs to look up")
|
||||
}
|
||||
},
|
||||
async ({ userIds }) => jsonText(await requireCore().userVolume(userIds))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"set_user_volume",
|
||||
{
|
||||
description: "Set a user's playback volume. 0 silences the user; 100 is default; 200 is the maximum.",
|
||||
inputSchema: {
|
||||
userId: z.string().describe("User ID"),
|
||||
volume: z.number().int().min(0).max(200).describe("Volume from 0 to 200")
|
||||
}
|
||||
},
|
||||
async ({ userId, volume }) => jsonText(await pacedVoice(() => requireCore().userSetVolume(userId, volume)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_audio_status",
|
||||
{
|
||||
@@ -405,11 +516,58 @@ function buildServer(): McpServer {
|
||||
description: "Queue text to speech into the bot's virtual microphone.",
|
||||
inputSchema: {
|
||||
text: z.string().min(1).max(1000).describe("Text to speak"),
|
||||
provider: z.enum(["local", "api"]).optional().describe("TTS provider"),
|
||||
voice: z.string().optional().describe("Voice name for the provider")
|
||||
provider: z
|
||||
.enum(["local", "espeak", "command", "api"])
|
||||
.optional()
|
||||
.describe("TTS engine: command is the configured VESKSLOP_TTS_CMD engine"),
|
||||
voice: z.string().optional().describe("Voice name for the provider"),
|
||||
speed: z.number().min(0.5).max(2).optional().describe("Rate multiplier, 1.0 is natural"),
|
||||
pitch: z.number().min(-12).max(12).optional().describe("Pitch offset in semitones"),
|
||||
expression: z
|
||||
.number()
|
||||
.min(0)
|
||||
.max(100)
|
||||
.optional()
|
||||
.describe("Flextalk Expression profile value (Keith Bell, Kerry Watson)"),
|
||||
profile: z
|
||||
.record(z.string(), z.number())
|
||||
.optional()
|
||||
.describe("Flextalk profile overrides (pitch, rate, prominences, head_size, lowering)"),
|
||||
highpass: z.number().min(40).max(400).optional().describe("High-pass filter Hz"),
|
||||
echo: z.number().min(0).max(100).optional().describe("Mono echo amount"),
|
||||
words: z.boolean().optional().describe("Synthesize each word and merge with gaps"),
|
||||
wordGapMs: z.number().min(0).max(500).optional().describe("Silence between words in ms")
|
||||
}
|
||||
},
|
||||
async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice }))
|
||||
async ({ text, provider, voice, speed, pitch, expression, profile, highpass, echo, words, wordGapMs }) =>
|
||||
jsonText(
|
||||
await audio.speak(text, {
|
||||
provider,
|
||||
voice,
|
||||
speed,
|
||||
pitch,
|
||||
expression,
|
||||
profile,
|
||||
highpass,
|
||||
echo,
|
||||
words,
|
||||
wordGapMs
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"start_tts_engine",
|
||||
{
|
||||
description: "Start (warm up) the TTS engine so the first spoken line is fast.",
|
||||
inputSchema: {
|
||||
engine: z
|
||||
.enum(["local", "espeak", "command", "api"])
|
||||
.optional()
|
||||
.describe("Engine to start. Defaults to the configured one.")
|
||||
}
|
||||
},
|
||||
async ({ engine }) => jsonText(await audio.warmTts(engine))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
@@ -620,8 +778,24 @@ app.delete("/mcp", (_req, res) => {
|
||||
);
|
||||
});
|
||||
|
||||
app.get("/health", (_req, res) => {
|
||||
res.json({ ok: true, coreConnected, socketPath: coreSocketPath, ownerId: puppet?.owner, queries: getQueryCount() });
|
||||
app.get("/health", async (_req, res) => {
|
||||
let selfId = puppet?.self;
|
||||
if (!selfId && core) {
|
||||
try {
|
||||
selfId = await core.getSelfId();
|
||||
} catch {
|
||||
// The core may not be ready yet.
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
coreConnected,
|
||||
socketPath: coreSocketPath,
|
||||
ownerId: puppet?.owner,
|
||||
selfId,
|
||||
queries: getQueryCount()
|
||||
});
|
||||
});
|
||||
|
||||
app.listen(PORT, HOST, (error?: Error) => {
|
||||
@@ -630,9 +804,13 @@ app.listen(PORT, HOST, (error?: Error) => {
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`[daemon] MCP listening on http://${HOST}:${PORT}/mcp`);
|
||||
|
||||
const stale = staleBundleWarning();
|
||||
if (stale) console.warn(`[daemon] ${stale}`);
|
||||
});
|
||||
|
||||
function shutdown() {
|
||||
stopSpeakingPoll();
|
||||
core?.close();
|
||||
void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0));
|
||||
}
|
||||
|
||||
@@ -196,10 +196,22 @@ export class CoreClient {
|
||||
return this.request("voice.users", { userIds });
|
||||
}
|
||||
|
||||
async voiceSpeaking() {
|
||||
return this.request("voice.speaking", {});
|
||||
}
|
||||
|
||||
async userStatus(userIds: string[]) {
|
||||
return this.request("user.status", { userIds });
|
||||
}
|
||||
|
||||
async userVolume(userIds: string[]) {
|
||||
return this.request("user.volume", { userIds });
|
||||
}
|
||||
|
||||
async userSetVolume(userId: string, volume: number) {
|
||||
return this.request("user.setVolume", { userId, volume });
|
||||
}
|
||||
|
||||
async voiceDevices() {
|
||||
return this.request("voice.devices");
|
||||
}
|
||||
|
||||
+32
-2
@@ -9,7 +9,7 @@
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
@@ -17,7 +17,37 @@ const ROOT = fileURLToPath(new URL("../..", import.meta.url));
|
||||
|
||||
const DEFAULT_ARGS = "--no-sandbox --disable-gpu --disable-dev-shm-usage";
|
||||
const XVFB_ARGS = "-screen 0 1280x720x24 -nolisten tcp";
|
||||
const MAX_OUTPUT_LINES = 20;
|
||||
const MAX_OUTPUT_LINES = 200;
|
||||
|
||||
/**
|
||||
* Warn when the built client bundle predates the core sources. Without this, a
|
||||
* new daemon paired with a stale client reports UNKNOWN_ACTION for new actions.
|
||||
*/
|
||||
export function staleBundleWarning(): string | null {
|
||||
const sources = [
|
||||
"src/renderer/puppet/actions.ts",
|
||||
"src/renderer/puppet/events.ts",
|
||||
"src/shared/puppetProtocol.ts",
|
||||
"src/main/puppetBridge.ts"
|
||||
];
|
||||
const bundles = ["dist/js/renderer.js", "dist/js/main.js"];
|
||||
|
||||
try {
|
||||
let newest = 0;
|
||||
for (const source of sources) newest = Math.max(newest, statSync(join(ROOT, source)).mtimeMs);
|
||||
|
||||
let oldest = Number.POSITIVE_INFINITY;
|
||||
for (const bundle of bundles) oldest = Math.min(oldest, statSync(join(ROOT, bundle)).mtimeMs);
|
||||
|
||||
if (oldest < newest) {
|
||||
return "the built client bundle is older than the core sources; run `pnpm build` and restart the client";
|
||||
}
|
||||
} catch {
|
||||
// A missing bundle is reported by the client start itself.
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface CoreExit {
|
||||
code: number | null;
|
||||
|
||||
@@ -100,6 +100,8 @@ const FAKE_HISTORY = Array.from({ length: 8 }, (_, index) => ({
|
||||
const MOCK_VOICE_CHANNEL = "200000000000000002";
|
||||
let botVoiceChannel: string | null = MOCK_VOICE_CHANNEL;
|
||||
let ownerVoiceChannel: string | null = MOCK_VOICE_CHANNEL;
|
||||
let outputVolume: number | null = 100;
|
||||
const userVolumes: Record<string, number> = {};
|
||||
|
||||
const STATE_FAIL_MS = Number(process.env.VESKSLOP_MOCK_STATE_FAIL_MS ?? 0);
|
||||
const startedAt = Date.now();
|
||||
@@ -165,6 +167,8 @@ function handle(action: string | undefined, params: any) {
|
||||
members: id === "999" ? mockVoiceMembers() : []
|
||||
}))
|
||||
};
|
||||
case "voice.speaking":
|
||||
return { channelId: botVoiceChannel, userIds: [] };
|
||||
case "user.status":
|
||||
return {
|
||||
users: (params?.userIds ?? []).map((id: string) => ({
|
||||
@@ -172,6 +176,20 @@ function handle(action: string | undefined, params: any) {
|
||||
status: id === "999" ? "online" : id === "177117296140091393" ? "idle" : "offline"
|
||||
}))
|
||||
};
|
||||
case "user.volume":
|
||||
return {
|
||||
outputVolume: outputVolume,
|
||||
users: (params?.userIds ?? []).map((id: string) => ({
|
||||
id,
|
||||
volume: userVolumes[id] ?? 100
|
||||
}))
|
||||
};
|
||||
case "user.setVolume": {
|
||||
const userId = String(params?.userId);
|
||||
const volume = Number(params?.volume);
|
||||
userVolumes[userId] = volume;
|
||||
return { userId, volume, outputVolume };
|
||||
}
|
||||
case "voice.setDevices":
|
||||
return { inputDeviceId: params?.inputDeviceId ?? null, outputDeviceId: params?.outputDeviceId ?? null };
|
||||
case "user.search": {
|
||||
|
||||
+145
-6
@@ -9,9 +9,61 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
export interface TtsPreferences {
|
||||
provider: "local" | "api" | null;
|
||||
provider: "local" | "espeak" | "command" | "api" | null;
|
||||
language: string | null;
|
||||
voice: string | null;
|
||||
/** Speech-rate multiplier, 1.0 is the engine's natural rate. Null uses the engine default. */
|
||||
speed: number | null;
|
||||
/** Pitch offset in semitones, 0 is the engine's natural pitch. Null uses the engine default. */
|
||||
pitch: number | null;
|
||||
/** Flextalk Expression profile value, 0 to 100. Null keeps the engine's registry default. */
|
||||
expression: number | null;
|
||||
/** Flextalk profile overrides (only the keys the user changed). */
|
||||
profile: Record<string, number>;
|
||||
/** Five saved profile slots; null means empty. `name` labels the slot. */
|
||||
profilePresets: Array<Record<string, number | string> | null>;
|
||||
/** High-pass filter Hz, 40-400. Null uses the engine default (40). */
|
||||
highpass: number | null;
|
||||
/** Mono echo amount, 0-100. Null or 0 disables. */
|
||||
echo: number | null;
|
||||
/** Synthesize each word and merge with silence gaps (Flextalk only). */
|
||||
words: boolean;
|
||||
/** Silence between words in milliseconds, 0-500. */
|
||||
wordGapMs: number | null;
|
||||
}
|
||||
|
||||
const PROFILE_RANGES: Record<string, [number, number]> = {
|
||||
pitch: [0, 255],
|
||||
rate: [0, 100],
|
||||
volume: [0, 65535],
|
||||
expression: [0, 100],
|
||||
prominences: [0, 100],
|
||||
head_size: [0, 100],
|
||||
lowering: [0, 100]
|
||||
};
|
||||
|
||||
function normalizeProfile(value: unknown): Record<string, number> {
|
||||
const profile: Record<string, number> = {};
|
||||
if (!value || typeof value !== "object") return profile;
|
||||
|
||||
for (const [key, range] of Object.entries(PROFILE_RANGES)) {
|
||||
const raw = (value as Record<string, unknown>)[key];
|
||||
if (isUnset(raw)) continue;
|
||||
const number = Number(raw);
|
||||
if (!Number.isFinite(number) || number < range[0] || number > range[1]) continue;
|
||||
profile[key] = Math.round(number);
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
function normalizePreset(value: unknown): Record<string, number | string> | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
|
||||
const preset: Record<string, number | string> = normalizeProfile(value);
|
||||
const name = (value as Record<string, unknown>).name;
|
||||
if (typeof name === "string" && name.trim()) preset.name = name.trim().slice(0, 40);
|
||||
return preset;
|
||||
}
|
||||
|
||||
export interface FavouritesPreferences {
|
||||
@@ -19,16 +71,36 @@ export interface FavouritesPreferences {
|
||||
channels: string[];
|
||||
}
|
||||
|
||||
export interface AudioPreferences {
|
||||
/** Duck the TTS while someone else speaks. */
|
||||
duck: boolean;
|
||||
}
|
||||
|
||||
export interface WebPreferences {
|
||||
/** null means every guild is visible. An empty array hides all. */
|
||||
visibleGuilds: string[] | null;
|
||||
tts: TtsPreferences;
|
||||
audio: AudioPreferences;
|
||||
favourites: FavouritesPreferences;
|
||||
}
|
||||
|
||||
const DEFAULTS: WebPreferences = {
|
||||
visibleGuilds: null,
|
||||
tts: { provider: null, language: null, voice: null },
|
||||
tts: {
|
||||
provider: null,
|
||||
language: null,
|
||||
voice: null,
|
||||
speed: null,
|
||||
pitch: null,
|
||||
expression: null,
|
||||
profile: {},
|
||||
profilePresets: [null, null, null, null, null],
|
||||
highpass: null,
|
||||
echo: null,
|
||||
words: false,
|
||||
wordGapMs: null
|
||||
},
|
||||
audio: { duck: true },
|
||||
favourites: { users: [], channels: [] }
|
||||
};
|
||||
|
||||
@@ -37,6 +109,42 @@ export interface PreferencesOptions {
|
||||
legacyFile?: string;
|
||||
}
|
||||
|
||||
function normalizeProvider(value: unknown): TtsPreferences["provider"] {
|
||||
return value === "api" || value === "local" || value === "espeak" || value === "command" ? value : null;
|
||||
}
|
||||
|
||||
function isUnset(value: unknown): boolean {
|
||||
return value === null || value === undefined || value === "";
|
||||
}
|
||||
|
||||
function normalizeSpeed(value: unknown): number | null {
|
||||
if (isUnset(value)) return null;
|
||||
const speed = Number(value);
|
||||
if (!Number.isFinite(speed) || speed < 0.5 || speed > 2) return null;
|
||||
return Math.round(speed * 100) / 100;
|
||||
}
|
||||
|
||||
function normalizePitch(value: unknown): number | null {
|
||||
if (isUnset(value)) return null;
|
||||
const pitch = Number(value);
|
||||
if (!Number.isFinite(pitch) || pitch < -12 || pitch > 12) return null;
|
||||
return Math.round(pitch * 10) / 10;
|
||||
}
|
||||
|
||||
function normalizeExpression(value: unknown): number | null {
|
||||
if (isUnset(value)) return null;
|
||||
const expression = Number(value);
|
||||
if (!Number.isFinite(expression) || expression < 0 || expression > 100) return null;
|
||||
return Math.round(expression);
|
||||
}
|
||||
|
||||
function normalizeRange(value: unknown, min: number, max: number): number | null {
|
||||
if (isUnset(value)) return null;
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < min || number > max) return null;
|
||||
return Math.round(number);
|
||||
}
|
||||
|
||||
export class PreferencesStore {
|
||||
private preferences: WebPreferences = structuredClone(DEFAULTS);
|
||||
|
||||
@@ -68,9 +176,26 @@ export class PreferencesStore {
|
||||
|
||||
if (patch.tts !== undefined) {
|
||||
this.preferences.tts = {
|
||||
provider: patch.tts?.provider === "api" ? "api" : patch.tts?.provider === "local" ? "local" : null,
|
||||
provider: normalizeProvider(patch.tts?.provider),
|
||||
language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null,
|
||||
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null
|
||||
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null,
|
||||
speed: normalizeSpeed(patch.tts?.speed),
|
||||
pitch: normalizePitch(patch.tts?.pitch),
|
||||
expression: normalizeExpression(patch.tts?.expression),
|
||||
profile: normalizeProfile(patch.tts?.profile),
|
||||
profilePresets: Array.isArray(patch.tts?.profilePresets)
|
||||
? Array.from({ length: 5 }, (_, index) => normalizePreset(patch.tts?.profilePresets?.[index]))
|
||||
: this.preferences.tts.profilePresets,
|
||||
highpass: normalizeRange(patch.tts?.highpass, 40, 400),
|
||||
echo: normalizeRange(patch.tts?.echo, 0, 100),
|
||||
words: patch.tts?.words === true,
|
||||
wordGapMs: normalizeRange(patch.tts?.wordGapMs, 0, 500)
|
||||
};
|
||||
}
|
||||
|
||||
if (patch.audio !== undefined) {
|
||||
this.preferences.audio = {
|
||||
duck: patch.audio?.duck !== false
|
||||
};
|
||||
}
|
||||
|
||||
@@ -112,9 +237,23 @@ export class PreferencesStore {
|
||||
? [...new Set<string>(raw.visibleGuilds.map((id: unknown) => String(id)))]
|
||||
: null,
|
||||
tts: {
|
||||
provider: provider === "api" || provider === "local" ? provider : null,
|
||||
provider: normalizeProvider(provider),
|
||||
language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null,
|
||||
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null
|
||||
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null,
|
||||
speed: normalizeSpeed(raw?.tts?.speed),
|
||||
pitch: normalizePitch(raw?.tts?.pitch),
|
||||
expression: normalizeExpression(raw?.tts?.expression),
|
||||
profile: normalizeProfile(raw?.tts?.profile),
|
||||
profilePresets: Array.isArray(raw?.tts?.profilePresets)
|
||||
? Array.from({ length: 5 }, (_, index) => normalizePreset(raw.tts.profilePresets[index]))
|
||||
: [null, null, null, null, null],
|
||||
highpass: normalizeRange(raw?.tts?.highpass, 40, 400),
|
||||
echo: normalizeRange(raw?.tts?.echo, 0, 100),
|
||||
words: raw?.tts?.words === true,
|
||||
wordGapMs: normalizeRange(raw?.tts?.wordGapMs, 0, 500)
|
||||
},
|
||||
audio: {
|
||||
duck: raw?.audio?.duck !== false
|
||||
},
|
||||
favourites: {
|
||||
users: Array.isArray(raw?.favourites?.users)
|
||||
|
||||
@@ -157,6 +157,11 @@ export class Puppet {
|
||||
private selfId?: string;
|
||||
private queue: Promise<void> = Promise.resolve();
|
||||
|
||||
/** The logged-in account's user ID, once the core has been asked for it. */
|
||||
get self(): string | undefined {
|
||||
return this.selfId;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly core: PuppetTransport,
|
||||
private readonly audio?: PuppetAudio
|
||||
|
||||
+229
-15
@@ -7,24 +7,51 @@
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
export class TtsError extends Error {}
|
||||
|
||||
export type TtsProvider = "local" | "api";
|
||||
export type TtsProvider = "local" | "espeak" | "command" | "api";
|
||||
|
||||
export interface TtsOptions {
|
||||
provider?: TtsProvider;
|
||||
voice?: string;
|
||||
/** Speech-rate multiplier; the engine maps it to its own units. */
|
||||
speed?: number;
|
||||
/** Pitch offset in semitones; the engine maps it to its own units. */
|
||||
pitch?: number;
|
||||
/** Flextalk Expression profile value, 0 to 100. */
|
||||
expression?: number;
|
||||
/** Flextalk profile overrides: pitch, rate, volume, expression, prominences, head_size, lowering. */
|
||||
profile?: Record<string, number>;
|
||||
/** High-pass filter Hz, 40 to 400. */
|
||||
highpass?: number;
|
||||
/** Mono echo amount, 0 to 100. */
|
||||
echo?: number;
|
||||
/** Synthesize each word and merge with silence gaps. */
|
||||
words?: boolean;
|
||||
/** Silence between words in milliseconds, 0 to 500. */
|
||||
wordGapMs?: number;
|
||||
}
|
||||
|
||||
export interface TtsEngine {
|
||||
id: "espeak" | "command" | "api";
|
||||
name: string;
|
||||
available: boolean;
|
||||
/** false when the engine has no language dimension (single-language command engines). */
|
||||
language: boolean;
|
||||
}
|
||||
|
||||
export interface TtsStatus {
|
||||
local: boolean;
|
||||
api: boolean;
|
||||
espeak: boolean;
|
||||
command: boolean;
|
||||
defaultProvider: TtsProvider;
|
||||
engine: string | null;
|
||||
engines: TtsEngine[];
|
||||
}
|
||||
|
||||
export interface TtsVoice {
|
||||
@@ -32,8 +59,19 @@ export interface TtsVoice {
|
||||
name: string;
|
||||
language: string;
|
||||
gender: "male" | "female" | null;
|
||||
/** false when the engine ignores pitch for this voice (for example Flextalk). */
|
||||
supportsPitch?: boolean;
|
||||
/** true when the voice has a Flextalk-style Expression profile parameter. */
|
||||
supportsExpression?: boolean;
|
||||
/** Registry default for the expression parameter, when known. */
|
||||
expressionDefault?: number;
|
||||
/** Per-voice Flextalk profile defaults (pitch, rate, prominences, head_size, lowering, expression). */
|
||||
profileDefaults?: Record<string, number>;
|
||||
}
|
||||
|
||||
/** espeak-ng voice variants, appended to every base voice. */
|
||||
const ESPEAK_VARIANTS = ["m1", "m2", "m3", "m4", "m5", "m6", "m7", "f1", "f2", "f3", "f4", "f5", "croak", "whisper"];
|
||||
|
||||
interface CommandResult {
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
@@ -81,11 +119,62 @@ function parseVoices(output: string): TtsVoice[] {
|
||||
return voices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Command-mode voice list. Entries are separated by commas or newlines. Each entry is
|
||||
* `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or
|
||||
* `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch`
|
||||
* (the engine ignores pitch) and `expression` / `expression=NN` (Flextalk profile).
|
||||
*/
|
||||
function parseEngineVoices(raw: string): TtsVoice[] {
|
||||
return raw
|
||||
.split(/[,\n]+/)
|
||||
.map(part => part.trim())
|
||||
.filter(Boolean)
|
||||
.map((entry): TtsVoice => {
|
||||
const [id, name, gender, language, flags] = entry.split("|").map(part => part?.trim());
|
||||
const flagList = flags?.split("+").map(flag => flag.trim()) ?? [];
|
||||
const expression = flagList.find(flag => flag === "expression" || flag.startsWith("expression="));
|
||||
const defaults = flagList.find(flag => flag.startsWith("defaults="));
|
||||
|
||||
let profileDefaults: Record<string, number> | undefined;
|
||||
if (defaults) {
|
||||
const parts = defaults.slice("defaults=".length).split(":");
|
||||
profileDefaults = {};
|
||||
for (let index = 0; index + 1 < parts.length; index += 2) {
|
||||
const value = Number(parts[index + 1]);
|
||||
if (Number.isFinite(value)) profileDefaults[parts[index]] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const expressionDefault = profileDefaults?.expression ??
|
||||
(expression?.includes("=") ? Number(expression.split("=")[1]) : undefined);
|
||||
|
||||
return {
|
||||
id,
|
||||
name: name || id,
|
||||
language: language || "en",
|
||||
gender: gender === "male" || gender === "female" ? gender : null,
|
||||
supportsPitch: !flagList.includes("no-pitch"),
|
||||
supportsExpression: !!expression || !!profileDefaults,
|
||||
...(Number.isFinite(expressionDefault) ? { expressionDefault } : {}),
|
||||
...(profileDefaults ? { profileDefaults } : {})
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Serialize a profile object to the wrapper's `key=value,key=value` form. */
|
||||
function serializeProfile(profile?: Record<string, number>): string {
|
||||
if (!profile) return "";
|
||||
return Object.entries(profile)
|
||||
.map(([key, value]) => `${key}=${value}`)
|
||||
.join(",");
|
||||
}
|
||||
|
||||
export class Tts {
|
||||
private readonly dir: string;
|
||||
private counter = 0;
|
||||
private espeak = false;
|
||||
private voiceCache?: TtsVoice[];
|
||||
private readonly voiceCache = new Map<string, TtsVoice[]>();
|
||||
|
||||
constructor(cacheDir?: string) {
|
||||
const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache");
|
||||
@@ -109,39 +198,128 @@ export class Tts {
|
||||
return this.dir;
|
||||
}
|
||||
|
||||
/** Display name for the configured command engine. */
|
||||
get commandName(): string {
|
||||
return process.env.VESKSLOP_TTS_NAME ?? "custom engine";
|
||||
}
|
||||
|
||||
async detect(): Promise<void> {
|
||||
const result = await run("espeak-ng", ["--version"]);
|
||||
this.espeak = result.code === 0;
|
||||
}
|
||||
|
||||
engines(): TtsEngine[] {
|
||||
return [
|
||||
{ id: "espeak", name: "espeak-ng", available: this.espeak, language: true },
|
||||
{ id: "command", name: this.commandName, available: !!this.localCommand, language: false },
|
||||
{ id: "api", name: "HTTP API", available: !!this.apiUrl, language: true }
|
||||
];
|
||||
}
|
||||
|
||||
private autoProvider(): TtsProvider {
|
||||
if (this.localCommand) return "command";
|
||||
if (this.espeak) return "espeak";
|
||||
return this.apiUrl ? "api" : "local";
|
||||
}
|
||||
|
||||
status(): TtsStatus {
|
||||
const local = !!this.localCommand || this.espeak;
|
||||
const override = process.env.VESKSLOP_TTS_PROVIDER;
|
||||
const allowed = new Set<TtsProvider>(["local", "espeak", "command", "api"]);
|
||||
|
||||
return {
|
||||
local,
|
||||
api: !!this.apiUrl,
|
||||
defaultProvider: override === "local" || override === "api" ? override : local ? "local" : "api",
|
||||
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null
|
||||
espeak: this.espeak,
|
||||
command: !!this.localCommand,
|
||||
defaultProvider: allowed.has(override as TtsProvider) ? (override as TtsProvider) : this.autoProvider(),
|
||||
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null,
|
||||
engines: this.engines()
|
||||
};
|
||||
}
|
||||
|
||||
async voices(): Promise<TtsVoice[]> {
|
||||
if (this.voiceCache) return this.voiceCache;
|
||||
if (!this.espeak) return (this.voiceCache = []);
|
||||
async voices(engine?: TtsProvider): Promise<TtsVoice[]> {
|
||||
const key = engine ?? this.autoProvider();
|
||||
const cached = this.voiceCache.get(key);
|
||||
if (cached) return cached;
|
||||
|
||||
let voices: TtsVoice[] = [];
|
||||
|
||||
if (key === "command" || (key === "local" && this.localCommand)) {
|
||||
const declared = process.env.VESKSLOP_TTS_VOICES;
|
||||
voices = declared ? parseEngineVoices(declared) : [];
|
||||
} else if (key === "espeak" || key === "local") {
|
||||
voices = await this.espeakVoices();
|
||||
}
|
||||
|
||||
this.voiceCache.set(key, voices);
|
||||
return voices;
|
||||
}
|
||||
|
||||
private async espeakVoices(): Promise<TtsVoice[]> {
|
||||
if (!this.espeak) return [];
|
||||
|
||||
const result = await run("espeak-ng", ["--voices"]);
|
||||
this.voiceCache = result.code === 0 ? parseVoices(result.stdout) : [];
|
||||
return this.voiceCache;
|
||||
const base = result.code === 0 ? parseVoices(result.stdout) : [];
|
||||
|
||||
return base.flatMap((voice): TtsVoice[] => [
|
||||
voice,
|
||||
...ESPEAK_VARIANTS.map(
|
||||
(variant): TtsVoice => ({
|
||||
id: `${voice.id}+${variant}`,
|
||||
name: `${voice.name} (${variant})`,
|
||||
language: voice.language,
|
||||
gender: variant.startsWith("m") ? "male" : variant.startsWith("f") ? "female" : null
|
||||
})
|
||||
)
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the engine so the first real line is fast: run one throwaway synthesis
|
||||
* (the command wrapper starts Xvfb, Wine, and the host), then delete the file.
|
||||
*/
|
||||
async warm(engine?: TtsProvider): Promise<{ engine: TtsProvider; ms: number; warmed: boolean }> {
|
||||
const key = engine ?? this.autoProvider();
|
||||
const start = Date.now();
|
||||
|
||||
if (key === "command" && !this.localCommand) {
|
||||
throw new TtsError("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
|
||||
const usesCommand = key === "command" || (key === "local" && !!this.localCommand);
|
||||
if (!usesCommand) {
|
||||
// espeak starts instantly and the HTTP provider is expected to stay warm.
|
||||
return { engine: key, ms: 0, warmed: false };
|
||||
}
|
||||
|
||||
const voice = process.env.VESKSLOP_TTS_VOICE || undefined;
|
||||
const file = await this.viaLocalCommand("The quick brown fox jumps over the lazy dog.", {
|
||||
provider: key,
|
||||
voice
|
||||
});
|
||||
|
||||
try {
|
||||
unlinkSync(file);
|
||||
} catch {
|
||||
// best effort
|
||||
}
|
||||
|
||||
return { engine: key, ms: Date.now() - start, warmed: true };
|
||||
}
|
||||
|
||||
async synthesize(text: string, options: TtsOptions = {}): Promise<string> {
|
||||
const provider = options.provider ?? this.status().defaultProvider;
|
||||
return provider === "api" ? this.viaApi(text, options) : this.viaLocalCommand(text, options);
|
||||
|
||||
if (provider === "api") return this.viaApi(text, options);
|
||||
if (provider === "espeak") return this.viaLocalCommand(text, options, true);
|
||||
if (provider === "command") return this.viaLocalCommand(text, options, false);
|
||||
|
||||
return this.viaLocalCommand(text, options);
|
||||
}
|
||||
|
||||
private async viaLocalCommand(text: string, options: TtsOptions): Promise<string> {
|
||||
const command = this.localCommand;
|
||||
private async viaLocalCommand(text: string, options: TtsOptions, forceEspeak = false): Promise<string> {
|
||||
const command = forceEspeak ? undefined : this.localCommand;
|
||||
if (!command && !this.espeak) {
|
||||
throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
@@ -162,6 +340,14 @@ export class Tts {
|
||||
.replace("{out}", out)
|
||||
.replace("{text_file}", textFile)
|
||||
.replace("{voice}", options.voice ?? "")
|
||||
.replace("{speed}", options.speed !== undefined ? String(options.speed) : "")
|
||||
.replace("{pitch}", options.pitch !== undefined ? String(options.pitch) : "")
|
||||
.replace("{expression}", options.expression !== undefined ? String(options.expression) : "")
|
||||
.replace("{profile}", serializeProfile(options.profile))
|
||||
.replace("{highpass}", options.highpass !== undefined ? String(options.highpass) : "")
|
||||
.replace("{echo}", options.echo !== undefined ? String(options.echo) : "")
|
||||
.replace("{words}", options.words ? "1" : "")
|
||||
.replace("{word_gap_ms}", options.wordGapMs !== undefined ? String(options.wordGapMs) : "")
|
||||
)
|
||||
: ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile];
|
||||
|
||||
@@ -169,7 +355,24 @@ export class Tts {
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(args[0], args.slice(1), {
|
||||
env: { ...process.env, VESKSLOP_TTS_VOICE: options.voice ?? "" },
|
||||
env: {
|
||||
...process.env,
|
||||
VESKSLOP_TTS_VOICE: options.voice ?? "",
|
||||
...(options.speed !== undefined ? { VESKSLOP_TTS_SPEED: String(options.speed) } : {}),
|
||||
...(options.pitch !== undefined ? { VESKSLOP_TTS_PITCH: String(options.pitch) } : {}),
|
||||
...(options.expression !== undefined
|
||||
? { VESKSLOP_TTS_EXPRESSION: String(options.expression) }
|
||||
: {}),
|
||||
...(options.profile && Object.keys(options.profile).length
|
||||
? { VESKSLOP_TTS_PROFILE: serializeProfile(options.profile) }
|
||||
: {}),
|
||||
...(options.highpass !== undefined ? { VESKSLOP_TTS_HIGHPASS: String(options.highpass) } : {}),
|
||||
...(options.echo !== undefined ? { VESKSLOP_TTS_ECHO: String(options.echo) } : {}),
|
||||
...(options.words ? { VESKSLOP_TTS_WORDS: "1" } : {}),
|
||||
...(options.wordGapMs !== undefined
|
||||
? { VESKSLOP_TTS_WORD_GAP_MS: String(options.wordGapMs) }
|
||||
: {})
|
||||
},
|
||||
stdio: ["ignore", "ignore", "pipe"]
|
||||
});
|
||||
let stderr = "";
|
||||
@@ -197,7 +400,18 @@ export class Tts {
|
||||
? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` }
|
||||
: {})
|
||||
},
|
||||
body: JSON.stringify({ text, ...(options.voice ? { voice: options.voice } : {}) })
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
...(options.voice ? { voice: options.voice } : {}),
|
||||
...(options.speed !== undefined ? { speed: options.speed } : {}),
|
||||
...(options.pitch !== undefined ? { pitch: options.pitch } : {}),
|
||||
...(options.expression !== undefined ? { expression: options.expression } : {}),
|
||||
...(options.profile && Object.keys(options.profile).length ? { profile: options.profile } : {}),
|
||||
...(options.highpass !== undefined ? { highpass: options.highpass } : {}),
|
||||
...(options.echo !== undefined ? { echo: options.echo } : {}),
|
||||
...(options.words ? { words: true } : {}),
|
||||
...(options.wordGapMs !== undefined ? { wordGapMs: options.wordGapMs } : {})
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`);
|
||||
|
||||
+103
-8
@@ -25,7 +25,7 @@ import type { PreferencesStore, WebPreferences } from "./preferences.js";
|
||||
import type { Puppet } from "./puppet.js";
|
||||
import { ensureTls } from "./tls.js";
|
||||
import type { TranscriptStore } from "./transcript.js";
|
||||
import type { TtsVoice } from "./tts.js";
|
||||
import type { TtsProvider, TtsVoice } from "./tts.js";
|
||||
|
||||
const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"];
|
||||
const LANGUAGE_NAMES: Record<string, string> = {
|
||||
@@ -401,7 +401,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
app.post(
|
||||
"/api/preferences",
|
||||
wrap(async (req, res) => {
|
||||
const { visibleGuilds, tts, favourites } = req.body ?? {};
|
||||
const { visibleGuilds, tts, audio, favourites } = req.body ?? {};
|
||||
|
||||
if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
|
||||
throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
|
||||
@@ -414,7 +414,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
patch.tts = {
|
||||
provider: tts?.provider ?? null,
|
||||
language: tts?.language ?? null,
|
||||
voice: tts?.voice ?? null
|
||||
voice: tts?.voice ?? null,
|
||||
speed: tts?.speed ?? null,
|
||||
pitch: tts?.pitch ?? null,
|
||||
expression: tts?.expression ?? null,
|
||||
profile: tts?.profile ?? {},
|
||||
profilePresets: Array.isArray(tts?.profilePresets)
|
||||
? tts.profilePresets.slice(0, 5)
|
||||
: deps.preferences.get().tts.profilePresets,
|
||||
highpass: tts?.highpass ?? null,
|
||||
echo: tts?.echo ?? null,
|
||||
words: tts?.words === true,
|
||||
wordGapMs: tts?.wordGapMs ?? null
|
||||
};
|
||||
}
|
||||
|
||||
@@ -425,6 +436,11 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
};
|
||||
}
|
||||
|
||||
if (audio !== undefined) {
|
||||
patch.audio = { duck: audio?.duck !== false };
|
||||
deps.audio.setDuckEnabled(patch.audio.duck);
|
||||
}
|
||||
|
||||
res.json({ ok: true, result: deps.preferences.update(patch) });
|
||||
})
|
||||
);
|
||||
@@ -486,6 +502,41 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/voice/volume",
|
||||
wrap(async (req, res) => {
|
||||
let userIds = req.query.userIds
|
||||
? String(req.query.userIds)
|
||||
.split(",")
|
||||
.map(id => id.trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
|
||||
if (!userIds.length) {
|
||||
const voice = (await requireCore().voiceGet()) as { members?: Array<{ id: string }> } | undefined;
|
||||
userIds = (voice?.members ?? []).map(member => member.id);
|
||||
}
|
||||
|
||||
if (!userIds.length) {
|
||||
res.json({ ok: true, result: { outputVolume: null, users: [] } });
|
||||
return;
|
||||
}
|
||||
|
||||
res.json({ ok: true, result: await requireCore().userVolume(userIds) });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/voice/volume",
|
||||
wrap(async (req, res) => {
|
||||
const userId = String(req.body?.userId ?? "");
|
||||
const volume = Number(req.body?.volume);
|
||||
const result = await pacedVoice(() => requireCore().userSetVolume(userId, volume));
|
||||
deps.getPuppet()?.appendAction({ type: "voice", action: "set-volume", userId, volume });
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/voice/join",
|
||||
wrap(async (req, res) => {
|
||||
@@ -526,8 +577,28 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
|
||||
app.get(
|
||||
"/api/audio/tts/voices",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) });
|
||||
wrap(async (req, res) => {
|
||||
const engine = req.query.engine ? String(req.query.engine) : undefined;
|
||||
res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices(engine as TtsProvider | undefined)) });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/tts/warm",
|
||||
wrap(async (req, res) => {
|
||||
const requested = req.body?.engine;
|
||||
const engine = ["local", "espeak", "command", "api"].includes(requested)
|
||||
? (requested as TtsProvider)
|
||||
: undefined;
|
||||
|
||||
const result = await deps.audio.warmTts(engine);
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "audio",
|
||||
action: "warm-engine",
|
||||
engine: result.engine,
|
||||
ms: result.ms
|
||||
});
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
@@ -568,11 +639,35 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
const text = String(req.body?.text ?? "");
|
||||
const prefs = deps.preferences.get();
|
||||
|
||||
const provider =
|
||||
req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider;
|
||||
const requested = req.body?.provider;
|
||||
const provider = ["api", "local", "espeak", "command"].includes(requested)
|
||||
? requested
|
||||
: prefs.tts.provider;
|
||||
const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
|
||||
const speed = req.body?.speed !== undefined ? Number(req.body.speed) : prefs.tts.speed ?? undefined;
|
||||
const pitch = req.body?.pitch !== undefined ? Number(req.body.pitch) : prefs.tts.pitch ?? undefined;
|
||||
const expression =
|
||||
req.body?.expression !== undefined ? Number(req.body.expression) : prefs.tts.expression ?? undefined;
|
||||
const profile = req.body?.profile && typeof req.body.profile === "object" ? req.body.profile : prefs.tts.profile;
|
||||
const highpass =
|
||||
req.body?.highpass !== undefined ? Number(req.body.highpass) : prefs.tts.highpass ?? undefined;
|
||||
const echo = req.body?.echo !== undefined ? Number(req.body.echo) : prefs.tts.echo ?? undefined;
|
||||
const words = req.body?.words !== undefined ? req.body.words === true : prefs.tts.words;
|
||||
const wordGapMs =
|
||||
req.body?.wordGapMs !== undefined ? Number(req.body.wordGapMs) : prefs.tts.wordGapMs ?? undefined;
|
||||
|
||||
const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice });
|
||||
const result = await deps.audio.speak(text, {
|
||||
provider: provider ?? undefined,
|
||||
voice,
|
||||
...(Number.isFinite(speed) ? { speed } : {}),
|
||||
...(Number.isFinite(pitch) ? { pitch } : {}),
|
||||
...(Number.isFinite(expression) ? { expression } : {}),
|
||||
...(profile && Object.keys(profile).length ? { profile } : {}),
|
||||
...(Number.isFinite(highpass) ? { highpass } : {}),
|
||||
...(Number.isFinite(echo) ? { echo } : {}),
|
||||
...(words ? { words: true } : {}),
|
||||
...(Number.isFinite(wordGapMs) ? { wordGapMs } : {})
|
||||
});
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "audio",
|
||||
action: "speak",
|
||||
|
||||
Reference in New Issue
Block a user