zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add MicroslopSam TTS, Flextalk voice profiles, effects, and engine controls

Browse Source
This commit is contained in:
Fisto
2026-09-15 15:09:23 +02:00
parent eab94132bb
commit 302436f95f
42 changed files with 4675 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/src/control-bridge.ts
+187 -9
View File
@@ -22,7 +22,7 @@ import { Stt } from "./stt.js";
import { TranscriptStore } from "./transcript.js";
import { DEFAULT_VAD } from "./vad.js";
import { CoreClient, coreSocketPath } from "./core-client.js";
import { CoreProcess } from "./core-process.js";
import { CoreProcess, staleBundleWarning } from "./core-process.js";
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js";
import { PreferencesStore } from "./preferences.js";
import { Puppet } from "./puppet.js";
@@ -49,6 +49,13 @@ async function connectLoop(): Promise<void> {
puppet = new Puppet(client, audio);
client.onEvent(event => {
// Speaking hints add to the reconciled set so the poller can
// clear users the store no longer reports as speaking.
if (event?.type === "VOICE_SPEAKING" && event.userId) {
if (event.speaking) speakingUsers.add(String(event.userId));
else speakingUsers.delete(String(event.userId));
}
void puppet!.enqueueEvent(event);
void audio.onEvent(event);
webui.broadcast(event);
@@ -56,6 +63,7 @@ async function connectLoop(): Promise<void> {
client.onClose(() => {
console.warn("[daemon] core bridge disconnected");
stopSpeakingPoll();
coreConnected = false;
core = undefined;
puppet = undefined;
@@ -66,7 +74,8 @@ async function connectLoop(): Promise<void> {
coreConnected = true;
console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`);
webui.broadcast({ type: "CORE_STATUS", coreConnected: true });
void autoConnectAudio(client);
startSpeakingPoll();
void autoConnectAudioWithRetry(client);
return;
} catch (error) {
@@ -89,6 +98,69 @@ function requirePuppet(): Puppet {
return puppet;
}
/*
* Discord's dispatcher only sends speaking-start events, so the UI and the
* transcript attribution need the store's authoritative set. Poll it once a
* second and synthesize the missing stop/start events. Dispatcher events feed
* the same set so a user the store no longer reports is cleared.
*/
let speakingTimer: ReturnType<typeof setInterval> | undefined;
const speakingUsers = new Set<string>();
function emitCoreEvent(event: any): void {
void puppet?.enqueueEvent(event);
void audio.onEvent(event);
webui.broadcast(event);
}
async function pollSpeaking(): Promise<void> {
if (!core) return;
try {
const result = (await core.voiceSpeaking()) as { channelId?: string | null; userIds?: string[] };
const next = new Set<string>(result?.channelId ? (result.userIds ?? []) : []);
for (const userId of next) {
if (!speakingUsers.has(userId)) {
emitCoreEvent({
type: "VOICE_SPEAKING",
userId,
speaking: true,
channelId: result.channelId ?? null
});
}
}
for (const userId of speakingUsers) {
if (!next.has(userId)) {
emitCoreEvent({
type: "VOICE_SPEAKING",
userId,
speaking: false,
channelId: result?.channelId ?? null
});
}
}
speakingUsers.clear();
for (const userId of next) speakingUsers.add(userId);
} catch {
// The close handler stops the poller.
}
}
function startSpeakingPoll(): void {
stopSpeakingPoll();
speakingTimer = setInterval(() => void pollSpeaking(), 1000);
}
function stopSpeakingPoll(): void {
if (speakingTimer) clearInterval(speakingTimer);
speakingTimer = undefined;
speakingUsers.clear();
}
const coreProcess = new CoreProcess({ tokenFile });
const transcripts = new TranscriptStore(
@@ -129,6 +201,7 @@ const audio = new AudioService({
onTranscript: event => webui.broadcast(event)
});
audio.setDuckEnabled(preferences.get().audio?.duck !== false);
void audio.init();
/** Point the client at the virtual microphone and capture sink when present. */
@@ -161,11 +234,25 @@ async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolea
console.log("[audio] client devices set:", JSON.stringify(result));
return { connected: true, message: "the client now uses the virtual devices" };
} catch (error) {
console.warn("[audio] device auto-connect failed:", (error as Error).message);
return { connected: false, message: (error as Error).message };
}
}
/**
* The bridge socket is up before the renderer finishes loading, so the first
* device calls can fail with "core is not ready" style errors. Retry.
*/
async function autoConnectAudioWithRetry(client: CoreClient): Promise<void> {
for (let attempt = 1; attempt <= 10; attempt++) {
if (attempt > 1) await sleep(2500);
const result = await autoConnectAudio(client);
if (result.connected) return;
console.warn(`[audio] device auto-connect attempt ${attempt}: ${result.message}`);
}
}
const auth = new AuthService(tokenFile, update => {
webui.broadcast({ type: "AUTH_QR", update });
@@ -385,12 +472,36 @@ function buildServer(): McpServer {
{
description: "Move the client to another voice channel.",
inputSchema: {
channelId: z.string().describe("Voice channel ID")
channelId: z.string().describe("Channel ID")
}
},
async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId)))
);
server.registerTool(
"get_user_volume",
{
description:
"Per-user playback volume (0 to 200, 100 is default) and the client's own output volume. A volume of 0 silences that user.",
inputSchema: {
userIds: z.array(z.string()).min(1).max(50).describe("User IDs to look up")
}
},
async ({ userIds }) => jsonText(await requireCore().userVolume(userIds))
);
server.registerTool(
"set_user_volume",
{
description: "Set a user's playback volume. 0 silences the user; 100 is default; 200 is the maximum.",
inputSchema: {
userId: z.string().describe("User ID"),
volume: z.number().int().min(0).max(200).describe("Volume from 0 to 200")
}
},
async ({ userId, volume }) => jsonText(await pacedVoice(() => requireCore().userSetVolume(userId, volume)))
);
server.registerTool(
"get_audio_status",
{
@@ -405,11 +516,58 @@ function buildServer(): McpServer {
description: "Queue text to speech into the bot's virtual microphone.",
inputSchema: {
text: z.string().min(1).max(1000).describe("Text to speak"),
provider: z.enum(["local", "api"]).optional().describe("TTS provider"),
voice: z.string().optional().describe("Voice name for the provider")
provider: z
.enum(["local", "espeak", "command", "api"])
.optional()
.describe("TTS engine: command is the configured VESKSLOP_TTS_CMD engine"),
voice: z.string().optional().describe("Voice name for the provider"),
speed: z.number().min(0.5).max(2).optional().describe("Rate multiplier, 1.0 is natural"),
pitch: z.number().min(-12).max(12).optional().describe("Pitch offset in semitones"),
expression: z
.number()
.min(0)
.max(100)
.optional()
.describe("Flextalk Expression profile value (Keith Bell, Kerry Watson)"),
profile: z
.record(z.string(), z.number())
.optional()
.describe("Flextalk profile overrides (pitch, rate, prominences, head_size, lowering)"),
highpass: z.number().min(40).max(400).optional().describe("High-pass filter Hz"),
echo: z.number().min(0).max(100).optional().describe("Mono echo amount"),
words: z.boolean().optional().describe("Synthesize each word and merge with gaps"),
wordGapMs: z.number().min(0).max(500).optional().describe("Silence between words in ms")
}
},
async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice }))
async ({ text, provider, voice, speed, pitch, expression, profile, highpass, echo, words, wordGapMs }) =>
jsonText(
await audio.speak(text, {
provider,
voice,
speed,
pitch,
expression,
profile,
highpass,
echo,
words,
wordGapMs
})
)
);
server.registerTool(
"start_tts_engine",
{
description: "Start (warm up) the TTS engine so the first spoken line is fast.",
inputSchema: {
engine: z
.enum(["local", "espeak", "command", "api"])
.optional()
.describe("Engine to start. Defaults to the configured one.")
}
},
async ({ engine }) => jsonText(await audio.warmTts(engine))
);
server.registerTool(
@@ -620,8 +778,24 @@ app.delete("/mcp", (_req, res) => {
);
});
app.get("/health", (_req, res) => {
res.json({ ok: true, coreConnected, socketPath: coreSocketPath, ownerId: puppet?.owner, queries: getQueryCount() });
app.get("/health", async (_req, res) => {
let selfId = puppet?.self;
if (!selfId && core) {
try {
selfId = await core.getSelfId();
} catch {
// The core may not be ready yet.
}
}
res.json({
ok: true,
coreConnected,
socketPath: coreSocketPath,
ownerId: puppet?.owner,
selfId,
queries: getQueryCount()
});
});
app.listen(PORT, HOST, (error?: Error) => {
@@ -630,9 +804,13 @@ app.listen(PORT, HOST, (error?: Error) => {
process.exit(1);
}
console.log(`[daemon] MCP listening on http://${HOST}:${PORT}/mcp`);
const stale = staleBundleWarning();
if (stale) console.warn(`[daemon] ${stale}`);
});
function shutdown() {
stopSpeakingPoll();
core?.close();
void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0));
}