Add voice audio, capture and transcript, browser monitor, and DM UI
This commit is contained in:
Fisto
+210
-7
@@ -15,10 +15,16 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
|
||||
import * as z from "zod/v4";
|
||||
|
||||
import { AuthService } from "./auth-service.js";
|
||||
import { AudioService } from "./audio-service.js";
|
||||
import { tokenFile } from "./config.js";
|
||||
import { dirname, join } from "node:path";
|
||||
import { Stt } from "./stt.js";
|
||||
import { TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD } from "./vad.js";
|
||||
import { CoreClient, coreSocketPath } from "./core-client.js";
|
||||
import { CoreProcess } from "./core-process.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, sleep } from "./pacing.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js";
|
||||
import { PreferencesStore } from "./preferences.js";
|
||||
import { Puppet } from "./puppet.js";
|
||||
import { startWebUi } from "./webui.js";
|
||||
|
||||
@@ -40,10 +46,11 @@ async function connectLoop(): Promise<void> {
|
||||
const client = await CoreClient.connect();
|
||||
|
||||
core = client;
|
||||
puppet = new Puppet(client);
|
||||
puppet = new Puppet(client, audio);
|
||||
|
||||
client.onEvent(event => {
|
||||
void puppet!.enqueueEvent(event);
|
||||
void audio.onEvent(event);
|
||||
webui.broadcast(event);
|
||||
});
|
||||
|
||||
@@ -59,6 +66,7 @@ async function connectLoop(): Promise<void> {
|
||||
coreConnected = true;
|
||||
console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`);
|
||||
webui.broadcast({ type: "CORE_STATUS", coreConnected: true });
|
||||
void autoConnectAudio(client);
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
@@ -83,6 +91,81 @@ function requirePuppet(): Puppet {
|
||||
|
||||
const coreProcess = new CoreProcess({ tokenFile });
|
||||
|
||||
const transcripts = new TranscriptStore(
|
||||
process.env.VESKSLOP_TRANSCRIPT ?? join(dirname(tokenFile), "transcripts.jsonl")
|
||||
);
|
||||
const preferences = new PreferencesStore(
|
||||
process.env.VESKSLOP_PREFERENCES ?? join(dirname(tokenFile), "preferences.json"),
|
||||
{ legacyFile: process.env.VESKSLOP_STATE_FILE ?? join(dirname(tokenFile), ".veskslop-state.json") }
|
||||
);
|
||||
const stt = new Stt();
|
||||
|
||||
const audio = new AudioService({
|
||||
cacheDir: process.env.VESKSLOP_CACHE_DIR,
|
||||
transcripts,
|
||||
stt,
|
||||
autoCapture: process.env.VESKSLOP_CAPTURE_AUTO === "1",
|
||||
vad: {
|
||||
...DEFAULT_VAD,
|
||||
thresholdDb: Number(process.env.VESKSLOP_VAD_THRESHOLD_DB ?? DEFAULT_VAD.thresholdDb),
|
||||
minSpeechMs: Number(process.env.VESKSLOP_VAD_MIN_SPEECH_MS ?? DEFAULT_VAD.minSpeechMs),
|
||||
silenceMs: Number(process.env.VESKSLOP_VAD_SILENCE_MS ?? DEFAULT_VAD.silenceMs),
|
||||
preRollMs: Number(process.env.VESKSLOP_VAD_PRE_ROLL_MS ?? DEFAULT_VAD.preRollMs),
|
||||
maxSegmentMs: Number(process.env.VESKSLOP_VAD_MAX_SEGMENT_MS ?? DEFAULT_VAD.maxSegmentMs)
|
||||
},
|
||||
getSelfId: async () => (core ? await core.getSelfId() : undefined),
|
||||
getVoiceContext: async () => {
|
||||
if (!core) return {};
|
||||
const state = (await core.voiceGet()) as { guildId?: string | null; channelId?: string | null };
|
||||
return { guildId: state?.guildId ?? null, channelId: state?.channelId ?? null };
|
||||
},
|
||||
getSpeakerName: async userId => {
|
||||
if (!core) return undefined;
|
||||
const state = (await core.voiceGet(userId)) as {
|
||||
members?: Array<{ id: string; username?: string }>;
|
||||
};
|
||||
return state?.members?.find(member => member.id === userId)?.username;
|
||||
},
|
||||
onTranscript: event => webui.broadcast(event)
|
||||
});
|
||||
|
||||
void audio.init();
|
||||
|
||||
/** Point the client at the virtual microphone and capture sink when present. */
|
||||
async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolean; message: string }> {
|
||||
if (process.env.VESKSLOP_AUDIO_AUTOCONNECT === "0") {
|
||||
return { connected: false, message: "auto-connect is disabled" };
|
||||
}
|
||||
|
||||
const status = audio.status();
|
||||
if (!status.available) {
|
||||
return { connected: false, message: status.hint ?? status.error ?? "audio is not available" };
|
||||
}
|
||||
|
||||
try {
|
||||
await audio.manager.ensureDevices();
|
||||
|
||||
const devices = (await client.voiceDevices()) as {
|
||||
input?: Array<{ id: string; label: string }>;
|
||||
output?: Array<{ id: string; label: string }>;
|
||||
};
|
||||
|
||||
const input = devices.input?.find(device => device.label.includes("VeskslopMic"));
|
||||
const output = devices.output?.find(device => device.label.includes("VeskslopCapture"));
|
||||
|
||||
if (!input && !output) {
|
||||
return { connected: false, message: "the client does not see the virtual devices yet" };
|
||||
}
|
||||
|
||||
const result = await client.voiceSetDevices(input?.id, output?.id);
|
||||
console.log("[audio] client devices set:", JSON.stringify(result));
|
||||
return { connected: true, message: "the client now uses the virtual devices" };
|
||||
} catch (error) {
|
||||
console.warn("[audio] device auto-connect failed:", (error as Error).message);
|
||||
return { connected: false, message: (error as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
const auth = new AuthService(tokenFile, update => {
|
||||
webui.broadcast({ type: "AUTH_QR", update });
|
||||
|
||||
@@ -100,7 +183,11 @@ const webui = startWebUi({
|
||||
getPuppet: () => puppet,
|
||||
isCoreConnected: () => coreConnected,
|
||||
coreProcess,
|
||||
auth
|
||||
auth,
|
||||
audio,
|
||||
transcripts,
|
||||
preferences,
|
||||
connectAudioDevices: () => autoConnectAudio(requireCore())
|
||||
});
|
||||
|
||||
coreProcess.onExit(() => {
|
||||
@@ -118,6 +205,9 @@ const READ_ONLY_TOOLS: Record<string, (args: any) => Promise<unknown> | unknown>
|
||||
get_voice_overview: args => requireCore().voiceOverview(args?.guildId),
|
||||
search_users: args => requireCore().userSearch(args?.query, args?.guildId, args?.limit),
|
||||
list_dms: args => requireCore().dmList(args?.limit),
|
||||
get_audio_status: () => audio.status(),
|
||||
list_audio_devices: () => requireCore().voiceDevices(),
|
||||
get_transcript: args => transcripts.readSegments(args?.limit ?? 100, args?.sessionId),
|
||||
get_context: args =>
|
||||
requirePuppet().getContext(args?.channelId, {
|
||||
maxMessages: args?.maxMessages,
|
||||
@@ -268,6 +358,110 @@ function buildServer(): McpServer {
|
||||
async ({ guildId }) => jsonText(await requireCore().voiceOverview(guildId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_join",
|
||||
{
|
||||
description: "Join a voice channel as the logged-in user. Joins muted by default.",
|
||||
inputSchema: {
|
||||
channelId: z.string().describe("Voice channel ID"),
|
||||
mute: z.boolean().optional().describe("Join muted. Defaults to true.")
|
||||
}
|
||||
},
|
||||
async ({ channelId, mute }) =>
|
||||
jsonText(await pacedVoice(() => requireCore().voiceJoin(channelId, mute ?? true)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_leave",
|
||||
{
|
||||
description: "Leave the current voice channel."
|
||||
},
|
||||
async () => jsonText(await pacedVoice(() => requireCore().voiceLeave()))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_move",
|
||||
{
|
||||
description: "Move the client to another voice channel.",
|
||||
inputSchema: {
|
||||
channelId: z.string().describe("Voice channel ID")
|
||||
}
|
||||
},
|
||||
async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_audio_status",
|
||||
{
|
||||
description: "Audio stack, virtual devices, TTS providers, queue, ducking, and capture state."
|
||||
},
|
||||
async () => jsonText(audio.status())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"speak",
|
||||
{
|
||||
description: "Queue text to speech into the bot's virtual microphone.",
|
||||
inputSchema: {
|
||||
text: z.string().min(1).max(1000).describe("Text to speak"),
|
||||
provider: z.enum(["local", "api"]).optional().describe("TTS provider"),
|
||||
voice: z.string().optional().describe("Voice name for the provider")
|
||||
}
|
||||
},
|
||||
async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice }))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_audio",
|
||||
{
|
||||
description: "Stop speech and clear the queue."
|
||||
},
|
||||
async () => {
|
||||
audio.stop();
|
||||
return jsonText(audio.status());
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"start_capture",
|
||||
{
|
||||
description: "Record the client output sink to a WAV file on the host."
|
||||
},
|
||||
async () => jsonText(await audio.startCapture())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_capture",
|
||||
{
|
||||
description: "Stop the current capture."
|
||||
},
|
||||
async () => {
|
||||
audio.stopCapture();
|
||||
return jsonText(audio.status());
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"list_audio_devices",
|
||||
{
|
||||
description: "List the client's input and output audio devices."
|
||||
},
|
||||
async () => jsonText(await requireCore().voiceDevices())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"set_audio_devices",
|
||||
{
|
||||
description: "Set the client's input and output audio devices.",
|
||||
inputSchema: {
|
||||
inputDeviceId: z.string().optional().describe("Input device ID"),
|
||||
outputDeviceId: z.string().optional().describe("Output device ID")
|
||||
}
|
||||
},
|
||||
async ({ inputDeviceId, outputDeviceId }) =>
|
||||
jsonText(await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"search_users",
|
||||
{
|
||||
@@ -303,6 +497,18 @@ function buildServer(): McpServer {
|
||||
async ({ limit }) => jsonText(requirePuppet().readActionLog(limit))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_transcript",
|
||||
{
|
||||
description: "Read captured call segments: time, speaker, WAV path, and text when STT is configured.",
|
||||
inputSchema: {
|
||||
limit: z.number().int().min(1).max(500).default(100).describe("How many segments"),
|
||||
sessionId: z.string().optional().describe("Only this capture session")
|
||||
}
|
||||
},
|
||||
async ({ limit, sessionId }) => jsonText(transcripts.readSegments(limit, sessionId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"run_commands",
|
||||
{
|
||||
@@ -427,10 +633,7 @@ app.listen(PORT, HOST, (error?: Error) => {
|
||||
|
||||
function shutdown() {
|
||||
core?.close();
|
||||
void coreProcess
|
||||
.dispose(4000)
|
||||
.catch(() => undefined)
|
||||
.finally(() => process.exit(0));
|
||||
void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0));
|
||||
}
|
||||
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
Reference in New Issue
Block a user