zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add voice processing settings and a speak-time bypass

Browse Source
This commit is contained in:
Fisto
2026-09-21 18:23:40 +02:00
parent 17d4339ea3
commit 63083c54ec
10 changed files with 230 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CURRENT.md
+8
View File
@@ -40,6 +40,14 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C72 Voice processing settings and the speak bypass. The TTS panel gained a **voice
processing** block (echo cancellation, auto gain, noise cancellation, and **off while
speaking**), stored in `preferences.voice` and applied live on save. While a TTS burst plays,
the daemon turns every supported gate off and restores the resting state when the queue drains;
`voice.join` applies the resting state after connecting. `voice.processing` gained a
`readOnly` probe (MCP `set_voice_processing` too). Verified live: gates ON at rest → all off
during speech → restored after. Krisp noise suppression is not a checkbox because its getter
ignores the setter in this client build.
- [x] C71 Voice processing gates off. The TTS line was ramping in because Discord's automatic
gain control (and echo/noise cancellation) were on. New `voice.processing` core action reads and
sets the gates (defaults off) plus an optional input volume; MCP `set_voice_processing` exposes
docs/voice.md
+7 -2
View File
@@ -58,8 +58,13 @@ Voice audio uses the system audio stack, not Discord internals. The daemon owns
Discord's own microphone processing fights a TTS line: **automatic gain control** ramps the volume
up over the first words (so the start is quiet or clipped), and echo/noise cancellation gates the
first syllable. The core reads and turns those gates off (`voice.processing`; MCP
`set_voice_processing`), and every `voice.join` applies it automatically before connecting. Discord
stores these per client, not per account, so a user's own app is unaffected.
`set_voice_processing`, `readOnly: true` to just report). The TTS panel has **voice processing**
checkboxes for the resting state — echo cancellation, auto gain, noise cancellation — plus **off
while speaking**, which turns every gate off for a TTS burst and restores the resting state when
the queue drains. Every `voice.join` applies the resting state after connecting (the core joins
with processing off). Discord stores these per client, not per account, so a user's own app is
unaffected. Krisp noise suppression is not exposed as a checkbox: its getter does not follow the
setter in the current client build, though the preference and API field remain.
The host needs a sound server. On Debian and Ubuntu:
poc/public/app.js
+36
View File
@@ -1788,6 +1788,37 @@ async function refreshAudio() {
await refreshSpeech();
}
/* Discord voice processing gates (Settings -> TTS panel). */
function fillVoiceGates(voice) {
const gates = voice ?? {};
for (const [id, key] of [
["#vp-echo", "echoCancellation"],
["#vp-agc", "automaticGainControl"],
["#vp-nc", "noiseCancellation"],
["#vp-bypass", "bypassWhileSpeaking"]
]) {
const box = $(id);
if (box) box.checked = gates[key] === true;
}
}
async function saveVoiceGates() {
const voice = {
noiseSuppression: false,
echoCancellation: $("#vp-echo").checked,
automaticGainControl: $("#vp-agc").checked,
noiseCancellation: $("#vp-nc").checked,
bypassWhileSpeaking: $("#vp-bypass").checked
};
try {
const prefs = await VK.api.post("/api/preferences", { voice });
fillVoiceGates(prefs.voice ?? voice);
} catch (error) {
toast(`voice processing save failed: ${error.message}`, "error");
}
}
/* Speech settings (server-side preferences). */
async function refreshSpeech() {
@@ -1815,6 +1846,7 @@ async function refreshSpeech() {
try {
const prefs = await VK.api.get("/api/preferences");
state.speech = prefs.tts ?? { provider: null, language: null, voice: null, speed: null, pitch: null };
fillVoiceGates(prefs.voice);
} catch {
// preferences are optional
}
@@ -2728,6 +2760,10 @@ function init() {
$("#transcript-refresh").addEventListener("click", () => void refreshTranscript());
$("#transcript-start").addEventListener("click", () => void setCapture(true));
$("#transcript-stop").addEventListener("click", () => void setCapture(false));
for (const id of ["#vp-echo", "#vp-agc", "#vp-nc", "#vp-bypass"]) {
$(id)?.addEventListener("change", () => void saveVoiceGates());
}
$("#tts-duck").addEventListener("click", async () => {
const on = !$("#tts-duck").classList.contains("vk-chip-on");
try {
poc/public/index.html
+9
View File
@@ -139,6 +139,15 @@
duck on voice
</button>
<span class="vk-chip vk-chip-on">queue lines</span>
<div class="vk-voice-gates" title="Discord's microphone processing. Leave off so TTS is not gated or ramped in.">
<span class="vk-micro-label">voice processing</span>
<label class="vk-tts-check"><input id="vp-echo" type="checkbox" /> echo cancellation</label>
<label class="vk-tts-check"><input id="vp-agc" type="checkbox" /> auto gain</label>
<label class="vk-tts-check"><input id="vp-nc" type="checkbox" /> noise cancellation</label>
<label class="vk-tts-check" title="Turn every gate off while a TTS line plays, then restore the states above">
<input id="vp-bypass" type="checkbox" /> off while speaking
</label>
</div>
<div class="vk-tts-speed">
<label class="vk-tts-slider" title="Speech rate; 1.0x is the engine's natural rate">
<span class="vk-micro-label">speed</span>
poc/src/audio-service.ts
+20
View File
@@ -25,6 +25,8 @@ export interface AudioServiceOptions {
getVoiceContext?: () => Promise<{ guildId?: string | null; channelId?: string | null } | undefined>;
getSpeakerName?: (userId: string) => Promise<string | undefined>;
onTranscript?: (event: Record<string, unknown>) => void;
/** Called true when a TTS burst starts and false when the queue drains. */
onSpeakActivity?: (active: boolean) => void;
}
export interface SpeakResult {
@@ -51,6 +53,8 @@ export class AudioService {
private readonly speakerIntervals: SpeakerInterval[] = [];
private readonly speakerNames = new Map<string, string>();
private playing = false;
/** True while a TTS burst holds the voice-processing bypass. */
private bypassed = false;
private duckEnabled = true;
private nextId = 1;
private ducked = false;
@@ -381,6 +385,14 @@ export class AudioService {
}
this.playing = true;
if (!this.bypassed) {
this.bypassed = true;
try {
this.options.onSpeakActivity?.(true);
} catch {
// Best effort: speech still plays.
}
}
try {
// Streaming word mode: the wrapper writes the WAV while it synthesizes, so
// playback starts on the first word instead of after the whole line.
@@ -408,6 +420,14 @@ export class AudioService {
} finally {
this.playing = false;
await this.applyDuck(false);
if (this.bypassed && this.queue.length === 0) {
this.bypassed = false;
try {
this.options.onSpeakActivity?.(false);
} catch {
// Best effort.
}
}
void this.pump();
}
}
poc/src/control-bridge.ts
+60 -5
View File
@@ -186,6 +186,19 @@ const media = new MediaService({
log: line => console.log(`[media] ${line}`)
});
/** State saved before a TTS burst turns Discord's mic processing off. */
let voiceProcessingSaved: Record<string, boolean> | null = null;
function restingVoiceProcessing(): Record<string, boolean> {
const prefs = preferences.get().voice;
return {
noiseSuppression: prefs.noiseSuppression,
echoCancellation: prefs.echoCancellation,
automaticGainControl: prefs.automaticGainControl,
noiseCancellation: prefs.noiseCancellation
};
}
const audio = new AudioService({
cacheDir: process.env.VESKSLOP_CACHE_DIR,
transcripts,
@@ -199,6 +212,32 @@ const audio = new AudioService({
preRollMs: Number(process.env.VESKSLOP_VAD_PRE_ROLL_MS ?? DEFAULT_VAD.preRollMs),
maxSegmentMs: Number(process.env.VESKSLOP_VAD_MAX_SEGMENT_MS ?? DEFAULT_VAD.maxSegmentMs)
},
onSpeakActivity: active => {
const prefs = preferences.get().voice;
if (!prefs.bypassWhileSpeaking) return;
const client = core;
if (!client || !coreConnected) return;
if (active) {
voiceProcessingSaved = null;
void client
.voiceProcessing({
noiseSuppression: false,
echoCancellation: false,
automaticGainControl: false,
noiseCancellation: false
})
.then((result: any) => {
voiceProcessingSaved = result?.before ?? null;
})
.catch(() => undefined);
} else {
const restore = voiceProcessingSaved ?? restingVoiceProcessing();
voiceProcessingSaved = null;
void client.voiceProcessing(restore).catch(() => undefined);
}
},
getSelfId: async () => (core ? await core.getSelfId() : undefined),
getVoiceContext: async () => {
if (!core) return {};
@@ -481,7 +520,14 @@ function buildServer(): McpServer {
}
},
async ({ channelId, mute }) =>
jsonText(await pacedVoice(() => requireCore().voiceJoin(channelId, mute ?? true)))
jsonText(
await pacedVoice(async () => {
const joined = await requireCore().voiceJoin(channelId, mute ?? true);
// The core joins with processing off; restore the resting state.
await requireCore().voiceProcessing(restingVoiceProcessing()).catch(() => undefined);
return joined;
})
)
);
server.registerTool(
@@ -497,18 +543,27 @@ function buildServer(): McpServer {
echoCancellation: z.boolean().optional(),
automaticGainControl: z.boolean().optional(),
noiseCancellation: z.boolean().optional(),
inputVolume: z.number().min(0).max(200).optional().describe("Discord input volume percent")
inputVolume: z.number().min(0).max(200).optional().describe("Discord input volume percent"),
readOnly: z.boolean().optional().describe("Only report the current gates, change nothing")
}
},
async ({ noiseSuppression, echoCancellation, automaticGainControl, noiseCancellation, inputVolume }) =>
async ({
noiseSuppression,
echoCancellation,
automaticGainControl,
noiseCancellation,
inputVolume,
readOnly
}) =>
jsonText(
await requireCore().voiceProcessing({
...(noiseSuppression !== undefined ? { noiseSuppression } : {}),
...(echoCancellation !== undefined ? { echoCancellation } : {}),
...(automaticGainControl !== undefined ? { automaticGainControl } : {}),
...(noiseCancellation !== undefined ? { noiseCancellation } : {}),
...(inputVolume !== undefined ? { inputVolume } : {})
})
...(inputVolume !== undefined ? { inputVolume } : {}),
...(readOnly === true ? { readOnly: true } : {})
} as any)
)
);
poc/src/preferences.ts
+36
View File
@@ -128,6 +128,16 @@ export interface MediaTunables {
};
}
/** Discord's microphone processing gates, plus a TTS bypass. */
export interface VoicePreferences {
noiseSuppression: boolean;
echoCancellation: boolean;
automaticGainControl: boolean;
noiseCancellation: boolean;
/** Turn every gate off while a TTS line plays, then restore the state above. */
bypassWhileSpeaking: boolean;
}
export interface MediaPreferences {
/** Combined size limit for the media DB root: databases, downloads, previews. */
limitBytes: number;
@@ -143,6 +153,7 @@ export interface WebPreferences {
visibleGuilds: string[] | null;
tts: TtsPreferences;
audio: AudioPreferences;
voice: VoicePreferences;
favourites: FavouritesPreferences;
media: MediaPreferences;
}
@@ -173,6 +184,13 @@ const DEFAULTS: WebPreferences = {
streamWords: true
},
audio: { duck: true },
voice: {
noiseSuppression: false,
echoCancellation: false,
automaticGainControl: false,
noiseCancellation: false,
bypassWhileSpeaking: true
},
favourites: { users: [], channels: [] },
media: {
limitBytes: 30 * 1024 * 1024 * 1024,
@@ -313,6 +331,19 @@ function normalizeExpression(value: unknown): number | null {
return Math.round(expression);
}
function normalizeVoice(raw: any, fallback: VoicePreferences): VoicePreferences {
return {
noiseSuppression: typeof raw?.noiseSuppression === "boolean" ? raw.noiseSuppression : fallback.noiseSuppression,
echoCancellation: typeof raw?.echoCancellation === "boolean" ? raw.echoCancellation : fallback.echoCancellation,
automaticGainControl:
typeof raw?.automaticGainControl === "boolean" ? raw.automaticGainControl : fallback.automaticGainControl,
noiseCancellation:
typeof raw?.noiseCancellation === "boolean" ? raw.noiseCancellation : fallback.noiseCancellation,
bypassWhileSpeaking:
typeof raw?.bypassWhileSpeaking === "boolean" ? raw.bypassWhileSpeaking : fallback.bypassWhileSpeaking
};
}
function normalizeLevel(value: unknown): number | null {
if (isUnset(value)) return null;
const number = Number(value);
@@ -399,6 +430,10 @@ export class PreferencesStore {
};
}
if (patch.voice !== undefined) {
this.preferences.voice = normalizeVoice(patch.voice, this.preferences.voice);
}
if (patch.media !== undefined) {
this.preferences.media = normalizeMedia(patch.media, this.preferences.media);
}
@@ -468,6 +503,7 @@ export class PreferencesStore {
audio: {
duck: raw?.audio?.duck !== false
},
voice: normalizeVoice(raw?.voice, DEFAULTS.voice),
media: normalizeMedia(raw?.media, DEFAULTS.media),
favourites: {
users: Array.isArray(raw?.favourites?.users)
poc/src/webui.ts
+38 -1
View File
@@ -484,7 +484,14 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
app.post(
"/api/preferences",
wrap(async (req, res) => {
const { visibleGuilds, tts, audio, favourites, media: mediaPrefs } = req.body ?? {};
const {
visibleGuilds,
tts,
audio,
voice: voicePrefs,
favourites,
media: mediaPrefs
} = req.body ?? {};
if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
@@ -549,8 +556,24 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
deps.audio.setDuckEnabled(patch.audio.duck);
}
if (voicePrefs !== undefined) {
patch.voice = {
noiseSuppression: voicePrefs?.noiseSuppression === true,
echoCancellation: voicePrefs?.echoCancellation === true,
automaticGainControl: voicePrefs?.automaticGainControl === true,
noiseCancellation: voicePrefs?.noiseCancellation === true,
bypassWhileSpeaking: voicePrefs?.bypassWhileSpeaking !== false
};
}
const updated = deps.preferences.update(patch);
if (mediaPrefs !== undefined) deps.media.applyTunables();
if (voicePrefs !== undefined) {
const core = deps.getCore();
if (core && deps.isCoreConnected()) {
void core.voiceProcessing(patch.voice).catch(() => undefined);
}
}
res.json({ ok: true, result: updated });
})
);
@@ -655,6 +678,20 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
const result = await pacedVoice(() => requireCore().voiceJoin(channelId, mute));
deps.getPuppet()?.appendAction({ type: "voice", action: "join", channelId, mute });
// The core joins with processing off; put the user's resting state back.
const resting = deps.preferences.get().voice;
if (deps.isCoreConnected()) {
void requireCore()
.voiceProcessing({
noiseSuppression: resting.noiseSuppression,
echoCancellation: resting.echoCancellation,
automaticGainControl: resting.automaticGainControl,
noiseCancellation: resting.noiseCancellation
})
.catch(() => undefined);
}
res.json({ ok: true, result });
})
);
poc/test/media-ui-smoke.mjs
+12 -1
View File
@@ -58,9 +58,20 @@ try {
});
const page = await context.newPage();
page.on("console", message => {
if (message.type() === "error") results.consoleErrors.push(message.text());
if (message.type() !== "error") return;
// Resource errors carry no URL; the response listener checks those precisely.
if (message.text().startsWith("Failed to load resource")) return;
results.consoleErrors.push(message.text());
});
page.on("pageerror", error => results.consoleErrors.push(String(error)));
page.on("response", response => {
if (response.status() < 400) return;
// The grid falls back to the original CDN link when nothing is cached; expired
// signed links 404 by design and the tile shows its placeholder.
if (/^https:\/\/(cdn|media)\.discordapp\.(com|net)\//.test(response.url())) return;
results.consoleErrors.push(`HTTP ${response.status()} ${response.url()}`);
});
await page.goto(URL, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForSelector("#scope-media", { timeout: 20000 });
src/renderer/puppet/actions.ts
+4
View File
@@ -891,6 +891,10 @@ export async function handlePuppetAction(action: string, params: Record<string,
}
case PuppetAction.VoiceProcessing: {
if (params.readOnly === true) {
return ok({ ...readVoiceProcessing(), inputVolume: null });
}
const wanted: Partial<Record<VoiceProcessingKey, boolean>> = {};
for (const key of Object.keys(VOICE_PROCESSING) as VoiceProcessingKey[]) {