zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add TTS volume, a browser preview, and the split-audio plan

Browse Source
This commit is contained in:
Fisto
2026-09-22 09:27:14 +02:00
parent d99d614c60
commit 420fdca3cf
11 changed files with 287 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/public/app.js
+41 -1
View File
@@ -2029,9 +2029,11 @@ async function warmEngine() {
function fillSpeechSliders(tts) {
const speed = typeof tts?.speed === "number" ? tts.speed : 1;
const pitch = typeof tts?.pitch === "number" ? tts.pitch : 0;
const volume = typeof tts?.volume === "number" ? tts.volume : 100;
$("#tts-speed").value = Number.isFinite(speed) ? speed : 1;
$("#tts-pitch").value = Number.isFinite(pitch) ? pitch : 0;
$("#tts-volume").value = Number.isFinite(volume) ? volume : 100;
syncSpeechSliderLabels();
}
@@ -2042,6 +2044,7 @@ function syncSpeechSliderLabels() {
$("#tts-speed-value").textContent = `${speed.toFixed(2)}x`;
$("#tts-pitch-value").textContent = pitchSlider.disabled ? "n/a" : `${pitch > 0 ? "+" : ""}${pitch} st`;
$("#tts-volume-value").textContent = `${Number($("#tts-volume").value)}%`;
const expressionSlider = $("#tts-expression");
if (expressionSlider) {
@@ -2177,6 +2180,7 @@ function effectOptions() {
profile: profileFromControls(),
highpass: Number($("#tts-highpass").value),
echo: Number($("#tts-echo").value),
volume: Number($("#tts-volume").value),
words: $("#tts-words").checked,
wordGapMs: Number($("#tts-word-gap").value),
// VOX and the Half-Life profile are part of the `vox`/`hev` voices now.
@@ -2463,6 +2467,40 @@ async function speakText() {
}
}
async function previewSpeech() {
const text = $("#tts-input").value.trim();
if (!text) return;
const button = $("#tts-preview");
button.disabled = true;
try {
const provider = $("#tts-provider").value;
const speed = Number($("#tts-speed").value);
const pitch = Number($("#tts-pitch").value);
const expressionEl = $("#tts-expression");
const expression =
expressionEl.dataset.dirty === "1" ? Number(expressionEl.value) : state.speech.expression ?? undefined;
const result = await VK.api.post("/api/audio/preview", {
text,
provider,
speed,
pitch,
...(typeof expression === "number" ? { expression } : {}),
...effectOptions()
});
const audio = new Audio(result.url);
audio.volume = Math.min(1, Math.max(0, Number($("#tts-volume").value) / 100));
await audio.play();
} catch (error) {
toast(error.message, "error");
} finally {
button.disabled = false;
}
}
/* Transcript */
function transcriptLine(segment) {
@@ -2880,11 +2918,12 @@ function init() {
});
$("#tts-expression").addEventListener("change", () => void saveSpeech());
buildProfileSliders();
for (const id of ["tts-highpass", "tts-echo", "tts-word-gap"]) {
for (const id of ["tts-highpass", "tts-echo", "tts-word-gap", "tts-volume"]) {
$(`#${id}`).addEventListener("input", () => {
$("#tts-highpass-value").textContent = `${$("#tts-highpass").value}Hz`;
$("#tts-echo-value").textContent = Number($("#tts-echo").value) > 0 ? $("#tts-echo").value : "off";
$("#tts-word-gap-value").textContent = `${$("#tts-word-gap").value}ms`;
$("#tts-volume-value").textContent = `${Number($("#tts-volume").value)}%`;
});
$(`#${id}`).addEventListener("change", () => void saveSpeech());
}
@@ -3065,6 +3104,7 @@ function init() {
$("#tts-close").addEventListener("click", () => {
$("#tts-panel").hidden = true;
});
$("#tts-preview").addEventListener("click", () => void previewSpeech());
$("#tts-speak").addEventListener("click", () => void speakText());
$("#tts-stop").addEventListener("click", async () => {
try {
poc/public/index.html
+6
View File
@@ -170,6 +170,11 @@
<input id="tts-pitch" type="range" min="-12" max="12" step="1" value="0" />
<span id="tts-pitch-value" class="vk-mono vk-dim">0 st</span>
</label>
<label class="vk-tts-slider" title="Playback volume for speech, 100% is the level as synthesized">
<span class="vk-micro-label">volume</span>
<input id="tts-volume" type="range" min="0" max="200" step="5" value="100" />
<span id="tts-volume-value" class="vk-mono vk-dim">100%</span>
</label>
<button id="profile-open" class="vk-chip vk-chip-off" type="button" title="Flextalk voice profile">
voice profile…
</button>
@@ -179,6 +184,7 @@
<div class="vk-tts-meta">
<span class="vk-mono vk-dim" id="tts-status">idle</span>
<button id="tts-stop" class="btn btn-secondary vk-small-btn" type="button">Stop</button>
<button id="tts-preview" class="btn btn-secondary vk-small-btn" type="button" title="Play this line in the browser, not in the call">Preview</button>
<button id="tts-speak" class="btn btn-primary vk-small-btn" type="button">Speak</button>
</div>
</div>
poc/src/audio-service.ts
+21 -4
View File
@@ -156,7 +156,7 @@ export class AudioService {
* grows. The wrapper touches `<file>.ready` after the first word, which is the reliable
* signal; a size threshold and a timeout cover providers that do not mark readiness.
*/
private async playWhenReady(file: string): Promise<void> {
private async playWhenReady(file: string, volume?: number): Promise<void> {
for (let attempt = 0; attempt < 400; attempt++) {
try {
if (statSync(`${file}.ready`).size >= 0) break;
@@ -172,7 +172,7 @@ export class AudioService {
}
try {
await this.manager.play(file);
await this.manager.play(file, volume);
} finally {
try {
unlinkSync(`${file}.ready`);
@@ -182,6 +182,23 @@ export class AudioService {
}
}
/** The TTS output directory, for serving preview files. */
get ttsDirectory(): string {
return this.tts.directory;
}
/**
* Synthesize a line for browser playback. This never touches the call:
* no queue, no ducking, no virtual microphone, no voice-processing bypass.
*/
async preview(text: string, options: TtsOptions = {}): Promise<string> {
const trimmed = text.trim();
if (!trimmed) throw new Error("text is required");
if (trimmed.length > 1000) throw new Error("text is too long (max 1000 characters)");
return this.tts.synthesize(trimmed, { ...options, stream: false, onOutputPath: undefined });
}
stop(): void {
this.queue.length = 0;
this.manager.stopPlayback();
@@ -402,7 +419,7 @@ export class AudioService {
options.onOutputPath = path => {
streaming = (async () => {
await this.applyDuck(this.shouldDuck());
await this.playWhenReady(path);
await this.playWhenReady(path, item.options.volume);
})();
};
}
@@ -413,7 +430,7 @@ export class AudioService {
await pending;
} else {
await this.applyDuck(this.shouldDuck());
await this.manager.play(file);
await this.manager.play(file, item.options.volume);
}
} catch (error) {
console.error("[audio] speak failed:", (error as Error).message);
poc/src/audio.ts
+12 -2
View File
@@ -276,7 +276,7 @@ export class AudioManager {
this.modules = [];
}
async play(file: string): Promise<void> {
async play(file: string, volume?: number): Promise<void> {
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay";
@@ -284,7 +284,17 @@ export class AudioManager {
// Lead-in silence: give Discord's voice pipeline a moment to settle
// before the first syllable, so the attack is not swallowed.
const args = ["-nodisp", "-autoexit", "-loglevel", "error"];
if (LEAD_MS > 0) args.push("-af", `adelay=${LEAD_MS}:all=1`);
const filters: string[] = [];
if (LEAD_MS > 0) filters.push(`adelay=${LEAD_MS}:all=1`);
// Volume percent, 0-200. 100 keeps the level as synthesized. ffplay's own
// -volume stops at 100, so use the volume filter and its dB scale.
if (typeof volume === "number" && Number.isFinite(volume) && volume !== 100) {
const gain = volume > 0 ? `${(20 * Math.log10(volume / 100)).toFixed(2)}dB` : "0";
filters.push(`volume=${gain}`);
}
if (filters.length) args.push("-af", filters.join(","));
args.push(file);
await new Promise<void>((resolve, reject) => {
poc/src/preferences.ts
+5
View File
@@ -14,6 +14,8 @@ export interface TtsPreferences {
provider: "local" | "espeak" | "command" | "api" | null;
language: string | null;
voice: string | null;
/** Playback volume percent, 0-200. Null uses the natural level. */
volume: number | null;
/** Speech-rate multiplier, 1.0 is the engine's natural rate. Null uses the engine default. */
speed: number | null;
/** Pitch offset in semitones, 0 is the engine's natural pitch. Null uses the engine default. */
@@ -176,6 +178,7 @@ const DEFAULTS: WebPreferences = {
provider: null,
language: null,
voice: null,
volume: null,
speed: null,
pitch: null,
expression: null,
@@ -416,6 +419,7 @@ export class PreferencesStore {
provider: normalizeProvider(patch.tts?.provider),
language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null,
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null,
volume: normalizeRange(patch.tts?.volume, 0, 200),
speed: normalizeSpeed(patch.tts?.speed),
pitch: normalizePitch(patch.tts?.pitch),
expression: normalizeExpression(patch.tts?.expression),
@@ -500,6 +504,7 @@ export class PreferencesStore {
provider: normalizeProvider(provider),
language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null,
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null,
volume: normalizeRange(raw?.tts?.volume, 0, 200),
speed: normalizeSpeed(raw?.tts?.speed),
pitch: normalizePitch(raw?.tts?.pitch),
expression: normalizeExpression(raw?.tts?.expression),
poc/src/tts.ts
+2
View File
@@ -18,6 +18,8 @@ export type TtsProvider = "local" | "espeak" | "command" | "api";
export interface TtsOptions {
provider?: TtsProvider;
voice?: string;
/** Playback volume percent, 0-200. The daemon applies it at play time. */
volume?: number;
/** Speech-rate multiplier; the engine maps it to its own units. */
speed?: number;
/** Pitch offset in semitones; the engine maps it to its own units. */
poc/src/webui.ts
+104 -66
View File
@@ -14,7 +14,7 @@ import { existsSync, readFileSync } from "node:fs";
import { Readable } from "node:stream";
import http from "node:http";
import https from "node:https";
import { dirname } from "node:path";
import { basename, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import QRCode from "qrcode";
import { WebSocketServer } from "ws";
@@ -25,11 +25,11 @@ import { CoreError, type CoreClient } from "./core-client.js";
import type { CoreProcess } from "./core-process.js";
import { MediaLimitError, type MediaService } from "./media.js";
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js";
import type { PreferencesStore, WebPreferences } from "./preferences.js";
import type { PreferencesStore, TtsPreferences, WebPreferences } from "./preferences.js";
import type { Puppet } from "./puppet.js";
import { ensureTls } from "./tls.js";
import type { TranscriptStore } from "./transcript.js";
import type { TtsProvider, TtsVoice } from "./tts.js";
import type { TtsOptions, TtsProvider, TtsVoice } from "./tts.js";
const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"];
const LANGUAGE_NAMES: Record<string, string> = {
@@ -231,6 +231,77 @@ function parseMediaFilters(query: Record<string, unknown>) {
};
}
/**
* Turn a speak/preview request body into TTS options. Missing fields fall back
* to the stored preferences.
*/
function buildSpeechOptions(body: any, tts: TtsPreferences, voice: string | undefined): TtsOptions {
const requested = body?.provider;
const provider = ["api", "local", "espeak", "command"].includes(requested)
? requested
: tts.provider;
// Number(null) is 0: null, undefined, and "" all mean "unset".
const optional = (value: unknown): number | undefined => {
if (value === undefined || value === null || value === "") return undefined;
const number = Number(value);
return Number.isFinite(number) ? number : undefined;
};
const speed = optional(body?.speed) ?? tts.speed ?? undefined;
const pitch = optional(body?.pitch) ?? tts.pitch ?? undefined;
const expression = optional(body?.expression) ?? tts.expression ?? undefined;
const profile = body?.profile && typeof body.profile === "object" ? body.profile : tts.profile;
const highpass = optional(body?.highpass) ?? tts.highpass ?? undefined;
const echo = optional(body?.echo) ?? tts.echo ?? undefined;
const words = body?.words !== undefined ? body.words === true : tts.words;
const wordGapMs = optional(body?.wordGapMs) ?? tts.wordGapMs ?? undefined;
// The Half-Life VOX voice carries its prosody with it.
const vox =
(body?.vox !== undefined ? body.vox === true : tts.vox) || voice === "vox";
const voxClean = body?.voxClean !== undefined ? body.voxClean === true : tts.voxClean;
const preset =
body?.preset === "vox" || body?.preset === "hev"
? body.preset
: body?.preset === null || body?.preset === ""
? null
: tts.preset;
const echoDelayMs = optional(body?.echoDelayMs) ?? tts.echoDelayMs ?? undefined;
const echoLevel = optional(body?.echoLevel) ?? tts.echoLevel ?? undefined;
const crunch =
["hl90", "hl90hq", "telephone", "adpcm", "crush"].includes(String(body?.crunch ?? ""))
? (String(body.crunch) as "hl90" | "hl90hq" | "telephone" | "adpcm" | "crush")
: body?.crunch === null || body?.crunch === "" || body?.crunch === "off"
? undefined
: tts.crunch ?? undefined;
const crunchBits = optional(body?.crunchBits) ?? tts.crunchBits ?? undefined;
const crunchRate = optional(body?.crunchRate) ?? tts.crunchRate ?? undefined;
const streamWords =
body?.streamWords !== undefined ? body.streamWords !== false : tts.streamWords;
return {
provider: provider ?? undefined,
voice,
...(Number.isFinite(speed) ? { speed } : {}),
...(Number.isFinite(pitch) ? { pitch } : {}),
...(Number.isFinite(expression) ? { expression } : {}),
...(profile && Object.keys(profile).length ? { profile } : {}),
...(Number.isFinite(highpass) ? { highpass } : {}),
...(Number.isFinite(echo) ? { echo } : {}),
...(words ? { words: true } : {}),
...(Number.isFinite(wordGapMs) ? { wordGapMs } : {}),
...(vox ? { vox: true } : {}),
...(voxClean ? { voxClean: true } : {}),
...(preset ? { preset } : {}),
...(Number.isFinite(echoDelayMs) ? { echoDelayMs } : {}),
...(Number.isFinite(echoLevel) ? { echoLevel } : {}),
...(crunch ? { crunch } : {}),
...(Number.isFinite(crunchBits) ? { crunchBits } : {}),
...(Number.isFinite(crunchRate) ? { crunchRate } : {}),
stream: streamWords && words === true
} as TtsOptions;
}
export function startWebUi(deps: WebUiDeps): WebUiHandle {
const app = express();
app.use(express.json({ limit: "64kb" }));
@@ -508,6 +579,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
provider: tts?.provider ?? null,
language: tts?.language ?? null,
voice: tts?.voice ?? null,
volume: tts?.volume ?? null,
speed: tts?.speed ?? null,
pitch: tts?.pitch ?? null,
expression: tts?.expression ?? null,
@@ -794,80 +866,46 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
const text = String(req.body?.text ?? "");
const prefs = deps.preferences.get();
const requested = req.body?.provider;
const provider = ["api", "local", "espeak", "command"].includes(requested)
? requested
: prefs.tts.provider;
const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
// Number(null) is 0: null, undefined, and "" all mean "unset".
const optional = (value: unknown): number | undefined => {
if (value === undefined || value === null || value === "") return undefined;
const number = Number(value);
return Number.isFinite(number) ? number : undefined;
};
const options = buildSpeechOptions(req.body ?? {}, prefs.tts, voice);
const speed = optional(req.body?.speed) ?? prefs.tts.speed ?? undefined;
const pitch = optional(req.body?.pitch) ?? prefs.tts.pitch ?? undefined;
const expression = optional(req.body?.expression) ?? prefs.tts.expression ?? undefined;
const profile = req.body?.profile && typeof req.body.profile === "object" ? req.body.profile : prefs.tts.profile;
const highpass = optional(req.body?.highpass) ?? prefs.tts.highpass ?? undefined;
const echo = optional(req.body?.echo) ?? prefs.tts.echo ?? undefined;
const words = req.body?.words !== undefined ? req.body.words === true : prefs.tts.words;
const wordGapMs = optional(req.body?.wordGapMs) ?? prefs.tts.wordGapMs ?? undefined;
// The Half-Life VOX voice carries its prosody with it.
const vox =
(req.body?.vox !== undefined ? req.body.vox === true : prefs.tts.vox) || voice === "vox";
const voxClean = req.body?.voxClean !== undefined ? req.body.voxClean === true : prefs.tts.voxClean;
const preset =
req.body?.preset === "vox" || req.body?.preset === "hev"
? req.body.preset
: req.body?.preset === null || req.body?.preset === ""
? null
: prefs.tts.preset;
const echoDelayMs = optional(req.body?.echoDelayMs) ?? prefs.tts.echoDelayMs ?? undefined;
const echoLevel = optional(req.body?.echoLevel) ?? prefs.tts.echoLevel ?? undefined;
const crunch =
["hl90", "hl90hq", "telephone", "adpcm", "crush"].includes(String(req.body?.crunch ?? ""))
? (String(req.body.crunch) as "hl90" | "hl90hq" | "telephone" | "adpcm" | "crush")
: req.body?.crunch === null || req.body?.crunch === "" || req.body?.crunch === "off"
? undefined
: prefs.tts.crunch ?? undefined;
const crunchBits = optional(req.body?.crunchBits) ?? prefs.tts.crunchBits ?? undefined;
const crunchRate = optional(req.body?.crunchRate) ?? prefs.tts.crunchRate ?? undefined;
const streamWords =
req.body?.streamWords !== undefined ? req.body.streamWords !== false : prefs.tts.streamWords;
const result = await deps.audio.speak(text, {
provider: provider ?? undefined,
voice,
...(Number.isFinite(speed) ? { speed } : {}),
...(Number.isFinite(pitch) ? { pitch } : {}),
...(Number.isFinite(expression) ? { expression } : {}),
...(profile && Object.keys(profile).length ? { profile } : {}),
...(Number.isFinite(highpass) ? { highpass } : {}),
...(Number.isFinite(echo) ? { echo } : {}),
...(words ? { words: true } : {}),
...(Number.isFinite(wordGapMs) ? { wordGapMs } : {}),
...(vox ? { vox: true } : {}),
...(voxClean ? { voxClean: true } : {}),
...(preset ? { preset } : {}),
...(Number.isFinite(echoDelayMs) ? { echoDelayMs } : {}),
...(Number.isFinite(echoLevel) ? { echoLevel } : {}),
...(crunch ? { crunch } : {}),
...(Number.isFinite(crunchBits) ? { crunchBits } : {}),
...(Number.isFinite(crunchRate) ? { crunchRate } : {}),
stream: streamWords && words === true
});
const result = await deps.audio.speak(text, options);
deps.getPuppet()?.appendAction({
type: "audio",
action: "speak",
provider: provider ?? deps.audio.status().tts.defaultProvider,
provider: options.provider ?? deps.audio.status().tts.defaultProvider,
chars: text.trim().length
});
res.json({ ok: true, result });
})
);
app.post(
"/api/audio/preview",
wrap(async (req, res) => {
const text = String(req.body?.text ?? "").trim();
const prefs = deps.preferences.get();
const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
const file = await deps.audio.preview(text, buildSpeechOptions(req.body ?? {}, prefs.tts, voice));
res.json({ ok: true, result: { url: `/api/audio/tts/${encodeURIComponent(basename(file))}` } });
})
);
// Preview files only: the name pattern blocks path traversal.
app.get(
"/api/audio/tts/:name",
wrap(async (req, res) => {
const name = String(req.params.name);
if (!/^tts-[0-9]+-[0-9]+\.wav$/.test(name)) throw new HttpError(400, "invalid audio name");
const file = join(deps.audio.ttsDirectory, name);
if (!existsSync(file)) throw new HttpError(404, "audio not found");
res.type("audio/wav").sendFile(file);
})
);
app.post(
"/api/audio/stop",
wrap(async (_req, res) => {