440 lines
17 KiB
TypeScript
440 lines
17 KiB
TypeScript
/*
|
|
* Veskslop daemon — text to speech.
|
|
*
|
|
* Providers: a local engine (espeak-ng when installed, or VESKSLOP_TTS_CMD)
|
|
* and an HTTP API. The local command receives an output path ({out}), a text
|
|
* file ({text_file}), and optionally a voice ({voice}).
|
|
*/
|
|
|
|
import { spawn } from "node:child_process";
|
|
import { mkdirSync, unlinkSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
export class TtsError extends Error {}
|
|
|
|
export type TtsProvider = "local" | "espeak" | "command" | "api";
|
|
|
|
export interface TtsOptions {
|
|
provider?: TtsProvider;
|
|
voice?: string;
|
|
/** Speech-rate multiplier; the engine maps it to its own units. */
|
|
speed?: number;
|
|
/** Pitch offset in semitones; the engine maps it to its own units. */
|
|
pitch?: number;
|
|
/** Flextalk Expression profile value, 0 to 100. */
|
|
expression?: number;
|
|
/** Flextalk profile overrides: pitch, rate, volume, expression, prominences, head_size, lowering. */
|
|
profile?: Record<string, number>;
|
|
/** High-pass filter Hz, 40 to 400. */
|
|
highpass?: number;
|
|
/** Mono echo amount, 0 to 100. */
|
|
echo?: number;
|
|
/** Synthesize each word and merge with silence gaps. */
|
|
words?: boolean;
|
|
/** Silence between words in milliseconds, 0 to 500. */
|
|
wordGapMs?: number;
|
|
}
|
|
|
|
export interface TtsEngine {
|
|
id: "espeak" | "command" | "api";
|
|
name: string;
|
|
available: boolean;
|
|
/** false when the engine has no language dimension (single-language command engines). */
|
|
language: boolean;
|
|
}
|
|
|
|
export interface TtsStatus {
|
|
local: boolean;
|
|
api: boolean;
|
|
espeak: boolean;
|
|
command: boolean;
|
|
defaultProvider: TtsProvider;
|
|
engine: string | null;
|
|
engines: TtsEngine[];
|
|
}
|
|
|
|
export interface TtsVoice {
|
|
id: string;
|
|
name: string;
|
|
language: string;
|
|
gender: "male" | "female" | null;
|
|
/** false when the engine ignores pitch for this voice (for example Flextalk). */
|
|
supportsPitch?: boolean;
|
|
/** true when the voice has a Flextalk-style Expression profile parameter. */
|
|
supportsExpression?: boolean;
|
|
/** Registry default for the expression parameter, when known. */
|
|
expressionDefault?: number;
|
|
/** Per-voice Flextalk profile defaults (pitch, rate, prominences, head_size, lowering, expression). */
|
|
profileDefaults?: Record<string, number>;
|
|
}
|
|
|
|
/** espeak-ng voice variants, appended to every base voice. */
|
|
const ESPEAK_VARIANTS = ["m1", "m2", "m3", "m4", "m5", "m6", "m7", "f1", "f2", "f3", "f4", "f5", "croak", "whisper"];
|
|
|
|
interface CommandResult {
|
|
code: number | null;
|
|
stdout: string;
|
|
stderr: string;
|
|
}
|
|
|
|
function run(command: string, args: string[]): Promise<CommandResult> {
|
|
return new Promise(resolve => {
|
|
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
let stdout = "";
|
|
let stderr = "";
|
|
|
|
child.stdout?.on("data", chunk => (stdout += chunk));
|
|
child.stderr?.on("data", chunk => (stderr += chunk));
|
|
child.on("error", error => resolve({ code: null, stdout, stderr: String(error) }));
|
|
child.on("close", code => resolve({ code, stdout, stderr }));
|
|
});
|
|
}
|
|
|
|
function parseVoices(output: string): TtsVoice[] {
|
|
const lines = output.split("\n");
|
|
const header = lines[0] ?? "";
|
|
const languageIndex = header.indexOf("Language");
|
|
const ageIndex = header.indexOf("Age/Gender");
|
|
const nameIndex = header.indexOf("VoiceName");
|
|
const fileIndex = header.indexOf("File");
|
|
|
|
if (languageIndex < 0 || ageIndex < 0 || nameIndex < 0 || fileIndex < 0) return [];
|
|
|
|
const voices: TtsVoice[] = [];
|
|
for (const line of lines.slice(1)) {
|
|
const language = line.slice(languageIndex, ageIndex).trim();
|
|
const ageGender = line.slice(ageIndex, nameIndex).trim();
|
|
const name = line.slice(nameIndex, fileIndex).trim();
|
|
if (!language || !name) continue;
|
|
|
|
voices.push({
|
|
id: language,
|
|
name,
|
|
language,
|
|
gender: ageGender.endsWith("F") ? "female" : ageGender.endsWith("M") ? "male" : null
|
|
});
|
|
}
|
|
|
|
return voices;
|
|
}
|
|
|
|
/**
|
|
* Command-mode voice list. Entries are separated by commas or newlines. Each entry is
|
|
* `id`, `id|Name`, `id|Name|gender`, `id|Name|gender|language`, or
|
|
* `id|Name|gender|language|flags`, where flags are `+`-separated. Flags: `no-pitch`
|
|
* (the engine ignores pitch) and `expression` / `expression=NN` (Flextalk profile).
|
|
*/
|
|
function parseEngineVoices(raw: string): TtsVoice[] {
|
|
return raw
|
|
.split(/[,\n]+/)
|
|
.map(part => part.trim())
|
|
.filter(Boolean)
|
|
.map((entry): TtsVoice => {
|
|
const [id, name, gender, language, flags] = entry.split("|").map(part => part?.trim());
|
|
const flagList = flags?.split("+").map(flag => flag.trim()) ?? [];
|
|
const expression = flagList.find(flag => flag === "expression" || flag.startsWith("expression="));
|
|
const defaults = flagList.find(flag => flag.startsWith("defaults="));
|
|
|
|
let profileDefaults: Record<string, number> | undefined;
|
|
if (defaults) {
|
|
const parts = defaults.slice("defaults=".length).split(":");
|
|
profileDefaults = {};
|
|
for (let index = 0; index + 1 < parts.length; index += 2) {
|
|
const value = Number(parts[index + 1]);
|
|
if (Number.isFinite(value)) profileDefaults[parts[index]] = value;
|
|
}
|
|
}
|
|
|
|
const expressionDefault = profileDefaults?.expression ??
|
|
(expression?.includes("=") ? Number(expression.split("=")[1]) : undefined);
|
|
|
|
return {
|
|
id,
|
|
name: name || id,
|
|
language: language || "en",
|
|
gender: gender === "male" || gender === "female" ? gender : null,
|
|
supportsPitch: !flagList.includes("no-pitch"),
|
|
supportsExpression: !!expression || !!profileDefaults,
|
|
...(Number.isFinite(expressionDefault) ? { expressionDefault } : {}),
|
|
...(profileDefaults ? { profileDefaults } : {})
|
|
};
|
|
});
|
|
}
|
|
|
|
/** Serialize a profile object to the wrapper's `key=value,key=value` form. */
|
|
function serializeProfile(profile?: Record<string, number>): string {
|
|
if (!profile) return "";
|
|
return Object.entries(profile)
|
|
.map(([key, value]) => `${key}=${value}`)
|
|
.join(",");
|
|
}
|
|
|
|
export class Tts {
|
|
private readonly dir: string;
|
|
private counter = 0;
|
|
private espeak = false;
|
|
private readonly voiceCache = new Map<string, TtsVoice[]>();
|
|
|
|
constructor(cacheDir?: string) {
|
|
const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache");
|
|
this.dir = join(base, "audio");
|
|
try {
|
|
mkdirSync(this.dir, { recursive: true });
|
|
} catch {
|
|
// the fallback directory works
|
|
}
|
|
}
|
|
|
|
get localCommand(): string | undefined {
|
|
return process.env.VESKSLOP_TTS_CMD;
|
|
}
|
|
|
|
get apiUrl(): string | undefined {
|
|
return process.env.VESKSLOP_TTS_API_URL;
|
|
}
|
|
|
|
get directory(): string {
|
|
return this.dir;
|
|
}
|
|
|
|
/** Display name for the configured command engine. */
|
|
get commandName(): string {
|
|
return process.env.VESKSLOP_TTS_NAME ?? "custom engine";
|
|
}
|
|
|
|
async detect(): Promise<void> {
|
|
const result = await run("espeak-ng", ["--version"]);
|
|
this.espeak = result.code === 0;
|
|
}
|
|
|
|
engines(): TtsEngine[] {
|
|
return [
|
|
{ id: "espeak", name: "espeak-ng", available: this.espeak, language: true },
|
|
{ id: "command", name: this.commandName, available: !!this.localCommand, language: false },
|
|
{ id: "api", name: "HTTP API", available: !!this.apiUrl, language: true }
|
|
];
|
|
}
|
|
|
|
private autoProvider(): TtsProvider {
|
|
if (this.localCommand) return "command";
|
|
if (this.espeak) return "espeak";
|
|
return this.apiUrl ? "api" : "local";
|
|
}
|
|
|
|
status(): TtsStatus {
|
|
const local = !!this.localCommand || this.espeak;
|
|
const override = process.env.VESKSLOP_TTS_PROVIDER;
|
|
const allowed = new Set<TtsProvider>(["local", "espeak", "command", "api"]);
|
|
|
|
return {
|
|
local,
|
|
api: !!this.apiUrl,
|
|
espeak: this.espeak,
|
|
command: !!this.localCommand,
|
|
defaultProvider: allowed.has(override as TtsProvider) ? (override as TtsProvider) : this.autoProvider(),
|
|
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null,
|
|
engines: this.engines()
|
|
};
|
|
}
|
|
|
|
async voices(engine?: TtsProvider): Promise<TtsVoice[]> {
|
|
const key = engine ?? this.autoProvider();
|
|
const cached = this.voiceCache.get(key);
|
|
if (cached) return cached;
|
|
|
|
let voices: TtsVoice[] = [];
|
|
|
|
if (key === "command" || (key === "local" && this.localCommand)) {
|
|
const declared = process.env.VESKSLOP_TTS_VOICES;
|
|
voices = declared ? parseEngineVoices(declared) : [];
|
|
} else if (key === "espeak" || key === "local") {
|
|
voices = await this.espeakVoices();
|
|
}
|
|
|
|
this.voiceCache.set(key, voices);
|
|
return voices;
|
|
}
|
|
|
|
private async espeakVoices(): Promise<TtsVoice[]> {
|
|
if (!this.espeak) return [];
|
|
|
|
const result = await run("espeak-ng", ["--voices"]);
|
|
const base = result.code === 0 ? parseVoices(result.stdout) : [];
|
|
|
|
return base.flatMap((voice): TtsVoice[] => [
|
|
voice,
|
|
...ESPEAK_VARIANTS.map(
|
|
(variant): TtsVoice => ({
|
|
id: `${voice.id}+${variant}`,
|
|
name: `${voice.name} (${variant})`,
|
|
language: voice.language,
|
|
gender: variant.startsWith("m") ? "male" : variant.startsWith("f") ? "female" : null
|
|
})
|
|
)
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Start the engine so the first real line is fast: run one throwaway synthesis
|
|
* (the command wrapper starts Xvfb, Wine, and the host), then delete the file.
|
|
*/
|
|
async warm(engine?: TtsProvider): Promise<{ engine: TtsProvider; ms: number; warmed: boolean }> {
|
|
const key = engine ?? this.autoProvider();
|
|
const start = Date.now();
|
|
|
|
if (key === "command" && !this.localCommand) {
|
|
throw new TtsError("the custom TTS engine is not configured (VESKSLOP_TTS_CMD)");
|
|
}
|
|
|
|
const usesCommand = key === "command" || (key === "local" && !!this.localCommand);
|
|
if (!usesCommand) {
|
|
// espeak starts instantly and the HTTP provider is expected to stay warm.
|
|
return { engine: key, ms: 0, warmed: false };
|
|
}
|
|
|
|
const voice = process.env.VESKSLOP_TTS_VOICE || undefined;
|
|
const file = await this.viaLocalCommand("The quick brown fox jumps over the lazy dog.", {
|
|
provider: key,
|
|
voice
|
|
});
|
|
|
|
try {
|
|
unlinkSync(file);
|
|
} catch {
|
|
// best effort
|
|
}
|
|
|
|
return { engine: key, ms: Date.now() - start, warmed: true };
|
|
}
|
|
|
|
async synthesize(text: string, options: TtsOptions = {}): Promise<string> {
|
|
const provider = options.provider ?? this.status().defaultProvider;
|
|
|
|
if (provider === "api") return this.viaApi(text, options);
|
|
if (provider === "espeak") return this.viaLocalCommand(text, options, true);
|
|
if (provider === "command") return this.viaLocalCommand(text, options, false);
|
|
|
|
return this.viaLocalCommand(text, options);
|
|
}
|
|
|
|
private async viaLocalCommand(text: string, options: TtsOptions, forceEspeak = false): Promise<string> {
|
|
const command = forceEspeak ? undefined : this.localCommand;
|
|
if (!command && !this.espeak) {
|
|
throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)");
|
|
}
|
|
if (command && !command.includes("{out}")) {
|
|
throw new TtsError("VESKSLOP_TTS_CMD must contain {out}");
|
|
}
|
|
|
|
const out = this.nextPath("wav");
|
|
const textFile = this.nextPath("txt");
|
|
writeFileSync(textFile, text);
|
|
|
|
const parts = command
|
|
? command
|
|
.split(/\s+/)
|
|
.filter(Boolean)
|
|
.map(part =>
|
|
part
|
|
.replace("{out}", out)
|
|
.replace("{text_file}", textFile)
|
|
.replace("{voice}", options.voice ?? "")
|
|
.replace("{speed}", options.speed !== undefined ? String(options.speed) : "")
|
|
.replace("{pitch}", options.pitch !== undefined ? String(options.pitch) : "")
|
|
.replace("{expression}", options.expression !== undefined ? String(options.expression) : "")
|
|
.replace("{profile}", serializeProfile(options.profile))
|
|
.replace("{highpass}", options.highpass !== undefined ? String(options.highpass) : "")
|
|
.replace("{echo}", options.echo !== undefined ? String(options.echo) : "")
|
|
.replace("{words}", options.words ? "1" : "")
|
|
.replace("{word_gap_ms}", options.wordGapMs !== undefined ? String(options.wordGapMs) : "")
|
|
)
|
|
: ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile];
|
|
|
|
const args = parts.filter(Boolean);
|
|
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn(args[0], args.slice(1), {
|
|
env: {
|
|
...process.env,
|
|
VESKSLOP_TTS_VOICE: options.voice ?? "",
|
|
...(options.speed !== undefined ? { VESKSLOP_TTS_SPEED: String(options.speed) } : {}),
|
|
...(options.pitch !== undefined ? { VESKSLOP_TTS_PITCH: String(options.pitch) } : {}),
|
|
...(options.expression !== undefined
|
|
? { VESKSLOP_TTS_EXPRESSION: String(options.expression) }
|
|
: {}),
|
|
...(options.profile && Object.keys(options.profile).length
|
|
? { VESKSLOP_TTS_PROFILE: serializeProfile(options.profile) }
|
|
: {}),
|
|
...(options.highpass !== undefined ? { VESKSLOP_TTS_HIGHPASS: String(options.highpass) } : {}),
|
|
...(options.echo !== undefined ? { VESKSLOP_TTS_ECHO: String(options.echo) } : {}),
|
|
...(options.words ? { VESKSLOP_TTS_WORDS: "1" } : {}),
|
|
...(options.wordGapMs !== undefined
|
|
? { VESKSLOP_TTS_WORD_GAP_MS: String(options.wordGapMs) }
|
|
: {})
|
|
},
|
|
stdio: ["ignore", "ignore", "pipe"]
|
|
});
|
|
let stderr = "";
|
|
|
|
child.stderr?.on("data", chunk => (stderr += chunk));
|
|
child.on("error", error => reject(new TtsError(String(error))));
|
|
child.on("close", code => {
|
|
if (code === 0) resolve();
|
|
else reject(new TtsError(stderr.trim() || `the TTS command exited with ${code}`));
|
|
});
|
|
});
|
|
|
|
return out;
|
|
}
|
|
|
|
private async viaApi(text: string, options: TtsOptions): Promise<string> {
|
|
const url = this.apiUrl;
|
|
if (!url) throw new TtsError("VESKSLOP_TTS_API_URL is not set");
|
|
|
|
const response = await fetch(url, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
...(process.env.VESKSLOP_TTS_API_KEY
|
|
? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` }
|
|
: {})
|
|
},
|
|
body: JSON.stringify({
|
|
text,
|
|
...(options.voice ? { voice: options.voice } : {}),
|
|
...(options.speed !== undefined ? { speed: options.speed } : {}),
|
|
...(options.pitch !== undefined ? { pitch: options.pitch } : {}),
|
|
...(options.expression !== undefined ? { expression: options.expression } : {}),
|
|
...(options.profile && Object.keys(options.profile).length ? { profile: options.profile } : {}),
|
|
...(options.highpass !== undefined ? { highpass: options.highpass } : {}),
|
|
...(options.echo !== undefined ? { echo: options.echo } : {}),
|
|
...(options.words ? { words: true } : {}),
|
|
...(options.wordGapMs !== undefined ? { wordGapMs: options.wordGapMs } : {})
|
|
})
|
|
});
|
|
|
|
if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`);
|
|
|
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
const type = response.headers.get("content-type") ?? "";
|
|
const extension =
|
|
type.includes("mpeg") || type.includes("mp3")
|
|
? "mp3"
|
|
: type.includes("ogg")
|
|
? "ogg"
|
|
: type.includes("wav")
|
|
? "wav"
|
|
: "mp3";
|
|
|
|
const out = this.nextPath(extension);
|
|
writeFileSync(out, buffer);
|
|
return out;
|
|
}
|
|
|
|
private nextPath(extension: string): string {
|
|
this.counter++;
|
|
return join(this.dir, `tts-${Date.now()}-${this.counter}.${extension}`);
|
|
}
|
|
}
|