From e45e0832050423f35d67052afc23657e916fb2d1 Mon Sep 17 00:00:00 2001 From: Fisto Date: Wed, 16 Sep 2026 08:04:13 +0200 Subject: [PATCH] VOX prosody: NetTTS parser port, wrapper VOX mode, docs --- README.md | 5 + docs/NOTES.md | 13 + docs/TTS-DESIGN.md | 46 +++- setup/veskslop-tts.sh | 33 ++- setup/vox_parser.py | 541 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 628 insertions(+), 10 deletions(-) create mode 100644 setup/vox_parser.py diff --git a/README.md b/README.md index b1ef54d..38d88b8 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ load: the wrapper writes it to the Wine registry before synthesis. `--voices` ma voices with `expression=NN`, where NN is the registry default. Mapping details are in `docs/TTS-DESIGN.md`. +**VOX prosody** (NetTTS-style, Flextalk voices): set `VESKSLOP_TTS_VOX=1` in the +command environment (`VESKSLOP_TTS_VOX_CLEAN=1` omits the `\!wH1/\!wH0` stutter +wrapper). See `docs/TTS-DESIGN.md`. + ## Layout - `host/sapi4host.c` — host: voice select, pitch/speed, text -> WAV @@ -75,6 +79,7 @@ voices with `expression=NN`, where NN is the registry default. Mapping details a - `setup/inf2reg.py` — INF AddReg -> `.reg` converter - `setup/run.sh` — run helper (prefix + Xvfb) - `setup/veskslop-tts.sh` — Veskslop command-provider wrapper (post-processing included) +- `setup/vox_parser.py` — NetTTS-style VOX prosody encoder (Flextalk voices) - `docs/NOTES.md` — registry/CLSID facts, runtime RE notes, gotchas - `docs/TTS-DESIGN.md` — integration contract with Veskslop plus engine clarifications diff --git a/docs/NOTES.md b/docs/NOTES.md index de90d3b..33c7bf7 100644 --- a/docs/NOTES.md +++ b/docs/NOTES.md @@ -93,6 +93,19 @@ final file `WatsonUNInstall.dll` is incomplete). verified audible (0/59/100 produce different waveforms at equal duration). Changing them requires a registry edit and re-selecting the voice. +## VOX parser + +- `setup/vox_parser.py` reimplements the NetTTS VOX prosody encoder (vendor tags + `\!wH1/\!wH0`, `\!br`, `\!sfNN`); the host already passes `TTSDATAFLAG_TAGGED`, so + the engine honors them. +- Validation: the reference `vox_parser.cpp` from a local clone of + `h4rm0n1c/NetTTS` was compiled with g++ in a throwaway harness and diffed against + the Python port; outputs are identical on the 31-line corpus. Re-check after any + parser change while the clone is available. +- VOX mode is Flextalk-only. The wrapper enables it with `VESKSLOP_TTS_VOX=1` + (`VESKSLOP_TTS_VOX_CLEAN=1` drops the stutter wrapper) and it takes precedence + over `VESKSLOP_TTS_WORDS`. + ## Decompiled corpora (Ghidra) - `/mnt/ghidra/decomp/msttssyn.dll/` (786 functions, engine) diff --git a/docs/TTS-DESIGN.md b/docs/TTS-DESIGN.md index 28a99f5..ef22846 100644 --- a/docs/TTS-DESIGN.md +++ b/docs/TTS-DESIGN.md @@ -218,7 +218,7 @@ record what the engine needs to interpret, add, or hide._ so one shared level target is required. The engine post-processes every line: resample to 48000 Hz, trim leading silence (<100 ms), 5-10 ms fades, DC removal, and level normalization (peak about -3 dBFS, integrated loudness -16 to -20 LUFS). -- Implemented in `setup/vekslop-tts.sh` with ffmpeg: high-pass at 40 Hz, silence trim at +- Implemented in `setup/veskslop-tts.sh` with ffmpeg: high-pass at 40 Hz, silence trim at both ends (trim-start, reverse, trim-start, reverse), 8 ms fades, then a **fixed gain** computed from an EBU R128 measurement (`ebur128=peak=true`): `gain = min(-16 LUFS target, -3 dBFS true-peak clamp)`, then 48 kHz mono s16le. @@ -253,7 +253,7 @@ record what the engine needs to interpret, add, or hide._ engine requires `REG_DWORD` for numeric speaker values; with strings it synthesizes a monotone buzz at maximum pitch. Fixed in `setup/setup-flextalk.sh`; no `no-pitch` flag is needed. -- Mapping implemented in `setup/vekslop-tts.sh`: `speed 1.0` = natural rate 150 for every +- Mapping implemented in `setup/veskslop-tts.sh`: `speed 1.0` = natural rate 150 for every voice, scaled by the multiplier and clamped to the engine range. `pitch` semitones scale the voice's natural pitch unit by `2^(st/12)` and clamp to the range. Natural units are about 100 (MSTTS), 106 (Keith Bell) and 186 (Kerry Watson). The engines' F0 response is @@ -335,14 +335,50 @@ How to apply a change: 5. Runtime `ITTSAttributes` pitch/speed/volume layer on top of the profile: the profile defines the natural values, the attributes shift them per request. +### VOX mode (NetTTS-inspired, Flextalk only) + +`VESKSLOP_TTS_VOX=1` routes the text through a rule-based prosody encoder before +synthesis. `setup/vox_parser.py` is an independent reimplementation of the VOX parser +from [h4rm0n1c/NetTTS](https://github.com/h4rm0n1c/NetTTS), which models Half-Life's +Black Mesa announcement delivery. It was verified byte-identical to the NetTTS parser +on a 31-line corpus (the reference was built locally with g++ for the comparison). + +What it does: sentence splitting; `the` -> `thee` before prepositions, proper nouns, +single letters or numbers; lead-in breaks ("Now...", "Please...", "A reminder..."); +time/number/degree normalization; token weighting (light/medium/heavy) with break +insertion at linkers, title-case runs, heavy gates and a 3-beat limiter; letter +tokens (`A` -> `Ay:`, `C` -> `C:`); `\!sf500 \!br` at sentence ends; and the +`\!wH1 ... \!wH0` staccato wrapper (disable with `VESKSLOP_TTS_VOX_CLEAN=1`). + +Requirements and scope: + +- Flextalk voices only: `\!br`, `\!sfNN` and `\!wH1/\!wH0` are FlexTalk vendor tags. + The host passes `TTSDATAFLAG_TAGGED`, so the engine honors them. Other voices ignore + `VESKSLOP_TTS_VOX`. +- Needs `python3` at runtime; the wrapper fails loudly when it is missing. +- Mutually exclusive with `VESKSLOP_TTS_WORDS` (VOX wins). +- Measured on the dev host for "Now approaching Sector C test lab. Please stand clear + of the test chamber.": plain 4.70 s with 1 pause, VOX 7.81 s with 8 pauses. + +Other NetTTS features and their relevance: + +- Inline `/rate N`, `/pitch N`, `[[pause N]]`: not ported. Veskslop has speed/pitch + controls and the profile window; `[[pause N]]` could be added as a wrapper-level + text splitter later (NetTTS supports it only outside VOX mode). +- TCP server with `START`/`STOP` status sockets: not needed; Veskslop owns playback + and knows when a line ends. +- Winetricks prefix automation (`winxp`, `vcrun6`, `mfc42`, `riched20`): not needed; + our prefix runs the engines with builtin Wine libraries. +- Win32 GUI and preset sliders: Veskslop provides the UI. + ### Transport and latency - Installation is one command: `setup/install.sh` (system dependencies with sudo, wine prefix + SAPI4 runtime + MSTTS voices, Flextalk extraction/registration, host build). Re-running is safe; a fresh machine needs nothing else. -- Command provider, implemented as `setup/vekslop-tts.sh`: - `VESKSLOP_TTS_CMD="/setup/vekslop-tts.sh {out} {text_file}"` and - `VESKSLOP_TTS_VOICES="$(/setup/vekslop-tts.sh --voices)"`. The wrapper reads the +- Command provider, implemented as `setup/veskslop-tts.sh`: + `VESKSLOP_TTS_CMD="/setup/veskslop-tts.sh {out} {text_file}"` and + `VESKSLOP_TTS_VOICES="$(/setup/veskslop-tts.sh --voices)"`. The wrapper reads the text file, transliterates UTF-8 to CP1252, starts or reuses an Xvfb when `DISPLAY` is unset, calls the engine, then post-processes (trim, fades, measured fixed gain, 48 kHz mono s16le). A stale private Xvfb is retried once. diff --git a/setup/veskslop-tts.sh b/setup/veskslop-tts.sh index 6b96170..b3e4dd1 100755 --- a/setup/veskslop-tts.sh +++ b/setup/veskslop-tts.sh @@ -18,6 +18,8 @@ # VESKSLOP_TTS_ECHO mono echo amount 0-100 (default 0) # VESKSLOP_TTS_WORDS 1 = synthesize each word, merge (Flextalk) # VESKSLOP_TTS_WORD_GAP_MS silence between words in ms (0-500) +# VESKSLOP_TTS_VOX 1 = VOX prosody, NetTTS-style (Flextalk) +# VESKSLOP_TTS_VOX_CLEAN 1 = VOX without the \!wH1/\!wH0 stutter # # Output: 48 kHz mono s16le WAV, trimmed, faded, loudness-normalized. set -eu @@ -49,6 +51,8 @@ HIGHPASS=${VESKSLOP_TTS_HIGHPASS:-40} ECHO=${VESKSLOP_TTS_ECHO:-0} WORDS=${VESKSLOP_TTS_WORDS:-0} WORD_GAP=${VESKSLOP_TTS_WORD_GAP_MS:-120} +VOX=${VESKSLOP_TTS_VOX:-0} +VOX_CLEAN=${VESKSLOP_TTS_VOX_CLEAN:-0} case "$VOICE" in sam) ENGVOICE="Sam" ;; @@ -197,12 +201,30 @@ if [ -n "$SPEAKER_GUID" ]; then fi fi -# SAPI4 is ANSI/CP1252; transliterate UTF-8 (emoji become '?'). +# SAPI4 is ANSI/CP1252. VOX mode (Flextalk only) first transforms the text into +# vendor tags (\!wH1/\!wH0, \!br, \!sfNN) with setup/vox_parser.py; the parser +# works on UTF-8, so it runs before the CP1252 transliteration. +TEXT_SRC="$TEXTFILE" +if [ "$VOX" = "1" ] && [ -n "$SPEAKER_GUID" ]; then + if command -v python3 >/dev/null 2>&1 && [ -f "$ROOT/setup/vox_parser.py" ]; then + if [ "$VOX_CLEAN" = "1" ]; then + python3 "$ROOT/setup/vox_parser.py" --clean < "$TEXTFILE" > "$TMP/vox.txt" + else + python3 "$ROOT/setup/vox_parser.py" < "$TEXTFILE" > "$TMP/vox.txt" + fi + TEXT_SRC="$TMP/vox.txt" + else + echo "vox mode needs python3 and setup/vox_parser.py" >&2 + exit 1 + fi +fi + +# transliterate UTF-8 (emoji become '?') if command -v iconv >/dev/null 2>&1; then - iconv -f UTF-8 -t WINDOWS-1252//TRANSLIT "$TEXTFILE" > "$TXT" 2>/dev/null \ - || cp "$TEXTFILE" "$TXT" + iconv -f UTF-8 -t WINDOWS-1252//TRANSLIT "$TEXT_SRC" > "$TXT" 2>/dev/null \ + || cp "$TEXT_SRC" "$TXT" else - cp "$TEXTFILE" "$TXT" + cp "$TEXT_SRC" "$TXT" fi HOST="$ROOT/build/sapi4host-winlib.exe" @@ -232,7 +254,8 @@ try_run_host() { # Word mode (Flextalk): synthesize each word and merge with silence gaps, which # gives the staccato radio cadence the Half-Life VOX/HEV voices are known for. -if [ "$WORDS" = "1" ] && [ -n "$SPEAKER_GUID" ]; then +# VOX mode takes precedence: it produces its own cadence with vendor tags. +if [ "$WORDS" = "1" ] && [ -n "$SPEAKER_GUID" ] && [ "$VOX" != "1" ]; then GAP=$WORD_GAP case "$GAP" in *[!0-9]*|"") GAP=120 ;; esac [ "$GAP" -gt 500 ] && GAP=500 diff --git a/setup/vox_parser.py b/setup/vox_parser.py new file mode 100644 index 0000000..879c014 --- /dev/null +++ b/setup/vox_parser.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""VOX prosody encoder for FlexTalk, ported from h4rm0n1c/NetTTS. + +Transforms plain text into FlexTalk vendor-tagged text: + + \\!wH1 ... \\!wH0 staccato word-cadence wrapper + \\!br beat boundary + \\!sfNN final pause (centiseconds) + +Our host passes TTSDATAFLAG_TAGGED, so the engine honors these tags. +Rule set studied from https://github.com/h4rm0n1c/NetTTS (vox_parser.cpp, +no license declared; this is an independent reimplementation). + +Usage: + vox_parser.py [--clean] [text] + echo "text" | vox_parser.py + +--clean omits the \\!wH1/\\!wH0 wrapper (plain cadence, no word stutter). +""" +import re +import sys + +BR = r'\!br' +WH1 = r'\!wH1' +WH0 = r'\!wH0' +TAGS = (BR, WH1, WH0) + +RE_LEADIN = re.compile( + r'^\s*((?:Now|Please|A\s+reminder|On\s+behalf\s+of|Good\s+(?:morning|evening)' + r'|This\s+(?:automated\s+train|tram)))\b', re.IGNORECASE | re.ASCII) +RE_TIME12 = re.compile( + r'\b(\d{1,2})[:.](\d{2})\s*((?:A\.?M\.?)|(?:P\.?M\.?))(?:\.)?\b', + re.IGNORECASE | re.ASCII) +RE_TIME_IS = re.compile(r'(\b[Tt]he\s+time\s+is)\s+(\d+)', re.ASCII) +RE_24H = re.compile(r'\b(\d{2})00\s+hours\b', re.IGNORECASE | re.ASCII) +RE_N3 = re.compile(r'\b([1-9])(\d)(\d)\b', re.ASCII) +RE_DEGREES = re.compile(r'\b(\d+)\s+degrees\b', re.IGNORECASE | re.ASCII) + +LIGHT, MEDIUM, HEAVY = 0, 1, 2 + +STOP_WORDS = { + 'the', 'thee', 'a', 'an', 'to', 'for', 'of', 'in', 'on', 'at', 'by', 'with', + 'from', 'as', 'is', 'are', 'was', 'were', 'this', 'that', +} +LIGHT_VERBS = { + 'welcome', 'wait', 'stand', 'provide', 'provided', 'return', 'remain', + 'maintain', 'maintained', 'verify', 'contact', 'board', 'arriving', + 'inbound', 'bound', 'commence', 'commences', +} +UNITS = {'degrees', 'hours', 'percent', '%'} +LINKERS = {'at', 'in', 'on', 'to', 'of', 'for', 'from', 'by'} +DETERMINERS = {'the', 'thee', 'a', 'an', 'this', 'that', 'these', 'those', + 'some', 'any', 'each', 'every', 'no'} +PUNCT = '.,;:!?\'"' + + +def trim(s): + return s.strip() + + +def lower(s): + return s.lower() + + +def bare(tok): + return strip_trailing_punct(tok) + + +def is_all_digits(t): + return bool(t) and all('0' <= c <= '9' for c in t) + + +def is_titlecase_word(t): + return bool(t) and t[0].isalpha() and t[0].isupper() + + +def strip_trailing_punct(t): + while t and t[-1] in PUNCT: + t = t[:-1] + return t + + +def split_tail(tok): + """Return (core, trailing punctuation).""" + core, tail = tok, '' + while core and core[-1] in PUNCT: + tail = core[-1] + tail + core = core[:-1] + return core, tail + + +def ends_with_br(s): + return s.endswith(' ' + BR) + + +def is_tag(tok): + return tok in TAGS + + +def split_sentences(text): + out, cur = [], '' + i = 0 + n = len(text) + while i < n: + c = text[i] + cur += c + if c in '.!?': + j = i + 1 + while j < n and text[j] in '"\'': + cur += text[j] + j += 1 + if trim(cur): + out.append(trim(cur)) + cur = '' + i = j - 1 + i += 1 + tail = trim(cur) + if tail: + out.append(tail) + return out + + +def apply_leadin_break(s): + m = RE_LEADIN.search(s) + if not m: + return s + head = trim(m.group(0)) + rest = trim(s[m.end():]) + return head + ' ' + BR + ' ' + rest + + +def is_single_letter(tok): + b = bare(tok) + if len(b) == 1 and b[0].isalpha() and b[0].isupper(): + return True + return len(b) == 2 and b[0].isalpha() and b[0].isupper() and b[1] == ':' + + +def is_number(tok): + b = bare(tok) + return is_all_digits(b) + + +def apply_thee_rule(s): + tok = s.split() + if not tok: + return s + + def prev_word(i, nth): + for k in range(i - 1, -1, -1): + if is_tag(tok[k]): + continue + nth -= 1 + if nth == 0: + return lower(split_tail(tok[k])[0]) + return '' + + def next_word(i, nth): + for k in range(i + 1, len(tok)): + if is_tag(tok[k]): + continue + nth -= 1 + if nth == 0: + return tok[k] + return '' + + for i in range(len(tok)): + if is_tag(tok[i]): + continue + core, tail = split_tail(tok[i]) + if lower(core) != 'the': + tok[i] = core + tail + continue + make_thee = False + p1 = prev_word(i, 1) + if p1: + if p1 in LINKERS: + make_thee = True + elif p1 == 'of': + p2, p3 = prev_word(i, 2), prev_word(i, 3) + if p2 == 'behalf' and p3 == 'on': + make_thee = True + if not make_thee: + n1 = next_word(i, 1) + if n1 and (is_titlecase_word(bare(n1)) or is_single_letter(n1) + or is_number(n1)): + make_thee = True + tok[i] = ('thee' if make_thee else core) + tail + return ' '.join(tok) + + +def apply_time_numbers_degrees(s): + def time12(m): + hh, mm, ap = m.group(1), m.group(2), lower(m.group(3)) + ap1 = 'P:' if ap.startswith('p') else 'Ay:' + return hh + ' ' + BR + ' ' + mm + ' ' + BR + ' ' + ap1 + ' ' + BR + ' M:' + + s = RE_TIME12.sub(time12, s) + s = RE_TIME_IS.sub(r'\1 ' + BR + r' \2', s) + s = RE_24H.sub(r'\1 ' + BR + ' hundred ' + BR + ' hours', s) + + def n3(m): + yz = m.group(2) + m.group(3) + if yz == '00': + return m.group(0) + rem = yz.lstrip('0') or yz + return m.group(1) + '00 ' + BR + ' and ' + BR + ' ' + rem + + s = RE_N3.sub(n3, s) + s = RE_DEGREES.sub(r'\1 ' + BR + ' degrees ' + BR, s) + s = re.sub(r'M:\s*\.', 'M:', s) + return s + + +def syllables(word): + if not word: + return 0 + w = lower(word) + vowels = set('aeiouy') + count = 0 + in_v = False + for c in w: + if c in vowels: + if not in_v: + count += 1 + in_v = True + else: + in_v = False + if count > 1 and len(w) > 1 and w[-1] == 'e' and not (len(w) > 2 and w[-2] == 'l'): + count -= 1 + if len(w) > 2 and w[-1] == 'e' and w[-2] == 'l': + count += 1 + return max(1, count) + + +def weight(tok): + if not tok: + return LIGHT + pfree = strip_trailing_punct(tok) + tl = lower(pfree) + if tl in STOP_WORDS or tl in LIGHT_VERBS: + return LIGHT + if is_all_digits(pfree): + return HEAVY + if '-' in pfree: + return HEAVY + if is_titlecase_word(pfree): + return HEAVY + if tl in UNITS: + return HEAVY + if syllables(pfree) >= 3 or len(pfree) >= 8: + return HEAVY + return MEDIUM + + +def is_and(tok): + return lower(strip_trailing_punct(tok)) == 'and' + + +def starts_with_punct(tok): + return bool(tok) and tok[0] in '.,;:!?)' + + +def is_open_punct(tok): + return bool(tok) and tok[0] in '("\'' + + +def emit_titlecase_run(v, i, j, out): + if out and not ends_with_br(out): + out += ' ' + BR + for k in range(i, j): + out += ' ' + v[k] + ' ' + BR + return out + + +def build_beats(sentence): + v = sentence.split() + if not v: + return sentence + + out = [''] + state = {'since_break_all_light': True, 'prev_w': LIGHT, 'content_run': 0} + + def reset(): + state['since_break_all_light'] = True + state['content_run'] = 0 + state['prev_w'] = LIGHT + + def ensure_break(): + if not ends_with_br(out[0]): + out[0] += ' ' + BR + + def append_token(tok): + if out[0] and not starts_with_punct(tok) and not is_open_punct(tok): + out[0] += ' ' + out[0] += tok + + def emit_dash_split(tok): + core, tail = split_tail(tok) + cuts = [] + for p, c in enumerate(core): + if c in ('-', '\u2013', '\u2014'): + if 0 < p < len(core) - 1 and core[p - 1].isalpha() and core[p + 1].isalpha(): + cuts.append(p) + else: + return False + if not cuts: + return False + start = 0 + for idx in range(len(cuts) + 1): + end = cuts[idx] if idx < len(cuts) else len(core) + part = core[start:end] + if out[0] and not ends_with_br(out[0]) and not starts_with_punct(part) and not is_open_punct(part): + out[0] += ' ' + if idx == len(cuts): + part += tail + out[0] += part + ensure_break() + reset() + start = end + (1 if idx < len(cuts) else 0) + return True + + i = 0 + while i < len(v): + # Proper-noun blocks + if bare(v[i]) in ('Area', 'Level', 'Sector'): + j = i + 1 + while j < len(v): + wj = bare(v[j]) + if is_all_digits(wj) or is_titlecase_word(wj): + j += 1 + else: + break + if j > i + 1: + out[0] = emit_titlecase_run(v, i, j, out[0]) + i = j + reset() + continue + if is_titlecase_word(bare(v[i])): + j = i + while j < len(v) and is_titlecase_word(bare(v[j])): + j += 1 + if j >= i + 2: + out[0] = emit_titlecase_run(v, i, j, out[0]) + i = j + reset() + continue + + # "and" between heavier items + if is_and(v[i]) and 0 < i < len(v) - 1: + if weight(v[i - 1]) >= MEDIUM and weight(v[i + 1]) >= MEDIUM: + ensure_break() + out[0] += ' and ' + BR + i += 1 + reset() + continue + + # Standalone dash + if v[i] in ('-', '\u2013', '\u2014'): + ensure_break() + reset() + i += 1 + continue + + # Symmetric linker isolation + tl_tok = lower(bare(v[i])) + if i >= 2 and tl_tok in LINKERS: + ensure_break() + reset() + append_token(v[i]) + ensure_break() + reset() + i += 1 + continue + + # Pre-copula break + if i >= 2 and lower(bare(v[i])) == 'is': + k = i + 1 + while k < len(v) and is_tag(v[k]): + k += 1 + trigger = False + if k < len(v): + n1 = v[k] + if (lower(strip_trailing_punct(n1)) in DETERMINERS + or is_titlecase_word(strip_trailing_punct(n1)) + or is_all_digits(strip_trailing_punct(n1)) + or is_single_letter(n1) + or weight(n1) == HEAVY): + trigger = True + if trigger and not ends_with_br(out[0]): + out[0] += ' ' + BR + reset() + + # Hyphenated compounds + if emit_dash_split(v[i]): + i += 1 + continue + + # Generic token logic + tok = v[i] + w_eff = weight(tok) + in_head_span = i < 2 + if i == 0 and is_titlecase_word(bare(tok)): + next_is_title = (i + 1 < len(v)) and is_titlecase_word(bare(v[i + 1])) + if not next_is_title and w_eff == HEAVY: + w_eff = MEDIUM + + if w_eff == HEAVY and state['content_run'] >= 1: + k = i - 1 + while k >= 0 and is_tag(v[k]): + k -= 1 + exempt = False + if k >= 0: + prev = v[k] + if (is_titlecase_word(bare(prev)) or is_all_digits(bare(prev)) + or is_single_letter(prev)): + exempt = True + if not exempt: + ensure_break() + reset() + + if w_eff == HEAVY and state['content_run'] >= 2: + ensure_break() + reset() + + if w_eff == HEAVY and state['since_break_all_light'] and not in_head_span: + if out[0] and not ends_with_br(out[0]): + out[0] += ' ' + BR + reset() + + if i > 0 and state['prev_w'] == HEAVY and w_eff == HEAVY and not in_head_span: + ensure_break() + reset() + + append_token(tok) + + if tok and tok[-1] in ',;' and not ends_with_br(out[0]): + out[0] += ' ' + BR + reset() + i += 1 + continue + + if w_eff == LIGHT: + pass + else: + state['content_run'] += 1 + state['since_break_all_light'] = False + state['prev_w'] = w_eff + i += 1 + + return trim(out[0]) + + +def normalize_letter_tokens(s): + tok = s.split() + if not tok: + return s + + def is_boundary(idx): + return idx < 0 or idx >= len(tok) or is_tag(tok[idx]) + + for i in range(len(tok)): + if is_tag(tok[i]): + continue + core, tail = tok[i], '' + had_colon = False + while core: + c = core[-1] + if c == ':' and not had_colon: + had_colon = True + core = core[:-1] + continue + if c in '.,;!?\'"': + tail = c + tail + core = core[:-1] + continue + break + near_bound = is_boundary(i - 1) or is_boundary(i + 1) + if core in ('A', 'a') and (near_bound or had_colon): + core = 'Ay' + had_colon = True + elif len(core) == 1 and core.isalpha() and core.isupper() and near_bound: + had_colon = True + tok[i] = core + (':' if had_colon else '') + tail + return ' '.join(tok) + + +def tidy(s): + t = re.sub(r'\s+', ' ', s) + t = re.sub(r'\s*' + re.escape(BR) + r'\s*', ' ' + BR + ' ', t) + t = re.sub(r'(\s*' + re.escape(BR) + r'\s*){2,}', ' ' + BR + ' ', t) + t = re.sub(r'\s+([.,!?;:])', r'\1', t) + return t.strip() + + +def vox_process(text, wrap_vox_tags=True): + out = '' + for sent in split_sentences(text): + if not sent: + continue + sent = apply_thee_rule(sent) + sent = apply_leadin_break(sent) + sent = apply_time_numbers_degrees(sent) + beats = build_beats(sent) + beats = normalize_letter_tokens(beats) + beats = tidy(beats) + if beats and not beats.endswith(' '): + beats += ' ' + beats += r'\!sf500 ' + BR + if out: + out += ' ' + out += beats + out = trim(out) + out = re.sub(r'(\s*' + re.escape(BR) + r'\s*){2,}', ' ' + BR + ' ', out) + out = re.sub(re.escape(BR) + r'\s*([,;:])', r'\1 ' + BR, out) + if out: + if wrap_vox_tags: + out = WH1 + ' ' + out + ' ' + WH0 + ' ' + else: + out = out + ' ' + return out + + +def main(): + args = sys.argv[1:] + clean = False + if '--clean' in args: + clean = True + args.remove('--clean') + if args: + lines = [' '.join(args)] + else: + lines = sys.stdin.read().splitlines() + out_lines = [vox_process(line, wrap_vox_tags=not clean) for line in lines if line.strip()] + sys.stdout.write('\n'.join(out_lines)) + if out_lines: + sys.stdout.write('\n') + + +if __name__ == '__main__': + main()