Add voice audio, capture and transcript, browser monitor, and DM UI
This commit is contained in:
Fisto
@@ -4,5 +4,10 @@ node_modules/
|
||||
.token
|
||||
.secrets/
|
||||
.veskslop-state.json
|
||||
preferences.json
|
||||
tls/
|
||||
models/
|
||||
actions.jsonl
|
||||
actions.jsonl.*
|
||||
transcripts.jsonl
|
||||
transcripts.jsonl.*
|
||||
|
||||
+2
-1
@@ -14,7 +14,8 @@
|
||||
"control:mock": "tsx src/mock-core.ts",
|
||||
"test:trigger": "tsx src/test-trigger.ts",
|
||||
"cdp:dispatch": "tsx src/cdp-dispatch.ts",
|
||||
"test:cache": "tsx src/test-cache.ts"
|
||||
"test:cache": "tsx src/test-cache.ts",
|
||||
"test:audio": "tsx src/test-audio.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.30.0",
|
||||
|
||||
+1054
-108
File diff suppressed because it is too large
Load Diff
+172
-19
@@ -15,6 +15,11 @@
|
||||
<span class="vk-brand-version">debug</span>
|
||||
</span>
|
||||
|
||||
<div class="vk-scope">
|
||||
<button id="scope-guild" class="vk-scope-btn active" type="button">Guild</button>
|
||||
<button id="scope-dm" class="vk-scope-btn" type="button">DM</button>
|
||||
</div>
|
||||
|
||||
<div class="vk-guild-picker">
|
||||
<button id="guild-picker" class="btn btn-secondary vk-guild-button" type="button">
|
||||
<span id="guild-picker-label">Direct messages</span>
|
||||
@@ -33,6 +38,17 @@
|
||||
</div>
|
||||
|
||||
<div class="vk-header-right">
|
||||
<button id="core-start-banner" class="btn btn-primary vk-small-btn vk-start-server" type="button" hidden>
|
||||
Start server
|
||||
</button>
|
||||
<button
|
||||
id="client-audio"
|
||||
class="btn btn-secondary vk-small-btn vk-client-audio"
|
||||
type="button"
|
||||
title="Play the call in this browser"
|
||||
>
|
||||
Client audio
|
||||
</button>
|
||||
<label class="vk-takeover" title="Read-only unless takeover is on">
|
||||
<input type="checkbox" id="takeover" />
|
||||
<span class="vk-switch"><span class="vk-knob"></span></span>
|
||||
@@ -51,6 +67,7 @@
|
||||
<aside class="vk-rail">
|
||||
<div id="guild-switcher" class="vk-guild-switcher"></div>
|
||||
|
||||
<div id="rail-guild" class="vk-rail-scope">
|
||||
<div class="vk-rail-section">
|
||||
<div class="vk-rail-label">Text</div>
|
||||
<div id="text-channels" class="vk-rail-list"></div>
|
||||
@@ -62,8 +79,16 @@
|
||||
</div>
|
||||
|
||||
<div class="vk-rail-section">
|
||||
<div class="vk-rail-label">Direct</div>
|
||||
<div id="direct-list" class="vk-rail-list"></div>
|
||||
<div class="vk-rail-label">People</div>
|
||||
<div id="people-list" class="vk-rail-list"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="rail-dm" class="vk-rail-scope" hidden>
|
||||
<div class="vk-rail-section">
|
||||
<div class="vk-rail-label">Direct messages</div>
|
||||
<div id="dm-rail-list" class="vk-rail-list"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="agent-identity" class="vk-rail-footer"></div>
|
||||
@@ -103,16 +128,18 @@
|
||||
</div>
|
||||
<div class="vk-tts-settings">
|
||||
<span class="vk-micro-label">Voice settings</span>
|
||||
<span class="vk-chip vk-chip-on">sv-SE · Mattias</span>
|
||||
<span class="vk-chip">rate 1.0×</span>
|
||||
<span class="vk-chip vk-chip-on">yield to call</span>
|
||||
<span class="vk-chip">duck 30%</span>
|
||||
<select id="tts-provider" class="vk-select">
|
||||
<option value="local">local TTS</option>
|
||||
<option value="api">API TTS</option>
|
||||
</select>
|
||||
<span class="vk-chip vk-chip-on">duck on voice</span>
|
||||
<span class="vk-chip vk-chip-on">queue lines</span>
|
||||
</div>
|
||||
<div class="vk-tts-body">
|
||||
<textarea id="tts-input" class="vk-tts-field" rows="3" placeholder="Speak text into the call…"></textarea>
|
||||
<div class="vk-tts-meta">
|
||||
<span class="vk-mono vk-dim">voice audio is not implemented yet</span>
|
||||
<span class="vk-mono vk-dim" id="tts-status">idle</span>
|
||||
<button id="tts-stop" class="btn btn-secondary vk-small-btn" type="button">Stop</button>
|
||||
<button id="tts-speak" class="btn btn-primary vk-small-btn" type="button">Speak</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -154,7 +181,6 @@
|
||||
<div id="right-dms" class="vk-right-list"></div>
|
||||
</section>
|
||||
</div>
|
||||
<div class="vk-right-footer vk-mono vk-dim">full roster sits behind the guild header</div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
@@ -186,9 +212,12 @@
|
||||
<div id="event-log" class="vk-event-log"></div>
|
||||
</div>
|
||||
<div id="console-panel-transcript" class="vk-console-panel">
|
||||
<div class="vk-roadmap">
|
||||
<span class="tag tag-outline">roadmap</span>
|
||||
<p>Voice transcription (STT/TTS) is not implemented yet. Transcript segments will appear here once voice audio lands.</p>
|
||||
<div class="vk-transcript">
|
||||
<div class="vk-transcript-toolbar">
|
||||
<span id="transcript-status" class="vk-mono vk-dim">no capture session</span>
|
||||
<button id="transcript-refresh" class="btn btn-secondary vk-small-btn" type="button">Refresh</button>
|
||||
</div>
|
||||
<div id="transcript-list" class="vk-transcript-list"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -205,6 +234,136 @@
|
||||
</div>
|
||||
|
||||
<div class="vk-settings-body">
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Visible guilds</span>
|
||||
<span class="vk-settings-actions">
|
||||
<button id="guilds-save" class="btn btn-primary vk-small-btn" type="button" disabled>Save</button>
|
||||
<button id="guilds-hide-all" class="btn btn-secondary vk-small-btn" type="button">Hide all</button>
|
||||
<button id="guilds-show-all" class="btn btn-secondary vk-small-btn" type="button">Show all</button>
|
||||
</span>
|
||||
</div>
|
||||
<div id="guild-visibility" class="vk-settings-card"></div>
|
||||
</section>
|
||||
|
||||
<details class="vk-settings-group" open>
|
||||
<summary class="vk-settings-summary">
|
||||
<span class="vk-icon vk-caret" data-icon="caretDown"></span>
|
||||
<span>Client</span>
|
||||
</summary>
|
||||
|
||||
<div class="vk-settings-group-body">
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Bot audio</span>
|
||||
<span class="vk-settings-actions">
|
||||
<button id="audio-connect" class="btn btn-primary vk-small-btn" type="button">Connect devices</button>
|
||||
<button id="audio-refresh" class="btn btn-secondary vk-small-btn" type="button">Refresh</button>
|
||||
<button id="audio-capture" class="btn btn-secondary vk-small-btn" type="button">Start capture</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="vk-settings-card">
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Audio engine</span>
|
||||
<span class="vk-status-value" id="audio-status">checking…</span>
|
||||
</div>
|
||||
<div class="vk-status-row" id="audio-hint-row" hidden>
|
||||
<span class="vk-status-label">Fix</span>
|
||||
<span class="vk-status-value vk-dim" id="audio-hint"></span>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">TTS</span>
|
||||
<span class="vk-status-value" id="audio-tts">—</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Speech</span>
|
||||
<span class="vk-mono vk-dim" id="speech-status"></span>
|
||||
</div>
|
||||
<div class="vk-settings-card">
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">TTS engine</span>
|
||||
<select class="vk-select" id="speech-provider">
|
||||
<option value="">Auto</option>
|
||||
<option value="local">Local</option>
|
||||
<option value="api">API</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Language</span>
|
||||
<select class="vk-select" id="speech-language">
|
||||
<option value="">Auto</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Voice</span>
|
||||
<select class="vk-select" id="speech-voice">
|
||||
<option value="">Default</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">STT</span>
|
||||
<span class="vk-status-value vk-dim" id="speech-stt">—</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Display</span>
|
||||
<span class="vk-mono vk-dim" id="timezone-status"></span>
|
||||
</div>
|
||||
<div class="vk-settings-card">
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Timezone</span>
|
||||
<select class="vk-select" id="timezone-select"></select>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Browser audio</span>
|
||||
<span class="vk-settings-actions">
|
||||
<button id="browser-grant" class="btn btn-primary vk-small-btn" type="button">Enable browser audio</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="vk-settings-card">
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Output device</span>
|
||||
<select class="vk-select" id="browser-output">
|
||||
<option value="">Browser default</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Input device</span>
|
||||
<select class="vk-select" id="browser-input" disabled>
|
||||
<option value="">Mic puppeting not implemented</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Call audio</span>
|
||||
<span class="vk-status-value vk-dim" id="browser-audio-status">off</span>
|
||||
</div>
|
||||
<div class="vk-status-row">
|
||||
<span class="vk-status-label">Note</span>
|
||||
<span class="vk-status-value vk-dim" id="browser-hint">Browser audio needs HTTPS or localhost.</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details class="vk-settings-group" open>
|
||||
<summary class="vk-settings-summary">
|
||||
<span class="vk-icon vk-caret" data-icon="caretDown"></span>
|
||||
<span>Server</span>
|
||||
</summary>
|
||||
|
||||
<div class="vk-settings-group-body">
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Core process</span>
|
||||
@@ -234,14 +393,6 @@
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Visible guilds</span>
|
||||
<button id="guilds-show-all" class="btn btn-secondary vk-small-btn" type="button">Show all</button>
|
||||
</div>
|
||||
<div id="guild-visibility" class="vk-settings-card"></div>
|
||||
</section>
|
||||
|
||||
<section class="vk-settings-section">
|
||||
<div class="vk-section-head">
|
||||
<span class="vk-rail-label vk-rail-label-flat">Action log</span>
|
||||
@@ -250,6 +401,8 @@
|
||||
<div id="action-log" class="vk-log"></div>
|
||||
</section>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -166,6 +166,12 @@ button {
|
||||
border-radius: var(--radius-sm);
|
||||
cursor: pointer;
|
||||
font-size: 13px;
|
||||
width: 100%;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-text);
|
||||
text-align: left;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.vk-menu-item:hover {
|
||||
@@ -268,6 +274,151 @@ button {
|
||||
font-size: 12.5px;
|
||||
}
|
||||
|
||||
.vk-start-server {
|
||||
padding: 5px 12px;
|
||||
font-size: 12px;
|
||||
animation: vk-bloom 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes vk-bloom {
|
||||
0%,
|
||||
100% {
|
||||
box-shadow: 0 0 0 0 transparent;
|
||||
}
|
||||
50% {
|
||||
box-shadow: 0 0 12px 2px color-mix(in srgb, var(--color-accent) 45%, transparent);
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.vk-start-server {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
|
||||
.vk-scope {
|
||||
display: inline-flex;
|
||||
border: 1px solid var(--color-divider);
|
||||
border-radius: var(--radius-md);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.vk-scope-btn {
|
||||
padding: 5px 12px;
|
||||
font-size: 12.5px;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--color-neutral-400);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.vk-scope-btn + .vk-scope-btn {
|
||||
border-left: 1px solid var(--color-divider);
|
||||
}
|
||||
|
||||
.vk-scope-btn:hover {
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
.vk-scope-btn.active {
|
||||
color: var(--color-accent);
|
||||
box-shadow: inset 0 0 0 1px var(--color-accent);
|
||||
}
|
||||
|
||||
/* The main areas dim and stop taking input while the bridge is down. */
|
||||
.vk-offline .vk-body,
|
||||
.vk-offline .vk-console {
|
||||
opacity: 0.5;
|
||||
filter: saturate(0.6);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.vk-rail-scope {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.vk-dm-row {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.vk-dm-badge {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
min-width: 17px;
|
||||
height: 16px;
|
||||
padding: 0 4px;
|
||||
margin-right: 2px;
|
||||
border-radius: 8px;
|
||||
font: 500 10px/1 ui-monospace, Menlo, monospace;
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.vk-dm-badge.unread {
|
||||
background: var(--color-accent-800);
|
||||
color: var(--color-accent-100);
|
||||
}
|
||||
|
||||
.vk-dm-badge.mention {
|
||||
background: color-mix(in srgb, var(--color-mention) 35%, transparent);
|
||||
color: var(--color-mention);
|
||||
}
|
||||
|
||||
.vk-dm-badge.muted {
|
||||
background: var(--color-neutral-800);
|
||||
color: var(--color-neutral-300);
|
||||
}
|
||||
|
||||
.vk-dm-time {
|
||||
margin-left: auto;
|
||||
font: 400 10px/1 ui-monospace, Menlo, monospace;
|
||||
color: var(--color-neutral-400);
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.vk-star {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: none;
|
||||
color: var(--color-neutral-500);
|
||||
cursor: pointer;
|
||||
flex: none;
|
||||
}
|
||||
|
||||
.vk-star:hover {
|
||||
color: var(--color-accent-300);
|
||||
}
|
||||
|
||||
.vk-star.on {
|
||||
color: var(--color-accent-400);
|
||||
}
|
||||
|
||||
.vk-person-name.speaking,
|
||||
.vk-voice-member .speaking,
|
||||
.vk-right-name .speaking {
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.vk-marks .unavailable {
|
||||
opacity: 0.35;
|
||||
}
|
||||
|
||||
:root {
|
||||
--color-mention: #d29922;
|
||||
}
|
||||
|
||||
.vk-client-audio.active {
|
||||
color: var(--color-accent);
|
||||
border-color: var(--color-accent);
|
||||
background: color-mix(in srgb, var(--color-accent) 12%, transparent);
|
||||
}
|
||||
|
||||
/* Call strip */
|
||||
|
||||
.vk-call-strip {
|
||||
@@ -484,6 +635,18 @@ button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
width: 100%;
|
||||
padding: 0;
|
||||
background: none;
|
||||
border: none;
|
||||
color: inherit;
|
||||
font: inherit;
|
||||
text-align: left;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.vk-rail-voice-head:hover .vk-rail-name {
|
||||
color: var(--color-accent-300);
|
||||
}
|
||||
|
||||
.vk-rail-voice-members {
|
||||
@@ -989,6 +1152,24 @@ a.vk-attachment:hover {
|
||||
padding: 10px 6px 0;
|
||||
}
|
||||
|
||||
.vk-right-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.vk-right-group-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 4px 6px;
|
||||
font-size: 12.5px;
|
||||
color: var(--color-neutral-300);
|
||||
background: var(--color-neutral-900);
|
||||
border-radius: var(--radius-sm);
|
||||
}
|
||||
|
||||
.vk-roadmap {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -1002,6 +1183,63 @@ a.vk-attachment:hover {
|
||||
color: var(--color-neutral-500);
|
||||
}
|
||||
|
||||
.vk-transcript {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.vk-transcript-toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.vk-transcript-toolbar .vk-small-btn {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.vk-transcript-list {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
font-family: ui-monospace, Menlo, monospace;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.vk-transcript-line {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: baseline;
|
||||
color: var(--color-neutral-400);
|
||||
}
|
||||
|
||||
.vk-transcript-line .vk-transcript-time {
|
||||
flex: none;
|
||||
color: var(--color-neutral-500);
|
||||
}
|
||||
|
||||
.vk-transcript-line .vk-transcript-speaker {
|
||||
flex: none;
|
||||
min-width: 84px;
|
||||
color: var(--color-accent-300);
|
||||
}
|
||||
|
||||
.vk-transcript-line .vk-transcript-text {
|
||||
color: var(--color-neutral-200);
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.vk-transcript-line .vk-transcript-wav {
|
||||
flex: none;
|
||||
color: var(--color-neutral-500);
|
||||
}
|
||||
|
||||
/* Voice state marks */
|
||||
|
||||
.vk-marks {
|
||||
@@ -1280,6 +1518,56 @@ a.vk-attachment:hover {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.vk-settings-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.vk-settings-summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
cursor: pointer;
|
||||
list-style: none;
|
||||
padding: 6px 0;
|
||||
font: 500 12px/1 var(--font-heading);
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
color: var(--color-neutral-300);
|
||||
}
|
||||
|
||||
.vk-settings-summary::-webkit-details-marker {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.vk-settings-summary:hover {
|
||||
color: var(--color-accent-300);
|
||||
}
|
||||
|
||||
.vk-settings-group:not([open]) .vk-caret {
|
||||
transform: rotate(-90deg);
|
||||
}
|
||||
|
||||
.vk-settings-group-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 22px;
|
||||
padding-top: 12px;
|
||||
}
|
||||
|
||||
.vk-select {
|
||||
background: var(--color-bg);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--color-divider);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 4px 8px;
|
||||
font-size: 12.5px;
|
||||
}
|
||||
|
||||
.vk-select:disabled {
|
||||
opacity: 0.55;
|
||||
}
|
||||
|
||||
.vk-settings-actions {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Veskslop STT helper — transcribe a 16-bit mono WAV with Vosk on the CPU."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: stt-vosk.py <wav>", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
model_path = os.environ.get("VESKSLOP_VOSK_MODEL")
|
||||
if not model_path:
|
||||
print("VESKSLOP_VOSK_MODEL is not set", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
from vosk import KaldiRecognizer, Model, SetLogLevel
|
||||
except ImportError:
|
||||
print("vosk is not installed (pip install vosk)", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
SetLogLevel(-1)
|
||||
model = Model(model_path)
|
||||
|
||||
with wave.open(sys.argv[1], "rb") as wav:
|
||||
if wav.getnchannels() != 1 or wav.getsampwidth() != 2:
|
||||
print("input must be 16-bit mono WAV", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
recognizer = KaldiRecognizer(model, wav.getframerate())
|
||||
parts = []
|
||||
|
||||
while True:
|
||||
data = wav.readframes(4000)
|
||||
if not data:
|
||||
break
|
||||
if recognizer.AcceptWaveform(data):
|
||||
parts.append(json.loads(recognizer.Result()).get("text", ""))
|
||||
|
||||
parts.append(json.loads(recognizer.FinalResult()).get("text", ""))
|
||||
|
||||
print(" ".join(part for part in parts if part).strip())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,344 @@
|
||||
/*
|
||||
* Veskslop daemon — TTS queue, ducking, capture, and transcription.
|
||||
*
|
||||
* Lines play one at a time. While another person is speaking, the virtual sink
|
||||
* is ducked. Capture reads the client output, splits it into utterances, stores
|
||||
* them in the transcript, and runs speech to text when a provider is set.
|
||||
*/
|
||||
|
||||
import { AudioManager } from "./audio.js";
|
||||
import type { CaptureSegment } from "./capture.js";
|
||||
import { CapturePipeline } from "./capture.js";
|
||||
import type { Stt } from "./stt.js";
|
||||
import { Tts, type TtsOptions, type TtsVoice } from "./tts.js";
|
||||
import type { TranscriptSegment, TranscriptSession, TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD, type VadConfig } from "./vad.js";
|
||||
|
||||
export interface AudioServiceOptions {
|
||||
cacheDir?: string;
|
||||
transcripts: TranscriptStore;
|
||||
stt: Stt;
|
||||
vad?: VadConfig;
|
||||
autoCapture?: boolean;
|
||||
getSelfId: () => Promise<string | undefined>;
|
||||
getVoiceContext?: () => Promise<{ guildId?: string | null; channelId?: string | null } | undefined>;
|
||||
getSpeakerName?: (userId: string) => Promise<string | undefined>;
|
||||
onTranscript?: (event: Record<string, unknown>) => void;
|
||||
}
|
||||
|
||||
export interface SpeakResult {
|
||||
queued: boolean;
|
||||
id: number;
|
||||
position: number;
|
||||
}
|
||||
|
||||
interface SpeakerInterval {
|
||||
userId: string;
|
||||
name?: string;
|
||||
start: number;
|
||||
end?: number;
|
||||
}
|
||||
|
||||
export class AudioService {
|
||||
readonly manager = new AudioManager();
|
||||
private readonly tts: Tts;
|
||||
private readonly transcripts: TranscriptStore;
|
||||
private readonly stt: Stt;
|
||||
private readonly vad: VadConfig;
|
||||
private readonly queue: Array<{ id: number; text: string; options: TtsOptions }> = [];
|
||||
private readonly speakers = new Set<string>();
|
||||
private readonly speakerIntervals: SpeakerInterval[] = [];
|
||||
private readonly speakerNames = new Map<string, string>();
|
||||
private playing = false;
|
||||
private nextId = 1;
|
||||
private ducked = false;
|
||||
private selfId?: string;
|
||||
private capture?: CapturePipeline;
|
||||
private session?: TranscriptSession;
|
||||
private segmentCount = 0;
|
||||
private initialized = false;
|
||||
|
||||
constructor(private readonly options: AudioServiceOptions) {
|
||||
this.tts = new Tts(options.cacheDir);
|
||||
this.transcripts = options.transcripts;
|
||||
this.stt = options.stt;
|
||||
this.vad = options.vad ?? DEFAULT_VAD;
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
if (this.initialized) return;
|
||||
this.initialized = true;
|
||||
|
||||
await this.manager.detect();
|
||||
await this.tts.detect();
|
||||
await this.stt.detect();
|
||||
if (!this.manager.status().available) {
|
||||
console.warn(`[audio] unavailable: ${this.manager.status().error}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.manager.ensureDevices();
|
||||
console.log(
|
||||
`[audio] devices ready: sink=${this.manager.status().sink} source=${this.manager.status().source}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("[audio] device setup failed:", (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
status() {
|
||||
return {
|
||||
...this.manager.status(),
|
||||
tts: this.tts.status(),
|
||||
stt: this.stt.status(),
|
||||
queue: this.queue.length,
|
||||
speaking: this.playing,
|
||||
capturing: !!this.capture,
|
||||
sessionId: this.session?.id ?? null,
|
||||
segments: this.segmentCount,
|
||||
transcript: this.transcripts.stats()
|
||||
};
|
||||
}
|
||||
|
||||
async speak(text: string, options: TtsOptions = {}): Promise<SpeakResult> {
|
||||
await this.init();
|
||||
|
||||
const manager = this.manager.status();
|
||||
if (!manager.available) throw new Error(manager.error ?? "audio is not available");
|
||||
|
||||
const tts = this.tts.status();
|
||||
const provider = options.provider ?? tts.defaultProvider;
|
||||
if (provider === "local" && !tts.local) {
|
||||
throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
if (provider === "api" && !tts.api) {
|
||||
throw new Error("API TTS is not configured (VESKSLOP_TTS_API_URL)");
|
||||
}
|
||||
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) throw new Error("text is required");
|
||||
if (trimmed.length > 1000) throw new Error("text is too long (max 1000 characters)");
|
||||
|
||||
const id = this.nextId++;
|
||||
this.queue.push({ id, text: trimmed, options: { ...options, provider } });
|
||||
void this.pump();
|
||||
|
||||
return { queued: true, id, position: this.queue.length };
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.queue.length = 0;
|
||||
this.manager.stopPlayback();
|
||||
}
|
||||
|
||||
/** Track speakers for ducking and attribution, and auto-capture calls. */
|
||||
async onEvent(event: any): Promise<void> {
|
||||
if (event?.type === "VOICE_STATE_UPDATE") {
|
||||
if (this.options.autoCapture) await this.handleVoiceState(event);
|
||||
return;
|
||||
}
|
||||
|
||||
if (event?.type !== "VOICE_SPEAKING") return;
|
||||
|
||||
if (!this.selfId) this.selfId = await this.options.getSelfId();
|
||||
if (this.selfId && event.userId === this.selfId) return;
|
||||
|
||||
const userId = String(event.userId);
|
||||
|
||||
if (event.speaking) {
|
||||
this.speakers.add(userId);
|
||||
this.speakerIntervals.push({ userId, name: await this.speakerName(userId), start: Date.now() });
|
||||
} else {
|
||||
this.speakers.delete(userId);
|
||||
const interval = [...this.speakerIntervals].reverse().find(item => item.userId === userId && !item.end);
|
||||
if (interval) interval.end = Date.now();
|
||||
}
|
||||
|
||||
this.pruneIntervals();
|
||||
await this.applyDuck(this.playing && this.speakers.size > 0);
|
||||
}
|
||||
|
||||
async startCapture(): Promise<{ sessionId: string }> {
|
||||
if (this.capture) throw new Error("capture is already running");
|
||||
await this.init();
|
||||
|
||||
const status = this.manager.status();
|
||||
if (!status.available) throw new Error(status.hint ?? status.error ?? "audio is not available");
|
||||
if (!status.ffmpeg) throw new Error("ffmpeg is missing; install it to capture audio");
|
||||
|
||||
const context = (await this.options.getVoiceContext?.()) ?? {};
|
||||
const session = this.transcripts.startSession(context);
|
||||
this.segmentCount = 0;
|
||||
|
||||
this.capture = new CapturePipeline({
|
||||
ffmpeg: process.env.VESKSLOP_FFMPEG ?? "ffmpeg",
|
||||
target: `${status.captureSink}.monitor`,
|
||||
directory: `${this.tts.directory}/captures`,
|
||||
sessionId: session.id,
|
||||
vad: this.vad,
|
||||
onSegment: segment => this.handleSegment(session, segment),
|
||||
onError: error => {
|
||||
console.error("[capture] stream error:", error.message);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_ERROR", error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
this.capture.start();
|
||||
this.session = session;
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION", session });
|
||||
|
||||
return { sessionId: session.id };
|
||||
}
|
||||
|
||||
stopCapture(): void {
|
||||
this.capture?.stop();
|
||||
this.capture = undefined;
|
||||
|
||||
if (this.session) {
|
||||
this.transcripts.endSession(this.session.id);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION_END", id: this.session.id });
|
||||
this.session = undefined;
|
||||
}
|
||||
|
||||
this.speakerIntervals.length = 0;
|
||||
}
|
||||
|
||||
/** Stream call audio to the browser. Returns a stop function. */
|
||||
startMonitor(onData: (chunk: Buffer) => void, onError?: (error: Error) => void): () => void {
|
||||
return this.manager.startMonitor(onData, onError);
|
||||
}
|
||||
|
||||
ttsVoices(): Promise<TtsVoice[]> {
|
||||
return this.tts.voices();
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.queue.length = 0;
|
||||
this.stopCapture();
|
||||
await this.manager.dispose();
|
||||
}
|
||||
|
||||
private async handleVoiceState(event: any): Promise<void> {
|
||||
if (!this.selfId) this.selfId = await this.options.getSelfId();
|
||||
if (!this.selfId || event.userId !== this.selfId) return;
|
||||
|
||||
if (event.channelId && !this.capture) {
|
||||
console.log("[capture] auto-start for", event.channelId);
|
||||
await this.startCapture().catch(error => console.warn("[capture] auto-start failed:", (error as Error).message));
|
||||
} else if (!event.channelId && this.capture) {
|
||||
console.log("[capture] auto-stop");
|
||||
this.stopCapture();
|
||||
}
|
||||
}
|
||||
|
||||
private handleSegment(session: TranscriptSession, segment: CaptureSegment) {
|
||||
this.segmentCount++;
|
||||
|
||||
const speaker = this.dominantSpeaker(segment.startedAt, segment.endedAt);
|
||||
const record: TranscriptSegment = {
|
||||
id: `segment-${Date.now()}-${this.segmentCount}`,
|
||||
sessionId: session.id,
|
||||
startedAt: segment.startedAt,
|
||||
endedAt: segment.endedAt,
|
||||
durationMs: segment.durationMs,
|
||||
speakerId: speaker?.userId ?? null,
|
||||
speakerName: speaker?.name ?? null,
|
||||
wav: segment.file,
|
||||
text: null
|
||||
};
|
||||
|
||||
this.transcripts.addSegment(record);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_SEGMENT", segment: record });
|
||||
|
||||
void this.transcribe(record);
|
||||
}
|
||||
|
||||
private async transcribe(record: TranscriptSegment) {
|
||||
if (!record.wav) return;
|
||||
|
||||
try {
|
||||
const text = await this.stt.transcribe(record.wav);
|
||||
if (!text) return;
|
||||
|
||||
this.transcripts.setSegmentText(record.id, text);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_TEXT", id: record.id, text });
|
||||
} catch (error) {
|
||||
console.error("[capture] transcription failed:", (error as Error).message);
|
||||
this.options.onTranscript?.({ type: "TRANSCRIPT_ERROR", error: (error as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
private dominantSpeaker(startedAt: string, endedAt: string): SpeakerInterval | undefined {
|
||||
const start = Date.parse(startedAt);
|
||||
const end = Date.parse(endedAt);
|
||||
const totals = new Map<string, { ms: number; interval: SpeakerInterval }>();
|
||||
|
||||
for (const interval of this.speakerIntervals) {
|
||||
const overlap = Math.min(end, interval.end ?? end) - Math.max(start, interval.start);
|
||||
if (overlap <= 0) continue;
|
||||
|
||||
const entry = totals.get(interval.userId) ?? { ms: 0, interval };
|
||||
entry.ms += overlap;
|
||||
totals.set(interval.userId, entry);
|
||||
}
|
||||
|
||||
let best: SpeakerInterval | undefined;
|
||||
let bestMs = 0;
|
||||
for (const entry of totals.values()) {
|
||||
if (entry.ms > bestMs) {
|
||||
best = entry.interval;
|
||||
bestMs = entry.ms;
|
||||
}
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
private async speakerName(userId: string): Promise<string | undefined> {
|
||||
if (this.speakerNames.has(userId)) return this.speakerNames.get(userId);
|
||||
const name = await this.options.getSpeakerName?.(userId).catch(() => undefined);
|
||||
if (name) this.speakerNames.set(userId, name);
|
||||
return name;
|
||||
}
|
||||
|
||||
private pruneIntervals() {
|
||||
const cutoff = Date.now() - 60_000;
|
||||
while (this.speakerIntervals.length && this.speakerIntervals[0].end && this.speakerIntervals[0].end < cutoff) {
|
||||
this.speakerIntervals.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private async pump(): Promise<void> {
|
||||
if (this.playing) return;
|
||||
|
||||
const item = this.queue.shift();
|
||||
if (!item) {
|
||||
await this.applyDuck(false);
|
||||
return;
|
||||
}
|
||||
|
||||
this.playing = true;
|
||||
try {
|
||||
const file = await this.tts.synthesize(item.text, item.options);
|
||||
await this.applyDuck(this.speakers.size > 0);
|
||||
await this.manager.play(file);
|
||||
} catch (error) {
|
||||
console.error("[audio] speak failed:", (error as Error).message);
|
||||
} finally {
|
||||
this.playing = false;
|
||||
await this.applyDuck(false);
|
||||
void this.pump();
|
||||
}
|
||||
}
|
||||
|
||||
private async applyDuck(on: boolean): Promise<void> {
|
||||
if (on === this.ducked) return;
|
||||
try {
|
||||
await this.manager.setDuck(on);
|
||||
this.ducked = on;
|
||||
} catch (error) {
|
||||
console.warn("[audio] duck failed:", (error as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
/*
|
||||
* Veskslop daemon — audio devices, playback, ducking, and capture.
|
||||
*
|
||||
* The daemon plays audio into a virtual sink. A remap source of that sink's
|
||||
* monitor becomes the client microphone. Client output can be routed to a
|
||||
* second sink that the daemon records. Every system command is optional: when
|
||||
* the host has no PulseAudio/PipeWire, the manager reports unavailable.
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
|
||||
const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts";
|
||||
const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic";
|
||||
const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture";
|
||||
const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30);
|
||||
|
||||
export class AudioUnavailableError extends Error {}
|
||||
|
||||
export interface AudioManagerStatus {
|
||||
available: boolean;
|
||||
backend: "pulse" | null;
|
||||
error?: string;
|
||||
hint?: string;
|
||||
sink: string;
|
||||
source: string;
|
||||
captureSink: string;
|
||||
ducked: boolean;
|
||||
duckPercent: number;
|
||||
playing: boolean;
|
||||
recording: boolean;
|
||||
player: boolean;
|
||||
ffmpeg: boolean;
|
||||
}
|
||||
|
||||
interface CommandResult {
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
errorCode?: string;
|
||||
}
|
||||
|
||||
function run(command: string, args: string[]): Promise<CommandResult> {
|
||||
return new Promise(resolve => {
|
||||
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout?.on("data", chunk => (stdout += chunk));
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error =>
|
||||
resolve({ code: null, stdout, stderr: String(error), errorCode: (error as NodeJS.ErrnoException).code })
|
||||
);
|
||||
child.on("close", code => resolve({ code, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export class AudioManager {
|
||||
private modules: number[] = [];
|
||||
private player?: ChildProcess;
|
||||
private recorder?: ChildProcess;
|
||||
private ducked = false;
|
||||
private available = false;
|
||||
private error?: string;
|
||||
private hint?: string;
|
||||
private backend: "pulse" | null = null;
|
||||
private playerAvailable = false;
|
||||
private ffmpegAvailable = false;
|
||||
|
||||
async detect(): Promise<void> {
|
||||
const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]);
|
||||
const ffmpeg = await run(process.env.VESKSLOP_FFMPEG ?? "ffmpeg", ["-version"]);
|
||||
this.playerAvailable = ffplay.code === 0;
|
||||
this.ffmpegAvailable = ffmpeg.code === 0;
|
||||
|
||||
let info = await run("pactl", ["info"]);
|
||||
if (info.code !== 0 && info.errorCode !== "ENOENT" && process.env.VESKSLOP_AUDIO_MANAGE_SERVER !== "0") {
|
||||
await this.tryStartServer();
|
||||
info = await run("pactl", ["info"]);
|
||||
}
|
||||
|
||||
this.available = info.code === 0;
|
||||
this.backend = this.available ? "pulse" : null;
|
||||
|
||||
if (this.available) {
|
||||
this.error = undefined;
|
||||
this.hint = this.playerAvailable ? undefined : "ffplay is missing; install ffmpeg to play audio.";
|
||||
return;
|
||||
}
|
||||
|
||||
if (info.errorCode === "ENOENT") {
|
||||
this.error = "pactl is not installed";
|
||||
this.hint =
|
||||
"Install a sound server and its tools, for example: apt install pulseaudio pulseaudio-utils ffmpeg";
|
||||
} else {
|
||||
this.error = info.stderr.trim() || "the audio server is not reachable";
|
||||
this.hint =
|
||||
"The audio server is not running. Start PulseAudio or PipeWire-Pulse. The daemon starts pulseaudio automatically when it is installed.";
|
||||
}
|
||||
}
|
||||
|
||||
/** Start a private PulseAudio daemon when the server is missing. */
|
||||
private async tryStartServer(): Promise<void> {
|
||||
const start = await run("pulseaudio", ["--start", "--exit-idle-time=-1"]);
|
||||
if (start.errorCode === "ENOENT") {
|
||||
this.hint = "pulseaudio is not installed; install it or start PipeWire-Pulse.";
|
||||
return;
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < 6; attempt++) {
|
||||
await sleep(500);
|
||||
const info = await run("pactl", ["info"]);
|
||||
if (info.code === 0) return;
|
||||
}
|
||||
}
|
||||
|
||||
status(): AudioManagerStatus {
|
||||
return {
|
||||
available: this.available,
|
||||
backend: this.backend,
|
||||
error: this.error,
|
||||
hint: this.hint,
|
||||
sink: SINK,
|
||||
source: SOURCE,
|
||||
captureSink: CAPTURE_SINK,
|
||||
ducked: this.ducked,
|
||||
duckPercent: DUCK_PERCENT,
|
||||
playing: !!this.player,
|
||||
recording: !!this.recorder,
|
||||
player: this.playerAvailable,
|
||||
ffmpeg: this.ffmpegAvailable
|
||||
};
|
||||
}
|
||||
|
||||
/** Create the virtual devices when they are missing. Safe to call twice. */
|
||||
async ensureDevices(): Promise<void> {
|
||||
if (!this.available) await this.detect();
|
||||
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
|
||||
|
||||
const sinks = (await this.pactl(["list", "short", "sinks"])).split("\n");
|
||||
const sources = (await this.pactl(["list", "short", "sources"])).split("\n");
|
||||
|
||||
if (!sinks.some(line => line.includes(SINK))) {
|
||||
const id = await this.pactl([
|
||||
"load-module",
|
||||
"module-null-sink",
|
||||
`sink_name=${SINK}`,
|
||||
"sink_properties=device.description=VeskslopTTS"
|
||||
]);
|
||||
this.modules.push(Number(id));
|
||||
}
|
||||
|
||||
if (!sources.some(line => line.includes(SOURCE))) {
|
||||
const id = await this.pactl([
|
||||
"load-module",
|
||||
"module-remap-source",
|
||||
`master=${SINK}.monitor`,
|
||||
`source_name=${SOURCE}`,
|
||||
"source_properties=device.description=VeskslopMic"
|
||||
]);
|
||||
this.modules.push(Number(id));
|
||||
}
|
||||
|
||||
if (!sinks.some(line => line.includes(CAPTURE_SINK))) {
|
||||
const id = await this.pactl([
|
||||
"load-module",
|
||||
"module-null-sink",
|
||||
`sink_name=${CAPTURE_SINK}`,
|
||||
"sink_properties=device.description=VeskslopCapture"
|
||||
]);
|
||||
this.modules.push(Number(id));
|
||||
}
|
||||
}
|
||||
|
||||
/** Stop children and unload the modules this daemon created. */
|
||||
async dispose(): Promise<void> {
|
||||
this.stopPlayback();
|
||||
this.stopCapture();
|
||||
if (this.ducked) await this.setDuck(false).catch(() => undefined);
|
||||
|
||||
for (const id of [...this.modules].reverse()) {
|
||||
await run("pactl", ["unload-module", String(id)]);
|
||||
}
|
||||
this.modules = [];
|
||||
}
|
||||
|
||||
async play(file: string): Promise<void> {
|
||||
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
|
||||
|
||||
const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay";
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], {
|
||||
env: { ...process.env, PULSE_SINK: SINK },
|
||||
stdio: ["ignore", "ignore", "pipe"]
|
||||
});
|
||||
|
||||
this.player = child;
|
||||
let stderr = "";
|
||||
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error => {
|
||||
this.player = undefined;
|
||||
reject(new AudioUnavailableError(String(error)));
|
||||
});
|
||||
child.on("close", code => {
|
||||
this.player = undefined;
|
||||
if (code === 0 || code === null) resolve();
|
||||
else reject(new AudioUnavailableError(stderr.trim() || `the player exited with ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
stopPlayback(): void {
|
||||
try {
|
||||
this.player?.kill("SIGTERM");
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
this.player = undefined;
|
||||
}
|
||||
|
||||
/** Lower the virtual sink volume while other people speak. */
|
||||
async setDuck(on: boolean): Promise<void> {
|
||||
if (this.ducked === on) return;
|
||||
await this.pactl(["set-sink-volume", SINK, on ? `${DUCK_PERCENT}%` : "100%"]);
|
||||
this.ducked = on;
|
||||
}
|
||||
|
||||
async startCapture(file: string): Promise<void> {
|
||||
if (this.recorder) throw new Error("capture is already running");
|
||||
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
|
||||
|
||||
const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
|
||||
const child = spawn(
|
||||
binary,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"pulse",
|
||||
"-i",
|
||||
`${CAPTURE_SINK}.monitor`,
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-y",
|
||||
file
|
||||
],
|
||||
{ stdio: ["ignore", "ignore", "pipe"] }
|
||||
);
|
||||
|
||||
this.recorder = child;
|
||||
child.on("close", () => {
|
||||
if (this.recorder === child) this.recorder = undefined;
|
||||
});
|
||||
}
|
||||
|
||||
stopCapture(): void {
|
||||
const child = this.recorder;
|
||||
this.recorder = undefined;
|
||||
try {
|
||||
child?.kill("SIGINT");
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
|
||||
/** Stream the capture sink as raw 48 kHz mono PCM. Returns a stop function. */
|
||||
startMonitor(onData: (chunk: Buffer) => void, onError?: (error: Error) => void): () => void {
|
||||
if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available");
|
||||
if (!this.ffmpegAvailable) {
|
||||
throw new AudioUnavailableError("ffmpeg is missing; install it to hear the call");
|
||||
}
|
||||
|
||||
const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg";
|
||||
const child = spawn(
|
||||
binary,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"pulse",
|
||||
"-i",
|
||||
`${CAPTURE_SINK}.monitor`,
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-f",
|
||||
"s16le",
|
||||
"-"
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] }
|
||||
);
|
||||
|
||||
child.stdout?.on("data", onData);
|
||||
let stderr = "";
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error => onError?.(new AudioUnavailableError(String(error))));
|
||||
child.on("close", code => {
|
||||
if (code) onError?.(new AudioUnavailableError(stderr.trim() || `ffmpeg exited with ${code}`));
|
||||
});
|
||||
|
||||
return () => {
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async pactl(args: string[]): Promise<string> {
|
||||
const result = await run("pactl", args);
|
||||
if (result.code !== 0) {
|
||||
throw new AudioUnavailableError(result.stderr.trim() || `pactl ${args[0]} failed`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Veskslop daemon — capture pipeline.
|
||||
*
|
||||
* Reads the client output monitor as raw PCM, splits it into utterances with
|
||||
* the VAD, and writes one WAV per utterance. The caller stores the metadata.
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { pcmToWav, Vad, type VadConfig, type VadSegment } from "./vad.js";
|
||||
|
||||
export interface CaptureSegment {
|
||||
file: string;
|
||||
durationMs: number;
|
||||
startedAt: string;
|
||||
endedAt: string;
|
||||
}
|
||||
|
||||
export interface CapturePipelineOptions {
|
||||
ffmpeg: string;
|
||||
target: string;
|
||||
directory: string;
|
||||
sessionId: string;
|
||||
vad: VadConfig;
|
||||
onSegment: (segment: CaptureSegment) => void;
|
||||
onError: (error: Error) => void;
|
||||
}
|
||||
|
||||
export class CapturePipeline {
|
||||
private child?: ChildProcess;
|
||||
private vad?: Vad;
|
||||
private counter = 0;
|
||||
private stopped = false;
|
||||
|
||||
constructor(private readonly options: CapturePipelineOptions) {}
|
||||
|
||||
start() {
|
||||
mkdirSync(this.options.directory, { recursive: true });
|
||||
this.vad = new Vad(this.options.vad, segment => this.emit(segment));
|
||||
|
||||
this.child = spawn(
|
||||
this.options.ffmpeg,
|
||||
[
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"pulse",
|
||||
"-i",
|
||||
this.options.target,
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
String(this.options.vad.sampleRate),
|
||||
"-f",
|
||||
"s16le",
|
||||
"-"
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] }
|
||||
);
|
||||
|
||||
let stderr = "";
|
||||
this.child.stdout?.on("data", chunk => this.vad?.push(chunk));
|
||||
this.child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
this.child.on("error", error => this.options.onError(new Error(String(error))));
|
||||
this.child.on("close", code => {
|
||||
if (!this.stopped && code) {
|
||||
this.options.onError(new Error(stderr.trim() || `ffmpeg exited with ${code}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stopped = true;
|
||||
this.vad?.flush();
|
||||
this.vad = undefined;
|
||||
|
||||
try {
|
||||
this.child?.kill("SIGTERM");
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
this.child = undefined;
|
||||
}
|
||||
|
||||
private emit(segment: VadSegment) {
|
||||
this.counter++;
|
||||
const endedAt = new Date();
|
||||
const startedAt = new Date(endedAt.getTime() - segment.durationMs);
|
||||
const file = join(this.options.directory, `${this.options.sessionId}-${this.counter}.wav`);
|
||||
|
||||
writeFileSync(file, pcmToWav(segment.pcm, this.options.vad?.sampleRate ?? 16000));
|
||||
|
||||
this.options.onSegment({
|
||||
file,
|
||||
durationMs: segment.durationMs,
|
||||
startedAt: startedAt.toISOString(),
|
||||
endedAt: endedAt.toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
+210
-7
@@ -15,10 +15,16 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
|
||||
import * as z from "zod/v4";
|
||||
|
||||
import { AuthService } from "./auth-service.js";
|
||||
import { AudioService } from "./audio-service.js";
|
||||
import { tokenFile } from "./config.js";
|
||||
import { dirname, join } from "node:path";
|
||||
import { Stt } from "./stt.js";
|
||||
import { TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD } from "./vad.js";
|
||||
import { CoreClient, coreSocketPath } from "./core-client.js";
|
||||
import { CoreProcess } from "./core-process.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, sleep } from "./pacing.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js";
|
||||
import { PreferencesStore } from "./preferences.js";
|
||||
import { Puppet } from "./puppet.js";
|
||||
import { startWebUi } from "./webui.js";
|
||||
|
||||
@@ -40,10 +46,11 @@ async function connectLoop(): Promise<void> {
|
||||
const client = await CoreClient.connect();
|
||||
|
||||
core = client;
|
||||
puppet = new Puppet(client);
|
||||
puppet = new Puppet(client, audio);
|
||||
|
||||
client.onEvent(event => {
|
||||
void puppet!.enqueueEvent(event);
|
||||
void audio.onEvent(event);
|
||||
webui.broadcast(event);
|
||||
});
|
||||
|
||||
@@ -59,6 +66,7 @@ async function connectLoop(): Promise<void> {
|
||||
coreConnected = true;
|
||||
console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`);
|
||||
webui.broadcast({ type: "CORE_STATUS", coreConnected: true });
|
||||
void autoConnectAudio(client);
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
@@ -83,6 +91,81 @@ function requirePuppet(): Puppet {
|
||||
|
||||
const coreProcess = new CoreProcess({ tokenFile });
|
||||
|
||||
const transcripts = new TranscriptStore(
|
||||
process.env.VESKSLOP_TRANSCRIPT ?? join(dirname(tokenFile), "transcripts.jsonl")
|
||||
);
|
||||
const preferences = new PreferencesStore(
|
||||
process.env.VESKSLOP_PREFERENCES ?? join(dirname(tokenFile), "preferences.json"),
|
||||
{ legacyFile: process.env.VESKSLOP_STATE_FILE ?? join(dirname(tokenFile), ".veskslop-state.json") }
|
||||
);
|
||||
const stt = new Stt();
|
||||
|
||||
const audio = new AudioService({
|
||||
cacheDir: process.env.VESKSLOP_CACHE_DIR,
|
||||
transcripts,
|
||||
stt,
|
||||
autoCapture: process.env.VESKSLOP_CAPTURE_AUTO === "1",
|
||||
vad: {
|
||||
...DEFAULT_VAD,
|
||||
thresholdDb: Number(process.env.VESKSLOP_VAD_THRESHOLD_DB ?? DEFAULT_VAD.thresholdDb),
|
||||
minSpeechMs: Number(process.env.VESKSLOP_VAD_MIN_SPEECH_MS ?? DEFAULT_VAD.minSpeechMs),
|
||||
silenceMs: Number(process.env.VESKSLOP_VAD_SILENCE_MS ?? DEFAULT_VAD.silenceMs),
|
||||
preRollMs: Number(process.env.VESKSLOP_VAD_PRE_ROLL_MS ?? DEFAULT_VAD.preRollMs),
|
||||
maxSegmentMs: Number(process.env.VESKSLOP_VAD_MAX_SEGMENT_MS ?? DEFAULT_VAD.maxSegmentMs)
|
||||
},
|
||||
getSelfId: async () => (core ? await core.getSelfId() : undefined),
|
||||
getVoiceContext: async () => {
|
||||
if (!core) return {};
|
||||
const state = (await core.voiceGet()) as { guildId?: string | null; channelId?: string | null };
|
||||
return { guildId: state?.guildId ?? null, channelId: state?.channelId ?? null };
|
||||
},
|
||||
getSpeakerName: async userId => {
|
||||
if (!core) return undefined;
|
||||
const state = (await core.voiceGet(userId)) as {
|
||||
members?: Array<{ id: string; username?: string }>;
|
||||
};
|
||||
return state?.members?.find(member => member.id === userId)?.username;
|
||||
},
|
||||
onTranscript: event => webui.broadcast(event)
|
||||
});
|
||||
|
||||
void audio.init();
|
||||
|
||||
/** Point the client at the virtual microphone and capture sink when present. */
|
||||
async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolean; message: string }> {
|
||||
if (process.env.VESKSLOP_AUDIO_AUTOCONNECT === "0") {
|
||||
return { connected: false, message: "auto-connect is disabled" };
|
||||
}
|
||||
|
||||
const status = audio.status();
|
||||
if (!status.available) {
|
||||
return { connected: false, message: status.hint ?? status.error ?? "audio is not available" };
|
||||
}
|
||||
|
||||
try {
|
||||
await audio.manager.ensureDevices();
|
||||
|
||||
const devices = (await client.voiceDevices()) as {
|
||||
input?: Array<{ id: string; label: string }>;
|
||||
output?: Array<{ id: string; label: string }>;
|
||||
};
|
||||
|
||||
const input = devices.input?.find(device => device.label.includes("VeskslopMic"));
|
||||
const output = devices.output?.find(device => device.label.includes("VeskslopCapture"));
|
||||
|
||||
if (!input && !output) {
|
||||
return { connected: false, message: "the client does not see the virtual devices yet" };
|
||||
}
|
||||
|
||||
const result = await client.voiceSetDevices(input?.id, output?.id);
|
||||
console.log("[audio] client devices set:", JSON.stringify(result));
|
||||
return { connected: true, message: "the client now uses the virtual devices" };
|
||||
} catch (error) {
|
||||
console.warn("[audio] device auto-connect failed:", (error as Error).message);
|
||||
return { connected: false, message: (error as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
const auth = new AuthService(tokenFile, update => {
|
||||
webui.broadcast({ type: "AUTH_QR", update });
|
||||
|
||||
@@ -100,7 +183,11 @@ const webui = startWebUi({
|
||||
getPuppet: () => puppet,
|
||||
isCoreConnected: () => coreConnected,
|
||||
coreProcess,
|
||||
auth
|
||||
auth,
|
||||
audio,
|
||||
transcripts,
|
||||
preferences,
|
||||
connectAudioDevices: () => autoConnectAudio(requireCore())
|
||||
});
|
||||
|
||||
coreProcess.onExit(() => {
|
||||
@@ -118,6 +205,9 @@ const READ_ONLY_TOOLS: Record<string, (args: any) => Promise<unknown> | unknown>
|
||||
get_voice_overview: args => requireCore().voiceOverview(args?.guildId),
|
||||
search_users: args => requireCore().userSearch(args?.query, args?.guildId, args?.limit),
|
||||
list_dms: args => requireCore().dmList(args?.limit),
|
||||
get_audio_status: () => audio.status(),
|
||||
list_audio_devices: () => requireCore().voiceDevices(),
|
||||
get_transcript: args => transcripts.readSegments(args?.limit ?? 100, args?.sessionId),
|
||||
get_context: args =>
|
||||
requirePuppet().getContext(args?.channelId, {
|
||||
maxMessages: args?.maxMessages,
|
||||
@@ -268,6 +358,110 @@ function buildServer(): McpServer {
|
||||
async ({ guildId }) => jsonText(await requireCore().voiceOverview(guildId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_join",
|
||||
{
|
||||
description: "Join a voice channel as the logged-in user. Joins muted by default.",
|
||||
inputSchema: {
|
||||
channelId: z.string().describe("Voice channel ID"),
|
||||
mute: z.boolean().optional().describe("Join muted. Defaults to true.")
|
||||
}
|
||||
},
|
||||
async ({ channelId, mute }) =>
|
||||
jsonText(await pacedVoice(() => requireCore().voiceJoin(channelId, mute ?? true)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_leave",
|
||||
{
|
||||
description: "Leave the current voice channel."
|
||||
},
|
||||
async () => jsonText(await pacedVoice(() => requireCore().voiceLeave()))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"voice_move",
|
||||
{
|
||||
description: "Move the client to another voice channel.",
|
||||
inputSchema: {
|
||||
channelId: z.string().describe("Voice channel ID")
|
||||
}
|
||||
},
|
||||
async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId)))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_audio_status",
|
||||
{
|
||||
description: "Audio stack, virtual devices, TTS providers, queue, ducking, and capture state."
|
||||
},
|
||||
async () => jsonText(audio.status())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"speak",
|
||||
{
|
||||
description: "Queue text to speech into the bot's virtual microphone.",
|
||||
inputSchema: {
|
||||
text: z.string().min(1).max(1000).describe("Text to speak"),
|
||||
provider: z.enum(["local", "api"]).optional().describe("TTS provider"),
|
||||
voice: z.string().optional().describe("Voice name for the provider")
|
||||
}
|
||||
},
|
||||
async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice }))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_audio",
|
||||
{
|
||||
description: "Stop speech and clear the queue."
|
||||
},
|
||||
async () => {
|
||||
audio.stop();
|
||||
return jsonText(audio.status());
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"start_capture",
|
||||
{
|
||||
description: "Record the client output sink to a WAV file on the host."
|
||||
},
|
||||
async () => jsonText(await audio.startCapture())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"stop_capture",
|
||||
{
|
||||
description: "Stop the current capture."
|
||||
},
|
||||
async () => {
|
||||
audio.stopCapture();
|
||||
return jsonText(audio.status());
|
||||
}
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"list_audio_devices",
|
||||
{
|
||||
description: "List the client's input and output audio devices."
|
||||
},
|
||||
async () => jsonText(await requireCore().voiceDevices())
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"set_audio_devices",
|
||||
{
|
||||
description: "Set the client's input and output audio devices.",
|
||||
inputSchema: {
|
||||
inputDeviceId: z.string().optional().describe("Input device ID"),
|
||||
outputDeviceId: z.string().optional().describe("Output device ID")
|
||||
}
|
||||
},
|
||||
async ({ inputDeviceId, outputDeviceId }) =>
|
||||
jsonText(await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"search_users",
|
||||
{
|
||||
@@ -303,6 +497,18 @@ function buildServer(): McpServer {
|
||||
async ({ limit }) => jsonText(requirePuppet().readActionLog(limit))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"get_transcript",
|
||||
{
|
||||
description: "Read captured call segments: time, speaker, WAV path, and text when STT is configured.",
|
||||
inputSchema: {
|
||||
limit: z.number().int().min(1).max(500).default(100).describe("How many segments"),
|
||||
sessionId: z.string().optional().describe("Only this capture session")
|
||||
}
|
||||
},
|
||||
async ({ limit, sessionId }) => jsonText(transcripts.readSegments(limit, sessionId))
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"run_commands",
|
||||
{
|
||||
@@ -427,10 +633,7 @@ app.listen(PORT, HOST, (error?: Error) => {
|
||||
|
||||
function shutdown() {
|
||||
core?.close();
|
||||
void coreProcess
|
||||
.dispose(4000)
|
||||
.catch(() => undefined)
|
||||
.finally(() => process.exit(0));
|
||||
void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0));
|
||||
}
|
||||
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
+54
-1
@@ -25,8 +25,11 @@ export type CoreEventListener = (event: any) => void;
|
||||
interface Pending {
|
||||
resolve: (value: any) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
const REQUEST_TIMEOUT_MS = Number(process.env.VESKSLOP_REQUEST_TIMEOUT_MS ?? 30000);
|
||||
|
||||
export class CoreClient {
|
||||
private nextId = 1;
|
||||
private readonly pending = new Map<number, Pending>();
|
||||
@@ -50,6 +53,7 @@ export class CoreClient {
|
||||
|
||||
socket.on("close", () => {
|
||||
for (const request of this.pending.values()) {
|
||||
clearTimeout(request.timer);
|
||||
request.reject(new CoreError("DISCONNECTED", "core disconnected"));
|
||||
}
|
||||
this.pending.clear();
|
||||
@@ -93,6 +97,7 @@ export class CoreClient {
|
||||
const request = this.pending.get(message.id);
|
||||
if (!request) return;
|
||||
this.pending.delete(message.id);
|
||||
clearTimeout(request.timer);
|
||||
|
||||
if (message.ok) request.resolve(message.result);
|
||||
else request.reject(new CoreError(message.error?.code ?? "UNKNOWN", message.error?.message ?? "core error"));
|
||||
@@ -102,7 +107,24 @@ export class CoreClient {
|
||||
const id = this.nextId++;
|
||||
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
this.pending.set(id, { resolve, reject });
|
||||
const timer = setTimeout(() => {
|
||||
if (this.pending.delete(id)) {
|
||||
reject(new CoreError("TIMEOUT", `the core did not answer ${action}`));
|
||||
}
|
||||
}, REQUEST_TIMEOUT_MS);
|
||||
|
||||
this.pending.set(id, {
|
||||
resolve: value => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
reject: error => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
},
|
||||
timer
|
||||
});
|
||||
|
||||
this.socket.write(JSON.stringify({ id, action, params }) + "\n");
|
||||
});
|
||||
}
|
||||
@@ -158,6 +180,37 @@ export class CoreClient {
|
||||
return this.request("voice.overview", { guildId });
|
||||
}
|
||||
|
||||
async voiceJoin(channelId: string, mute = true) {
|
||||
return this.request("voice.join", { channelId, mute });
|
||||
}
|
||||
|
||||
async voiceLeave() {
|
||||
return this.request("voice.leave");
|
||||
}
|
||||
|
||||
async voiceMove(channelId: string) {
|
||||
return this.request("voice.move", { channelId });
|
||||
}
|
||||
|
||||
async voiceUsers(userIds: string[]) {
|
||||
return this.request("voice.users", { userIds });
|
||||
}
|
||||
|
||||
async userStatus(userIds: string[]) {
|
||||
return this.request("user.status", { userIds });
|
||||
}
|
||||
|
||||
async voiceDevices() {
|
||||
return this.request("voice.devices");
|
||||
}
|
||||
|
||||
async voiceSetDevices(inputDeviceId?: string, outputDeviceId?: string) {
|
||||
return this.request("voice.setDevices", {
|
||||
...(inputDeviceId ? { inputDeviceId } : {}),
|
||||
...(outputDeviceId ? { outputDeviceId } : {})
|
||||
});
|
||||
}
|
||||
|
||||
async userSearch(query: string, guildId?: string, limit?: number) {
|
||||
return this.request("user.search", {
|
||||
query,
|
||||
|
||||
+79
-20
@@ -94,36 +94,76 @@ const FAKE_HISTORY = Array.from({ length: 8 }, (_, index) => ({
|
||||
: []
|
||||
}));
|
||||
|
||||
const MOCK_VOICE_CHANNEL = "200000000000000002";
|
||||
let botVoiceChannel: string | null = MOCK_VOICE_CHANNEL;
|
||||
|
||||
const STATE_FAIL_MS = Number(process.env.VESKSLOP_MOCK_STATE_FAIL_MS ?? 0);
|
||||
const startedAt = Date.now();
|
||||
|
||||
function mockVoiceMembers() {
|
||||
return [
|
||||
{ id: "999", username: "mockuser", avatar: null, mute: false, deaf: false },
|
||||
{ id: "177117296140091393", username: "zzib", avatar: null, mute: false, deaf: false }
|
||||
];
|
||||
}
|
||||
|
||||
function handle(action: string | undefined, params: any) {
|
||||
switch (action) {
|
||||
case "ping":
|
||||
return "pong";
|
||||
case "state.get":
|
||||
if (STATE_FAIL_MS && Date.now() - startedAt < STATE_FAIL_MS) {
|
||||
throw new Error("renderer is not ready");
|
||||
}
|
||||
return { user: { id: "999", username: "mockuser", avatar: null }, guilds: FAKE_GUILDS };
|
||||
case "voice.get":
|
||||
if (!botVoiceChannel) {
|
||||
return { guildId: params?.guildId ?? null, channelId: null, channelName: null, members: [] };
|
||||
}
|
||||
return {
|
||||
guildId: params?.guildId ?? "100000000000000001",
|
||||
channelId: "200000000000000002",
|
||||
channelId: botVoiceChannel,
|
||||
channelName: "voice",
|
||||
members: [
|
||||
{ id: params?.userId ?? "999", username: "mockuser", mute: false, deaf: false },
|
||||
{ id: "177117296140091393", username: "zzib", mute: false, deaf: false }
|
||||
]
|
||||
members: mockVoiceMembers()
|
||||
};
|
||||
case "voice.overview":
|
||||
return {
|
||||
guildId: params?.guildId,
|
||||
channels: [
|
||||
{
|
||||
channelId: "200000000000000002",
|
||||
channelName: "voice",
|
||||
members: [
|
||||
{ id: "999", username: "mockuser", mute: false, deaf: false },
|
||||
{ id: "177117296140091393", username: "zzib", mute: false, deaf: false }
|
||||
]
|
||||
}
|
||||
]
|
||||
channels: botVoiceChannel
|
||||
? [{ channelId: botVoiceChannel, channelName: "voice", members: mockVoiceMembers() }]
|
||||
: []
|
||||
};
|
||||
case "voice.join":
|
||||
case "voice.move":
|
||||
botVoiceChannel = params?.channelId ?? null;
|
||||
return { channelId: params?.channelId, joined: true, muted: params?.mute !== false };
|
||||
case "voice.leave":
|
||||
botVoiceChannel = null;
|
||||
return { left: true };
|
||||
case "voice.devices":
|
||||
return {
|
||||
input: [{ id: "mock-mic", label: "VeskslopMic (mock)" }],
|
||||
output: [{ id: "mock-out", label: "VeskslopCapture (mock)" }]
|
||||
};
|
||||
case "voice.users":
|
||||
return {
|
||||
users: (params?.userIds ?? []).map((id: string) => ({
|
||||
id,
|
||||
channelId: id === "999" ? MOCK_VOICE_CHANNEL : null,
|
||||
channelName: id === "999" ? "voice" : null,
|
||||
guildId: id === "999" ? "100000000000000001" : null,
|
||||
members: id === "999" ? mockVoiceMembers() : []
|
||||
}))
|
||||
};
|
||||
case "user.status":
|
||||
return {
|
||||
users: (params?.userIds ?? []).map((id: string) => ({
|
||||
id,
|
||||
status: id === "999" ? "online" : id === "177117296140091393" ? "idle" : "offline"
|
||||
}))
|
||||
};
|
||||
case "voice.setDevices":
|
||||
return { inputDeviceId: params?.inputDeviceId ?? null, outputDeviceId: params?.outputDeviceId ?? null };
|
||||
case "user.search": {
|
||||
const needle = String(params?.query ?? "")
|
||||
.replace(/^@/, "")
|
||||
@@ -144,8 +184,18 @@ function handle(action: string | undefined, params: any) {
|
||||
name: null,
|
||||
icon: null,
|
||||
lastMessageId: "710000000000000001",
|
||||
lastMessageAt: new Date(Date.now() - 42 * 60_000).toISOString(),
|
||||
unread: 2,
|
||||
mentions: 0,
|
||||
recipients: [
|
||||
{ id: "177117296140091393", username: "zzib", globalName: "Zzib", avatar: null, bot: false }
|
||||
{
|
||||
id: "177117296140091393",
|
||||
username: "zzib",
|
||||
globalName: "Zzib",
|
||||
avatar: null,
|
||||
status: "idle",
|
||||
bot: false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -154,9 +204,12 @@ function handle(action: string | undefined, params: any) {
|
||||
name: "oboys + 2",
|
||||
icon: null,
|
||||
lastMessageId: "710000000000000002",
|
||||
lastMessageAt: new Date(Date.now() - 26 * 60 * 60_000).toISOString(),
|
||||
unread: 0,
|
||||
mentions: 3,
|
||||
recipients: [
|
||||
{ id: "888", username: "testbot", globalName: null, avatar: null, bot: true },
|
||||
{ id: "999", username: "mockuser", globalName: "Mock User", avatar: null, bot: false }
|
||||
{ id: "888", username: "testbot", globalName: null, avatar: null, status: "offline", bot: true },
|
||||
{ id: "999", username: "mockuser", globalName: "Mock User", avatar: null, status: "online", bot: false }
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -208,9 +261,15 @@ const server = createServer(socket => {
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (!line) continue;
|
||||
|
||||
let request: any;
|
||||
try {
|
||||
const request = JSON.parse(line);
|
||||
request = JSON.parse(line);
|
||||
} catch {
|
||||
socket.write(JSON.stringify({ ok: false, error: { code: "INVALID_REQUEST", message: "invalid JSON" } }) + "\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
if (request.action === "events.subscribe") {
|
||||
subscribers.add(socket);
|
||||
socket.write(JSON.stringify({ id: request.id, ok: true, result: { subscribed: true } }) + "\n");
|
||||
@@ -225,7 +284,7 @@ const server = createServer(socket => {
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.log(`[mock] ${new Date().toISOString()} ERROR ${message}`);
|
||||
socket.write(JSON.stringify({ ok: false, error: { code: "UNKNOWN_ACTION", message } }) + "\n");
|
||||
socket.write(JSON.stringify({ id: request.id, ok: false, error: { code: "UNKNOWN_ACTION", message } }) + "\n");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
+20
-1
@@ -21,7 +21,9 @@ export const PACING = {
|
||||
idleMinMs: 5000,
|
||||
idleMaxMs: 20000,
|
||||
readMinMs: 100,
|
||||
readMaxMs: 300
|
||||
readMaxMs: 300,
|
||||
voiceMinMs: 250,
|
||||
voiceMaxMs: 750
|
||||
};
|
||||
|
||||
function randomBetween(min: number, max: number): number {
|
||||
@@ -95,6 +97,23 @@ export function paced<T>(task: () => Promise<T>): Promise<T> {
|
||||
return run;
|
||||
}
|
||||
|
||||
/** Serialize a voice action with a short gap. Voice state changes are not sends. */
|
||||
export function pacedVoice<T>(task: () => Promise<T>): Promise<T> {
|
||||
const delay = randomBetween(PACING.voiceMinMs, PACING.voiceMaxMs);
|
||||
const run = queue.then(async () => {
|
||||
await sleep(delay);
|
||||
if (process.env.DEBUG === "1") {
|
||||
console.error(`[voice] action (+${delay}ms)`);
|
||||
}
|
||||
return task();
|
||||
});
|
||||
queue = run.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
);
|
||||
return run;
|
||||
}
|
||||
|
||||
export function getQueryCount(): number {
|
||||
return queryCount;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Veskslop daemon — web UI preferences.
|
||||
*
|
||||
* Stored next to the token file, not in the puppet state, so the web UI can
|
||||
* read and write them while the Discord client is disconnected.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
export interface TtsPreferences {
|
||||
provider: "local" | "api" | null;
|
||||
language: string | null;
|
||||
voice: string | null;
|
||||
}
|
||||
|
||||
export interface FavouritesPreferences {
|
||||
users: string[];
|
||||
channels: string[];
|
||||
}
|
||||
|
||||
export interface WebPreferences {
|
||||
/** null means every guild is visible. An empty array hides all. */
|
||||
visibleGuilds: string[] | null;
|
||||
tts: TtsPreferences;
|
||||
favourites: FavouritesPreferences;
|
||||
}
|
||||
|
||||
const DEFAULTS: WebPreferences = {
|
||||
visibleGuilds: null,
|
||||
tts: { provider: null, language: null, voice: null },
|
||||
favourites: { users: [], channels: [] }
|
||||
};
|
||||
|
||||
export interface PreferencesOptions {
|
||||
/** The old state file that may still hold preferences. */
|
||||
legacyFile?: string;
|
||||
}
|
||||
|
||||
export class PreferencesStore {
|
||||
private preferences: WebPreferences = structuredClone(DEFAULTS);
|
||||
|
||||
constructor(
|
||||
private readonly file = process.env.VESKSLOP_PREFERENCES ?? ".veskslop-preferences.json",
|
||||
options: PreferencesOptions = {}
|
||||
) {
|
||||
try {
|
||||
if (existsSync(this.file)) {
|
||||
this.preferences = this.normalize(JSON.parse(readFileSync(this.file, "utf8")));
|
||||
} else {
|
||||
this.migrate(options.legacyFile);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("[preferences] could not read", this.file, (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
get(): WebPreferences {
|
||||
return structuredClone(this.preferences);
|
||||
}
|
||||
|
||||
update(patch: Partial<WebPreferences>): WebPreferences {
|
||||
if (patch.visibleGuilds !== undefined) {
|
||||
this.preferences.visibleGuilds = Array.isArray(patch.visibleGuilds)
|
||||
? [...new Set(patch.visibleGuilds.map(String))]
|
||||
: null;
|
||||
}
|
||||
|
||||
if (patch.tts !== undefined) {
|
||||
this.preferences.tts = {
|
||||
provider: patch.tts?.provider === "api" ? "api" : patch.tts?.provider === "local" ? "local" : null,
|
||||
language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null,
|
||||
voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null
|
||||
};
|
||||
}
|
||||
|
||||
if (patch.favourites !== undefined) {
|
||||
this.preferences.favourites = {
|
||||
users: Array.isArray(patch.favourites?.users)
|
||||
? [...new Set(patch.favourites.users.map(String))].slice(0, 100)
|
||||
: [],
|
||||
channels: Array.isArray(patch.favourites?.channels)
|
||||
? [...new Set(patch.favourites.channels.map(String))].slice(0, 100)
|
||||
: []
|
||||
};
|
||||
}
|
||||
|
||||
this.save();
|
||||
return this.get();
|
||||
}
|
||||
|
||||
private migrate(legacyFile?: string) {
|
||||
if (!legacyFile || !existsSync(legacyFile)) return;
|
||||
|
||||
try {
|
||||
const legacy = JSON.parse(readFileSync(legacyFile, "utf8"));
|
||||
if (legacy?.preferences) {
|
||||
this.preferences = this.normalize(legacy.preferences);
|
||||
this.save();
|
||||
console.log("[preferences] migrated from", legacyFile);
|
||||
}
|
||||
} catch {
|
||||
// nothing to migrate
|
||||
}
|
||||
}
|
||||
|
||||
private normalize(raw: any): WebPreferences {
|
||||
const provider = raw?.tts?.provider;
|
||||
|
||||
return {
|
||||
visibleGuilds: Array.isArray(raw?.visibleGuilds)
|
||||
? [...new Set<string>(raw.visibleGuilds.map((id: unknown) => String(id)))]
|
||||
: null,
|
||||
tts: {
|
||||
provider: provider === "api" || provider === "local" ? provider : null,
|
||||
language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null,
|
||||
voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null
|
||||
},
|
||||
favourites: {
|
||||
users: Array.isArray(raw?.favourites?.users)
|
||||
? [...new Set<string>(raw.favourites.users.map((id: unknown) => String(id)))].slice(0, 100)
|
||||
: [],
|
||||
channels: Array.isArray(raw?.favourites?.channels)
|
||||
? [...new Set<string>(raw.favourites.channels.map((id: unknown) => String(id)))].slice(0, 100)
|
||||
: []
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private save() {
|
||||
try {
|
||||
mkdirSync(dirname(this.file), { recursive: true });
|
||||
writeFileSync(this.file, JSON.stringify(this.preferences, null, 4));
|
||||
} catch (error) {
|
||||
console.error("[preferences] write failed:", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
+64
-4
@@ -99,6 +99,12 @@ export interface PuppetTransport {
|
||||
reactionAdd(channelId: string, messageId: string, emoji: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
/** The audio actions a chat command can trigger. */
|
||||
export interface PuppetAudio {
|
||||
speak(text: string, options?: { provider?: "local" | "api"; voice?: string }): Promise<unknown>;
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
interface PollState {
|
||||
channelId: string;
|
||||
authorId: string;
|
||||
@@ -142,7 +148,10 @@ export class Puppet {
|
||||
private selfId?: string;
|
||||
private queue: Promise<void> = Promise.resolve();
|
||||
|
||||
constructor(private readonly core: PuppetTransport) {
|
||||
constructor(
|
||||
private readonly core: PuppetTransport,
|
||||
private readonly audio?: PuppetAudio
|
||||
) {
|
||||
try {
|
||||
const state = JSON.parse(readFileSync(STATE_FILE, "utf8")) as PersistedState;
|
||||
if (state.ownerId) this.ownerId = state.ownerId;
|
||||
@@ -159,8 +168,7 @@ export class Puppet {
|
||||
|
||||
if (state.config) {
|
||||
this.config = {
|
||||
contextMessages: clamp(state.config.contextMessages ?? DEFAULT_CONFIG.contextMessages, 1, 50),
|
||||
contextChars: clamp(state.config.contextChars ?? DEFAULT_CONFIG.contextChars, 200, 20000),
|
||||
contextMessages: clamp(state.config.contextMessages ?? DEFAULT_CONFIG.contextMessages, 1, 50), contextChars: clamp(state.config.contextChars ?? DEFAULT_CONFIG.contextChars, 200, 20000),
|
||||
fetchCooldownSeconds: clamp(
|
||||
state.config.fetchCooldownSeconds ?? DEFAULT_CONFIG.fetchCooldownSeconds,
|
||||
0,
|
||||
@@ -181,6 +189,7 @@ export class Puppet {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
} catch {
|
||||
// first run: defaults apply
|
||||
}
|
||||
@@ -263,7 +272,11 @@ export class Puppet {
|
||||
writeFileSync(
|
||||
STATE_FILE,
|
||||
JSON.stringify(
|
||||
{ ownerId: this.ownerId, whitelist: this.whitelist, config: this.config },
|
||||
{
|
||||
ownerId: this.ownerId,
|
||||
whitelist: this.whitelist,
|
||||
config: this.config
|
||||
},
|
||||
null,
|
||||
4
|
||||
)
|
||||
@@ -446,6 +459,8 @@ export class Puppet {
|
||||
|
||||
if (command?.name === "whitelist") {
|
||||
await this.handleWhitelistCommand(command, author.id, tier, channelId);
|
||||
} else if (command?.name === "speak") {
|
||||
await this.handleSpeakCommand(command, author.id, channelId);
|
||||
} else if (POLL_ENABLED && command?.name === "poll") {
|
||||
await this.handlePollCommand(author.id, channelId, text);
|
||||
}
|
||||
@@ -541,6 +556,51 @@ export class Puppet {
|
||||
}
|
||||
}
|
||||
|
||||
/* Text to speech */
|
||||
|
||||
private async handleSpeakCommand(command: ParsedCommand, authorId: string, channelId: string) {
|
||||
if (!this.audio) {
|
||||
return this.respond(authorId, channelId, "speak", "Voice audio is not available.");
|
||||
}
|
||||
|
||||
const text = command.args.join(" ").trim();
|
||||
if (!text || text === "help") {
|
||||
return this.respond(
|
||||
authorId,
|
||||
channelId,
|
||||
"speak",
|
||||
["speak — text to speech.", "Usage:", " speak <text>", " speak stop"].join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
if (text === "stop") {
|
||||
this.audio.stop();
|
||||
this.actionLog.append({ type: "command", command: "speak stop", requesterId: authorId, result: "ok" });
|
||||
return this.respond(authorId, channelId, "speak", "Stopped.");
|
||||
}
|
||||
|
||||
try {
|
||||
await this.audio.speak(text, {});
|
||||
this.actionLog.append({
|
||||
type: "command",
|
||||
command: "speak",
|
||||
requesterId: authorId,
|
||||
result: "queued",
|
||||
chars: text.length
|
||||
});
|
||||
return this.respond(authorId, channelId, "speak", "Speaking.");
|
||||
} catch (error) {
|
||||
this.actionLog.append({
|
||||
type: "command",
|
||||
command: "speak",
|
||||
requesterId: authorId,
|
||||
result: "error",
|
||||
error: String(error)
|
||||
});
|
||||
return this.respond(authorId, channelId, "speak", `Could not speak: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/* Polls */
|
||||
|
||||
private async handlePollCommand(authorId: string, channelId: string, text: string) {
|
||||
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* Veskslop daemon — speech to text.
|
||||
*
|
||||
* Providers, in order: VESKSLOP_STT_CMD, a local Vosk model
|
||||
* (VESKSLOP_VOSK_MODEL or a `vosk-model*` folder under poc/models), and an
|
||||
* HTTP API. The local command receives the WAV path ({wav}) and prints text.
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export class SttError extends Error {}
|
||||
|
||||
export interface SttStatus {
|
||||
local: boolean;
|
||||
api: boolean;
|
||||
engine: string | null;
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
export class Stt {
|
||||
private voskModel?: string;
|
||||
|
||||
get localCommand(): string | undefined {
|
||||
return process.env.VESKSLOP_STT_CMD;
|
||||
}
|
||||
|
||||
get apiUrl(): string | undefined {
|
||||
return process.env.VESKSLOP_STT_API_URL;
|
||||
}
|
||||
|
||||
async detect(): Promise<void> {
|
||||
this.voskModel = this.findVoskModel();
|
||||
}
|
||||
|
||||
status(): SttStatus {
|
||||
const local = !!(this.localCommand || this.voskModel);
|
||||
|
||||
return {
|
||||
local,
|
||||
api: !!this.apiUrl,
|
||||
engine: this.localCommand ? "command" : this.voskModel ? "vosk" : null,
|
||||
hint:
|
||||
local || this.apiUrl
|
||||
? undefined
|
||||
: "Install vosk (pip install vosk) and set VESKSLOP_VOSK_MODEL, or set VESKSLOP_STT_CMD."
|
||||
};
|
||||
}
|
||||
|
||||
/** Returns undefined when no provider is configured. */
|
||||
async transcribe(wavPath: string): Promise<string | undefined> {
|
||||
if (this.localCommand) return this.viaLocalCommand(wavPath, this.localCommand);
|
||||
|
||||
if (this.voskModel) {
|
||||
const script = fileURLToPath(new URL("../scripts/stt-vosk.py", import.meta.url));
|
||||
return this.viaLocalCommand(wavPath, `python3 ${script} {wav}`, {
|
||||
VESKSLOP_VOSK_MODEL: this.voskModel
|
||||
});
|
||||
}
|
||||
|
||||
if (this.apiUrl) return this.viaApi(wavPath);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private findVoskModel(): string | undefined {
|
||||
const explicit = process.env.VESKSLOP_VOSK_MODEL;
|
||||
if (explicit) return explicit;
|
||||
|
||||
const base = process.env.VESKSLOP_VOSK_DIR ?? fileURLToPath(new URL("../models", import.meta.url));
|
||||
|
||||
try {
|
||||
const model = readdirSync(base, { withFileTypes: true }).find(
|
||||
entry => entry.isDirectory() && entry.name.startsWith("vosk-model")
|
||||
);
|
||||
return model ? join(base, model.name) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private async viaLocalCommand(
|
||||
wavPath: string,
|
||||
command: string,
|
||||
extraEnv: Record<string, string> = {}
|
||||
): Promise<string> {
|
||||
if (!command.includes("{wav}")) throw new SttError("the STT command must contain {wav}");
|
||||
|
||||
const parts = command
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map(part => part.replace("{wav}", wavPath));
|
||||
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const child = spawn(parts[0], parts.slice(1), {
|
||||
env: { ...process.env, ...extraEnv },
|
||||
stdio: ["ignore", "pipe", "pipe"]
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout?.on("data", chunk => (stdout += chunk));
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error => reject(new SttError(String(error))));
|
||||
child.on("close", code => {
|
||||
if (code === 0) resolve(stdout.trim());
|
||||
else reject(new SttError(stderr.trim() || `the STT command exited with ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async viaApi(wavPath: string): Promise<string> {
|
||||
const response = await fetch(this.apiUrl!, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "audio/wav",
|
||||
...(process.env.VESKSLOP_STT_API_KEY
|
||||
? { Authorization: `Bearer ${process.env.VESKSLOP_STT_API_KEY}` }
|
||||
: {})
|
||||
},
|
||||
body: readFileSync(wavPath)
|
||||
});
|
||||
|
||||
if (!response.ok) throw new SttError(`the STT API returned ${response.status}`);
|
||||
|
||||
const type = response.headers.get("content-type") ?? "";
|
||||
if (type.includes("json")) {
|
||||
const data: any = await response.json();
|
||||
return String(data.text ?? data.transcription ?? "").trim();
|
||||
}
|
||||
|
||||
return (await response.text()).trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Veskslop — unit tests for the capture pipeline logic.
|
||||
*
|
||||
* Usage: pnpm test:audio
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { Stt } from "./stt.js";
|
||||
import { TranscriptStore } from "./transcript.js";
|
||||
import { DEFAULT_VAD, pcmToWav, rms, Vad, type VadConfig } from "./vad.js";
|
||||
|
||||
// rms
|
||||
{
|
||||
const silence = Buffer.alloc(640);
|
||||
const loud = Buffer.alloc(640);
|
||||
for (let index = 0; index < 320; index++) {
|
||||
loud.writeInt16LE(Math.round(Math.sin(index / 5) * 0.4 * 32767), index * 2);
|
||||
}
|
||||
|
||||
assert.equal(rms(silence), 0);
|
||||
assert.ok(rms(loud) > 0.2, `rms ${rms(loud)}`);
|
||||
}
|
||||
|
||||
// VAD segments speech, keeps the pre-roll, and buffers partial frames
|
||||
{
|
||||
const config: VadConfig = { ...DEFAULT_VAD, minSpeechMs: 40, silenceMs: 100, preRollMs: 40, maxSegmentMs: 5000 };
|
||||
const frame = (loud: boolean) => {
|
||||
const buffer = Buffer.alloc(((config.sampleRate * config.frameMs) / 1000) * 2);
|
||||
if (loud) {
|
||||
for (let index = 0; index < buffer.length / 2; index++) {
|
||||
buffer.writeInt16LE(Math.round(Math.sin(index / 4) * 0.5 * 32767), index * 2);
|
||||
}
|
||||
}
|
||||
return buffer;
|
||||
};
|
||||
|
||||
const durations: number[] = [];
|
||||
const vad = new Vad(config, segment => durations.push(segment.durationMs));
|
||||
|
||||
for (let index = 0; index < 25; index++) vad.push(frame(false));
|
||||
for (let index = 0; index < 25; index++) vad.push(frame(true));
|
||||
for (let index = 0; index < 10; index++) vad.push(frame(false));
|
||||
|
||||
assert.equal(durations.length, 1);
|
||||
assert.ok(durations[0] >= 500 && durations[0] <= 700, `duration ${durations[0]}`);
|
||||
|
||||
const splitDurations: number[] = [];
|
||||
const split = new Vad(config, segment => splitDurations.push(segment.durationMs));
|
||||
const pcm = Buffer.concat([
|
||||
...Array(25).fill(frame(false)),
|
||||
...Array(25).fill(frame(true)),
|
||||
...Array(10).fill(frame(false))
|
||||
]);
|
||||
for (let offset = 0; offset < pcm.length; offset += 333) {
|
||||
split.push(pcm.subarray(offset, offset + 333));
|
||||
}
|
||||
assert.equal(splitDurations.length, 1);
|
||||
}
|
||||
|
||||
// WAV header
|
||||
{
|
||||
const wav = pcmToWav(Buffer.alloc(320), 16000);
|
||||
assert.equal(wav.length, 44 + 320);
|
||||
assert.equal(wav.toString("ascii", 0, 4), "RIFF");
|
||||
assert.equal(wav.toString("ascii", 8, 12), "WAVE");
|
||||
assert.equal(wav.readUInt32LE(24), 16000);
|
||||
}
|
||||
|
||||
// Transcript store: segments gain text later
|
||||
{
|
||||
const dir = mkdtempSync(join(tmpdir(), "veskslop-test-"));
|
||||
const store = new TranscriptStore(join(dir, "transcript.jsonl"));
|
||||
const session = store.startSession({ guildId: "g1", channelId: "c1" });
|
||||
|
||||
store.addSegment({
|
||||
id: "s1",
|
||||
sessionId: session.id,
|
||||
startedAt: new Date().toISOString(),
|
||||
endedAt: new Date().toISOString(),
|
||||
durationMs: 1200,
|
||||
speakerId: "u1",
|
||||
speakerName: "Zibbe",
|
||||
wav: "/tmp/x.wav",
|
||||
text: null
|
||||
});
|
||||
store.setSegmentText("s1", "hello world");
|
||||
store.endSession(session.id);
|
||||
|
||||
const sessions = store.readSessions();
|
||||
assert.equal(sessions.length, 1);
|
||||
assert.ok(sessions[0].endedAt);
|
||||
|
||||
const segments = store.readSegments();
|
||||
assert.equal(segments.length, 1);
|
||||
assert.equal(segments[0].text, "hello world");
|
||||
assert.equal(segments[0].speakerName, "Zibbe");
|
||||
assert.deepEqual(store.stats(), { sessions: 1, segments: 1, transcribed: 1 });
|
||||
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// STT local command
|
||||
{
|
||||
const dir = mkdtempSync(join(tmpdir(), "veskslop-stt-"));
|
||||
const wav = join(dir, "a.wav");
|
||||
writeFileSync(wav, pcmToWav(Buffer.alloc(320), 16000));
|
||||
|
||||
const script = join(dir, "fake-stt.js");
|
||||
writeFileSync(script, 'process.stdout.write("hello from stt\\n");');
|
||||
|
||||
process.env.VESKSLOP_STT_CMD = `node ${script} {wav}`;
|
||||
const stt = new Stt();
|
||||
assert.equal(await stt.transcribe(wav), "hello from stt");
|
||||
delete process.env.VESKSLOP_STT_CMD;
|
||||
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log("all audio tests passed");
|
||||
@@ -40,7 +40,15 @@ const reactionEvent = {
|
||||
emoji: { name: process.env.TEST_REACTION_EMOJI ?? "1️⃣", id: null }
|
||||
};
|
||||
|
||||
const event = isReaction ? reactionEvent : messageEvent;
|
||||
const speakingUser = process.env.TEST_TRIGGER_SPEAKING;
|
||||
const speakingEvent = {
|
||||
type: "VOICE_SPEAKING",
|
||||
userId: speakingUser ?? "999",
|
||||
speaking: process.env.TEST_TRIGGER_SPEAKING_OFF !== "1",
|
||||
channelId: process.env.TEST_TRIGGER_CHANNEL ?? "200000000000000002"
|
||||
};
|
||||
|
||||
const event = isReaction ? reactionEvent : speakingUser ? speakingEvent : messageEvent;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = net.connect(coreSocketPath);
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Veskslop daemon — self-signed TLS for the web UI.
|
||||
*
|
||||
* Browser media APIs need a secure context. The daemon generates one
|
||||
* self-signed certificate (openssl) covering localhost and the host's LAN
|
||||
* addresses, and serves the UI over HTTPS. WEBUI_TLS=0 disables this.
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { networkInterfaces } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
export interface TlsPaths {
|
||||
key: string;
|
||||
cert: string;
|
||||
}
|
||||
|
||||
function localAddresses(): string[] {
|
||||
const names = ["DNS:localhost", "IP:127.0.0.1"];
|
||||
|
||||
for (const list of Object.values(networkInterfaces())) {
|
||||
for (const info of list ?? []) {
|
||||
if (info.family === "IPv4" && !info.internal) names.push(`IP:${info.address}`);
|
||||
}
|
||||
}
|
||||
|
||||
return names;
|
||||
}
|
||||
|
||||
export function ensureTls(storeDir: string): TlsPaths | undefined {
|
||||
if (process.env.WEBUI_TLS === "0") return undefined;
|
||||
|
||||
const keyOverride = process.env.WEBUI_TLS_KEY;
|
||||
const certOverride = process.env.WEBUI_TLS_CERT;
|
||||
if (keyOverride && certOverride) {
|
||||
return existsSync(keyOverride) && existsSync(certOverride)
|
||||
? { key: keyOverride, cert: certOverride }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
const dir = join(storeDir, "tls");
|
||||
const key = join(dir, "key.pem");
|
||||
const cert = join(dir, "cert.pem");
|
||||
if (existsSync(key) && existsSync(cert)) return { key, cert };
|
||||
|
||||
try {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
execFileSync(
|
||||
"openssl",
|
||||
[
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
key,
|
||||
"-out",
|
||||
cert,
|
||||
"-days",
|
||||
"825",
|
||||
"-subj",
|
||||
"/CN=veskslop",
|
||||
"-addext",
|
||||
`subjectAltName=${localAddresses().join(",")}`
|
||||
],
|
||||
{ stdio: "ignore" }
|
||||
);
|
||||
return { key, cert };
|
||||
} catch (error) {
|
||||
console.warn("[webui] self-signed certificate generation failed:", (error as Error).message);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* Veskslop daemon — transcript store.
|
||||
*
|
||||
* Sessions and segments append to a JSONL file. Text can arrive after a
|
||||
* segment, so a text update appends a patch line instead of rewriting the file.
|
||||
*/
|
||||
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const MAX_BYTES = Number(process.env.VESKSLOP_TRANSCRIPT_MAX_BYTES ?? 20 * 1024 * 1024);
|
||||
|
||||
export interface TranscriptSession {
|
||||
id: string;
|
||||
startedAt: string;
|
||||
endedAt?: string;
|
||||
guildId?: string | null;
|
||||
channelId?: string | null;
|
||||
}
|
||||
|
||||
export interface TranscriptSegment {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
startedAt: string;
|
||||
endedAt: string;
|
||||
durationMs: number;
|
||||
speakerId?: string | null;
|
||||
speakerName?: string | null;
|
||||
wav?: string | null;
|
||||
text?: string | null;
|
||||
}
|
||||
|
||||
export class TranscriptStore {
|
||||
private counter = 0;
|
||||
|
||||
constructor(private readonly file = process.env.VESKSLOP_TRANSCRIPT ?? ".veskslop-transcripts.jsonl") {
|
||||
try {
|
||||
mkdirSync(dirname(this.file), { recursive: true });
|
||||
} catch {
|
||||
// the current directory works
|
||||
}
|
||||
}
|
||||
|
||||
get path(): string {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
startSession(meta: { guildId?: string | null; channelId?: string | null } = {}): TranscriptSession {
|
||||
const session: TranscriptSession = {
|
||||
id: this.nextId("session"),
|
||||
startedAt: new Date().toISOString(),
|
||||
guildId: meta.guildId ?? null,
|
||||
channelId: meta.channelId ?? null
|
||||
};
|
||||
this.append({ type: "session", session });
|
||||
return session;
|
||||
}
|
||||
|
||||
endSession(id: string) {
|
||||
this.append({ type: "session-end", id, endedAt: new Date().toISOString() });
|
||||
}
|
||||
|
||||
addSegment(segment: TranscriptSegment) {
|
||||
this.append({ type: "segment", segment });
|
||||
}
|
||||
|
||||
setSegmentText(id: string, text: string) {
|
||||
this.append({ type: "segment-text", id, text });
|
||||
}
|
||||
|
||||
readSessions(limit = 20): TranscriptSession[] {
|
||||
const sessions = new Map<string, TranscriptSession>();
|
||||
|
||||
for (const line of this.lines()) {
|
||||
if (line.type === "session" && line.session) sessions.set(line.session.id, line.session);
|
||||
if (line.type === "session-end") {
|
||||
const session = sessions.get(line.id);
|
||||
if (session) session.endedAt = line.endedAt;
|
||||
}
|
||||
}
|
||||
|
||||
return [...sessions.values()].slice(-limit);
|
||||
}
|
||||
|
||||
readSegments(limit = 100, sessionId?: string): TranscriptSegment[] {
|
||||
const segments = new Map<string, TranscriptSegment>();
|
||||
|
||||
for (const line of this.lines()) {
|
||||
if (line.type === "segment" && line.segment) {
|
||||
if (!sessionId || line.segment.sessionId === sessionId) segments.set(line.segment.id, line.segment);
|
||||
}
|
||||
if (line.type === "segment-text") {
|
||||
const segment = segments.get(line.id);
|
||||
if (segment) segment.text = line.text;
|
||||
}
|
||||
}
|
||||
|
||||
return [...segments.values()].slice(-limit);
|
||||
}
|
||||
|
||||
stats(): { sessions: number; segments: number; transcribed: number } {
|
||||
const sessions = this.readSessions(1000);
|
||||
const segments = this.readSegments(10000);
|
||||
return {
|
||||
sessions: sessions.length,
|
||||
segments: segments.length,
|
||||
transcribed: segments.filter(segment => !!segment.text).length
|
||||
};
|
||||
}
|
||||
|
||||
private lines(): any[] {
|
||||
try {
|
||||
if (!existsSync(this.file)) return [];
|
||||
return readFileSync(this.file, "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(line => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private append(entry: Record<string, unknown> & { type: string }) {
|
||||
const line = JSON.stringify({ timestamp: new Date().toISOString(), ...entry }) + "\n";
|
||||
|
||||
try {
|
||||
if (existsSync(this.file) && statSync(this.file).size + line.length > MAX_BYTES) {
|
||||
renameSync(this.file, `${this.file}.1`);
|
||||
}
|
||||
appendFileSync(this.file, line);
|
||||
} catch (error) {
|
||||
console.error("[transcript] write failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
private nextId(prefix: string): string {
|
||||
this.counter++;
|
||||
return `${prefix}-${Date.now()}-${this.counter}`;
|
||||
}
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Veskslop daemon — text to speech.
|
||||
*
|
||||
* Providers: a local engine (espeak-ng when installed, or VESKSLOP_TTS_CMD)
|
||||
* and an HTTP API. The local command receives an output path ({out}), a text
|
||||
* file ({text_file}), and optionally a voice ({voice}).
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
export class TtsError extends Error {}
|
||||
|
||||
export type TtsProvider = "local" | "api";
|
||||
|
||||
export interface TtsOptions {
|
||||
provider?: TtsProvider;
|
||||
voice?: string;
|
||||
}
|
||||
|
||||
export interface TtsStatus {
|
||||
local: boolean;
|
||||
api: boolean;
|
||||
defaultProvider: TtsProvider;
|
||||
engine: string | null;
|
||||
}
|
||||
|
||||
export interface TtsVoice {
|
||||
id: string;
|
||||
name: string;
|
||||
language: string;
|
||||
gender: "male" | "female" | null;
|
||||
}
|
||||
|
||||
interface CommandResult {
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
function run(command: string, args: string[]): Promise<CommandResult> {
|
||||
return new Promise(resolve => {
|
||||
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout?.on("data", chunk => (stdout += chunk));
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error => resolve({ code: null, stdout, stderr: String(error) }));
|
||||
child.on("close", code => resolve({ code, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
function parseVoices(output: string): TtsVoice[] {
|
||||
const lines = output.split("\n");
|
||||
const header = lines[0] ?? "";
|
||||
const languageIndex = header.indexOf("Language");
|
||||
const ageIndex = header.indexOf("Age/Gender");
|
||||
const nameIndex = header.indexOf("VoiceName");
|
||||
const fileIndex = header.indexOf("File");
|
||||
|
||||
if (languageIndex < 0 || ageIndex < 0 || nameIndex < 0 || fileIndex < 0) return [];
|
||||
|
||||
const voices: TtsVoice[] = [];
|
||||
for (const line of lines.slice(1)) {
|
||||
const language = line.slice(languageIndex, ageIndex).trim();
|
||||
const ageGender = line.slice(ageIndex, nameIndex).trim();
|
||||
const name = line.slice(nameIndex, fileIndex).trim();
|
||||
if (!language || !name) continue;
|
||||
|
||||
voices.push({
|
||||
id: language,
|
||||
name,
|
||||
language,
|
||||
gender: ageGender.endsWith("F") ? "female" : ageGender.endsWith("M") ? "male" : null
|
||||
});
|
||||
}
|
||||
|
||||
return voices;
|
||||
}
|
||||
|
||||
export class Tts {
|
||||
private readonly dir: string;
|
||||
private counter = 0;
|
||||
private espeak = false;
|
||||
private voiceCache?: TtsVoice[];
|
||||
|
||||
constructor(cacheDir?: string) {
|
||||
const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache");
|
||||
this.dir = join(base, "audio");
|
||||
try {
|
||||
mkdirSync(this.dir, { recursive: true });
|
||||
} catch {
|
||||
// the fallback directory works
|
||||
}
|
||||
}
|
||||
|
||||
get localCommand(): string | undefined {
|
||||
return process.env.VESKSLOP_TTS_CMD;
|
||||
}
|
||||
|
||||
get apiUrl(): string | undefined {
|
||||
return process.env.VESKSLOP_TTS_API_URL;
|
||||
}
|
||||
|
||||
get directory(): string {
|
||||
return this.dir;
|
||||
}
|
||||
|
||||
async detect(): Promise<void> {
|
||||
const result = await run("espeak-ng", ["--version"]);
|
||||
this.espeak = result.code === 0;
|
||||
}
|
||||
|
||||
status(): TtsStatus {
|
||||
const local = !!this.localCommand || this.espeak;
|
||||
const override = process.env.VESKSLOP_TTS_PROVIDER;
|
||||
|
||||
return {
|
||||
local,
|
||||
api: !!this.apiUrl,
|
||||
defaultProvider: override === "local" || override === "api" ? override : local ? "local" : "api",
|
||||
engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null
|
||||
};
|
||||
}
|
||||
|
||||
async voices(): Promise<TtsVoice[]> {
|
||||
if (this.voiceCache) return this.voiceCache;
|
||||
if (!this.espeak) return (this.voiceCache = []);
|
||||
|
||||
const result = await run("espeak-ng", ["--voices"]);
|
||||
this.voiceCache = result.code === 0 ? parseVoices(result.stdout) : [];
|
||||
return this.voiceCache;
|
||||
}
|
||||
|
||||
async synthesize(text: string, options: TtsOptions = {}): Promise<string> {
|
||||
const provider = options.provider ?? this.status().defaultProvider;
|
||||
return provider === "api" ? this.viaApi(text, options) : this.viaLocalCommand(text, options);
|
||||
}
|
||||
|
||||
private async viaLocalCommand(text: string, options: TtsOptions): Promise<string> {
|
||||
const command = this.localCommand;
|
||||
if (!command && !this.espeak) {
|
||||
throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)");
|
||||
}
|
||||
if (command && !command.includes("{out}")) {
|
||||
throw new TtsError("VESKSLOP_TTS_CMD must contain {out}");
|
||||
}
|
||||
|
||||
const out = this.nextPath("wav");
|
||||
const textFile = this.nextPath("txt");
|
||||
writeFileSync(textFile, text);
|
||||
|
||||
const parts = command
|
||||
? command
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map(part =>
|
||||
part
|
||||
.replace("{out}", out)
|
||||
.replace("{text_file}", textFile)
|
||||
.replace("{voice}", options.voice ?? "")
|
||||
)
|
||||
: ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile];
|
||||
|
||||
const args = parts.filter(Boolean);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(args[0], args.slice(1), {
|
||||
env: { ...process.env, VESKSLOP_TTS_VOICE: options.voice ?? "" },
|
||||
stdio: ["ignore", "ignore", "pipe"]
|
||||
});
|
||||
let stderr = "";
|
||||
|
||||
child.stderr?.on("data", chunk => (stderr += chunk));
|
||||
child.on("error", error => reject(new TtsError(String(error))));
|
||||
child.on("close", code => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new TtsError(stderr.trim() || `the TTS command exited with ${code}`));
|
||||
});
|
||||
});
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private async viaApi(text: string, options: TtsOptions): Promise<string> {
|
||||
const url = this.apiUrl;
|
||||
if (!url) throw new TtsError("VESKSLOP_TTS_API_URL is not set");
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(process.env.VESKSLOP_TTS_API_KEY
|
||||
? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` }
|
||||
: {})
|
||||
},
|
||||
body: JSON.stringify({ text, ...(options.voice ? { voice: options.voice } : {}) })
|
||||
});
|
||||
|
||||
if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`);
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const type = response.headers.get("content-type") ?? "";
|
||||
const extension =
|
||||
type.includes("mpeg") || type.includes("mp3")
|
||||
? "mp3"
|
||||
: type.includes("ogg")
|
||||
? "ogg"
|
||||
: type.includes("wav")
|
||||
? "wav"
|
||||
: "mp3";
|
||||
|
||||
const out = this.nextPath(extension);
|
||||
writeFileSync(out, buffer);
|
||||
return out;
|
||||
}
|
||||
|
||||
private nextPath(extension: string): string {
|
||||
this.counter++;
|
||||
return join(this.dir, `tts-${Date.now()}-${this.counter}.${extension}`);
|
||||
}
|
||||
}
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
/*
|
||||
* Veskslop daemon — voice activity detection and WAV writing (pure).
|
||||
*
|
||||
* The capture pipeline feeds 16-bit mono PCM frames into the detector. It emits
|
||||
* one segment per utterance: speech plus a small pre-roll and a trailing gap.
|
||||
*/
|
||||
|
||||
export interface VadConfig {
|
||||
sampleRate: number;
|
||||
frameMs: number;
|
||||
/** Below this level a frame counts as silence. */
|
||||
thresholdDb: number;
|
||||
/** Speech must last this long before a segment opens. */
|
||||
minSpeechMs: number;
|
||||
/** Silence for this long closes an open segment. */
|
||||
silenceMs: number;
|
||||
/** Audio kept from before the speech started. */
|
||||
preRollMs: number;
|
||||
/** A segment is force-closed at this length. */
|
||||
maxSegmentMs: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_VAD: VadConfig = {
|
||||
sampleRate: 16000,
|
||||
frameMs: 20,
|
||||
thresholdDb: -35,
|
||||
minSpeechMs: 120,
|
||||
silenceMs: 600,
|
||||
preRollMs: 300,
|
||||
maxSegmentMs: 30000
|
||||
};
|
||||
|
||||
export interface VadSegment {
|
||||
pcm: Buffer;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export function rms(frame: Buffer): number {
|
||||
if (frame.length < 2) return 0;
|
||||
|
||||
let sum = 0;
|
||||
const samples = Math.floor(frame.length / 2);
|
||||
for (let index = 0; index < samples; index++) {
|
||||
const sample = frame.readInt16LE(index * 2) / 32768;
|
||||
sum += sample * sample;
|
||||
}
|
||||
|
||||
return Math.sqrt(sum / samples);
|
||||
}
|
||||
|
||||
export function pcmToWav(pcm: Buffer, sampleRate: number): Buffer {
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF", 0);
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVE", 8);
|
||||
header.write("fmt ", 12);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(sampleRate, 24);
|
||||
header.writeUInt32LE(sampleRate * 2, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
|
||||
export class Vad {
|
||||
private readonly frameBytes: number;
|
||||
private readonly threshold: number;
|
||||
private readonly minSpeechFrames: number;
|
||||
private readonly silenceFrameLimit: number;
|
||||
private readonly preRollFrameLimit: number;
|
||||
private readonly maxSegmentFrames: number;
|
||||
|
||||
private pending: Buffer = Buffer.alloc(0);
|
||||
private preRoll: Buffer[] = [];
|
||||
private segment: Buffer[] = [];
|
||||
private speechFrames = 0;
|
||||
private silenceFrames = 0;
|
||||
private open = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: VadConfig,
|
||||
private readonly onSegment: (segment: VadSegment) => void
|
||||
) {
|
||||
this.frameBytes = Math.round((config.sampleRate * config.frameMs) / 1000) * 2;
|
||||
this.threshold = 10 ** (config.thresholdDb / 20);
|
||||
this.minSpeechFrames = Math.max(1, Math.ceil(config.minSpeechMs / config.frameMs));
|
||||
this.silenceFrameLimit = Math.max(1, Math.ceil(config.silenceMs / config.frameMs));
|
||||
this.preRollFrameLimit = Math.max(0, Math.ceil(config.preRollMs / config.frameMs));
|
||||
this.maxSegmentFrames = Math.max(1, Math.ceil(config.maxSegmentMs / config.frameMs));
|
||||
}
|
||||
|
||||
get inSpeech(): boolean {
|
||||
return this.open;
|
||||
}
|
||||
|
||||
push(chunk: Buffer) {
|
||||
this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk;
|
||||
|
||||
while (this.pending.length >= this.frameBytes) {
|
||||
const frame = this.pending.subarray(0, this.frameBytes);
|
||||
this.pending = this.pending.subarray(this.frameBytes);
|
||||
this.consumeFrame(frame);
|
||||
}
|
||||
}
|
||||
|
||||
/** Close an open segment. Call when the capture stops. */
|
||||
flush() {
|
||||
if (!this.open) return;
|
||||
this.closeSegment();
|
||||
}
|
||||
|
||||
private consumeFrame(frame: Buffer) {
|
||||
const loud = rms(frame) >= this.threshold;
|
||||
|
||||
if (!this.open) {
|
||||
if (this.preRollFrameLimit > 0) {
|
||||
this.preRoll.push(frame);
|
||||
if (this.preRoll.length > this.preRollFrameLimit) this.preRoll.shift();
|
||||
}
|
||||
|
||||
if (loud) {
|
||||
this.speechFrames++;
|
||||
if (this.speechFrames >= this.minSpeechFrames) {
|
||||
this.open = true;
|
||||
this.segment = [...this.preRoll];
|
||||
this.preRoll = [];
|
||||
this.silenceFrames = 0;
|
||||
}
|
||||
} else {
|
||||
this.speechFrames = 0;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.segment.push(frame);
|
||||
|
||||
if (loud) {
|
||||
this.silenceFrames = 0;
|
||||
} else {
|
||||
this.silenceFrames++;
|
||||
}
|
||||
|
||||
if (this.silenceFrames >= this.silenceFrameLimit || this.segment.length >= this.maxSegmentFrames) {
|
||||
this.closeSegment();
|
||||
}
|
||||
}
|
||||
|
||||
private closeSegment() {
|
||||
const frames = this.silenceFrames > 0 ? this.segment.slice(0, -this.silenceFrames) : this.segment;
|
||||
this.open = false;
|
||||
this.speechFrames = 0;
|
||||
this.silenceFrames = 0;
|
||||
this.segment = [];
|
||||
|
||||
if (!frames.length) return;
|
||||
|
||||
const pcm = Buffer.concat(frames);
|
||||
this.onSegment({ pcm, durationMs: (frames.length * this.config.frameMs) });
|
||||
}
|
||||
}
|
||||
+342
-8
@@ -8,14 +8,57 @@
|
||||
*/
|
||||
|
||||
import express, { type Request, type Response } from "express";
|
||||
import { readFileSync } from "node:fs";
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import QRCode from "qrcode";
|
||||
import { WebSocketServer } from "ws";
|
||||
|
||||
import type { AuthService, AuthStatus } from "./auth-service.js";
|
||||
import type { AudioService } from "./audio-service.js";
|
||||
import { CoreError, type CoreClient } from "./core-client.js";
|
||||
import type { CoreProcess } from "./core-process.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend } from "./pacing.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js";
|
||||
import type { PreferencesStore, WebPreferences } from "./preferences.js";
|
||||
import type { Puppet } from "./puppet.js";
|
||||
import { ensureTls } from "./tls.js";
|
||||
import type { TranscriptStore } from "./transcript.js";
|
||||
import type { TtsVoice } from "./tts.js";
|
||||
|
||||
const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"];
|
||||
const LANGUAGE_NAMES: Record<string, string> = {
|
||||
en: "English",
|
||||
sv: "Swedish",
|
||||
de: "German",
|
||||
da: "Danish",
|
||||
nb: "Norwegian",
|
||||
no: "Norwegian",
|
||||
ru: "Russian",
|
||||
chr: "Cherokee"
|
||||
};
|
||||
|
||||
function groupVoices(voices: TtsVoice[]) {
|
||||
const groups = new Map<string, { id: string; name: string; voices: TtsVoice[] }>();
|
||||
|
||||
for (const voice of voices) {
|
||||
const base = voice.language.split("-")[0];
|
||||
const group = groups.get(base) ?? { id: base, name: LANGUAGE_NAMES[base] ?? voice.language, voices: [] };
|
||||
group.voices.push(voice);
|
||||
groups.set(base, group);
|
||||
}
|
||||
|
||||
const all = [...groups.values()];
|
||||
const pinned = PINNED_LANGUAGES.map(id => all.find(group => group.id === id)).filter(
|
||||
(group): group is { id: string; name: string; voices: TtsVoice[] } => !!group
|
||||
);
|
||||
const rest = all
|
||||
.filter(group => !PINNED_LANGUAGES.includes(group.id))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
return [...pinned, ...rest];
|
||||
}
|
||||
|
||||
const HOST = process.env.WEBUI_HOST ?? "0.0.0.0";
|
||||
const PORT = Number(process.env.WEBUI_PORT ?? 8789);
|
||||
@@ -26,6 +69,10 @@ export interface WebUiDeps {
|
||||
isCoreConnected: () => boolean;
|
||||
coreProcess: CoreProcess;
|
||||
auth: AuthService;
|
||||
audio: AudioService;
|
||||
transcripts: TranscriptStore;
|
||||
preferences: PreferencesStore;
|
||||
connectAudioDevices: () => Promise<{ connected: boolean; message: string }>;
|
||||
}
|
||||
|
||||
export interface WebUiHandle {
|
||||
@@ -54,9 +101,84 @@ async function qrDataUrl(status: AuthStatus): Promise<string | undefined> {
|
||||
return QRCode.toDataURL(session.url, { margin: 1, width: 280 });
|
||||
}
|
||||
|
||||
function stateDir(): string {
|
||||
const stateFile =
|
||||
process.env.VESKSLOP_STATE_FILE ?? fileURLToPath(new URL("../.veskslop-state.json", import.meta.url));
|
||||
return dirname(stateFile);
|
||||
}
|
||||
|
||||
/** Live browser audio: the call output streamed as raw PCM over a WebSocket. */
|
||||
function startAudioSocket(server: http.Server | https.Server, deps: WebUiDeps): void {
|
||||
const wss = new WebSocketServer({ server, path: "/audio" });
|
||||
|
||||
wss.on("connection", socket => {
|
||||
let stop: (() => void) | undefined;
|
||||
|
||||
const send = (message: unknown) => {
|
||||
if (socket.readyState === socket.OPEN) socket.send(JSON.stringify(message));
|
||||
};
|
||||
|
||||
socket.on("message", raw => {
|
||||
let message: any;
|
||||
try {
|
||||
message = JSON.parse(raw.toString());
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.type !== "listen") return;
|
||||
|
||||
if (!message.on) {
|
||||
stop?.();
|
||||
stop = undefined;
|
||||
send({ type: "status", listening: false });
|
||||
return;
|
||||
}
|
||||
|
||||
if (stop) return;
|
||||
|
||||
try {
|
||||
const status = deps.audio.status();
|
||||
if (!status.available) {
|
||||
throw new Error(status.hint ?? status.error ?? "audio is not available");
|
||||
}
|
||||
if (!status.ffmpeg) {
|
||||
throw new Error("ffmpeg is missing; install it to hear the call");
|
||||
}
|
||||
|
||||
stop = deps.audio.startMonitor(
|
||||
chunk => {
|
||||
if (socket.readyState === socket.OPEN) socket.send(chunk);
|
||||
},
|
||||
error => {
|
||||
send({ type: "error", error: error.message });
|
||||
stop?.();
|
||||
stop = undefined;
|
||||
}
|
||||
);
|
||||
send({ type: "status", listening: true });
|
||||
} catch (error) {
|
||||
send({ type: "error", error: (error as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
const cleanup = () => {
|
||||
stop?.();
|
||||
stop = undefined;
|
||||
};
|
||||
|
||||
socket.on("close", cleanup);
|
||||
socket.on("error", cleanup);
|
||||
});
|
||||
}
|
||||
|
||||
export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
const app = express();
|
||||
app.use(express.json({ limit: "64kb" }));
|
||||
app.use("/api", (_req, res, next) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
next();
|
||||
});
|
||||
app.use(express.static(fileURLToPath(new URL("../public", import.meta.url))));
|
||||
|
||||
const clients = new Set<Response>();
|
||||
@@ -269,6 +391,84 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/preferences",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: deps.preferences.get() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/preferences",
|
||||
wrap(async (req, res) => {
|
||||
const { visibleGuilds, tts, favourites } = req.body ?? {};
|
||||
|
||||
if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
|
||||
throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
|
||||
}
|
||||
|
||||
const patch: Partial<WebPreferences> = {};
|
||||
if (visibleGuilds !== undefined) patch.visibleGuilds = visibleGuilds;
|
||||
|
||||
if (tts !== undefined) {
|
||||
patch.tts = {
|
||||
provider: tts?.provider ?? null,
|
||||
language: tts?.language ?? null,
|
||||
voice: tts?.voice ?? null
|
||||
};
|
||||
}
|
||||
|
||||
if (favourites !== undefined) {
|
||||
patch.favourites = {
|
||||
users: Array.isArray(favourites?.users) ? favourites.users : [],
|
||||
channels: Array.isArray(favourites?.channels) ? favourites.channels : []
|
||||
};
|
||||
}
|
||||
|
||||
res.json({ ok: true, result: deps.preferences.update(patch) });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/favourites",
|
||||
wrap(async (_req, res) => {
|
||||
const prefs = deps.preferences.get();
|
||||
const userIds = prefs.favourites.users;
|
||||
|
||||
if (!userIds.length) {
|
||||
res.json({ ok: true, result: { users: [], channels: prefs.favourites.channels } });
|
||||
return;
|
||||
}
|
||||
|
||||
const [statuses, voices] = await Promise.all([
|
||||
requireCore().userStatus(userIds) as Promise<{ users?: Array<{ id: string; status: string }> }>,
|
||||
requireCore().voiceUsers(userIds) as Promise<{ users?: any[] }>
|
||||
]);
|
||||
|
||||
const statusMap = new Map((statuses.users ?? []).map(user => [user.id, user.status]));
|
||||
const voiceMap = new Map((voices.users ?? []).map(user => [user.id, user]));
|
||||
|
||||
res.json({
|
||||
ok: true,
|
||||
result: {
|
||||
users: userIds.map(id => {
|
||||
const voice = voiceMap.get(id);
|
||||
return {
|
||||
id,
|
||||
username: voice?.username ?? id,
|
||||
avatar: voice?.avatar ?? null,
|
||||
status: statusMap.get(id) ?? "offline",
|
||||
channelId: voice?.channelId ?? null,
|
||||
channelName: voice?.channelName ?? null,
|
||||
guildId: voice?.guildId ?? null
|
||||
};
|
||||
}),
|
||||
channels: prefs.favourites.channels
|
||||
}
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/voice",
|
||||
wrap(async (req, res) => {
|
||||
@@ -286,13 +486,137 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.post("/api/voice/join", (_req, res) => {
|
||||
res.status(501).json({ ok: false, error: "voice.join is not implemented yet" });
|
||||
});
|
||||
app.post(
|
||||
"/api/voice/join",
|
||||
wrap(async (req, res) => {
|
||||
const channelId = String(req.body?.channelId ?? "");
|
||||
const mute = req.body?.mute !== false;
|
||||
|
||||
app.post("/api/voice/leave", (_req, res) => {
|
||||
res.status(501).json({ ok: false, error: "voice.leave is not implemented yet" });
|
||||
const result = await pacedVoice(() => requireCore().voiceJoin(channelId, mute));
|
||||
deps.getPuppet()?.appendAction({ type: "voice", action: "join", channelId, mute });
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/voice/leave",
|
||||
wrap(async (_req, res) => {
|
||||
const result = await pacedVoice(() => requireCore().voiceLeave());
|
||||
deps.getPuppet()?.appendAction({ type: "voice", action: "leave" });
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/voice/move",
|
||||
wrap(async (req, res) => {
|
||||
const channelId = String(req.body?.channelId ?? "");
|
||||
const result = await pacedVoice(() => requireCore().voiceMove(channelId));
|
||||
deps.getPuppet()?.appendAction({ type: "voice", action: "move", channelId });
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/audio",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: deps.audio.status() });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/audio/tts/voices",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/audio/devices",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: await requireCore().voiceDevices() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/connect",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: await deps.connectAudioDevices() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/devices",
|
||||
wrap(async (req, res) => {
|
||||
const inputDeviceId = req.body?.inputDeviceId ? String(req.body.inputDeviceId) : undefined;
|
||||
const outputDeviceId = req.body?.outputDeviceId ? String(req.body.outputDeviceId) : undefined;
|
||||
|
||||
const result = await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId);
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "audio",
|
||||
action: "devices",
|
||||
inputDeviceId: inputDeviceId ?? null,
|
||||
outputDeviceId: outputDeviceId ?? null
|
||||
});
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/speak",
|
||||
wrap(async (req, res) => {
|
||||
const text = String(req.body?.text ?? "");
|
||||
const prefs = deps.preferences.get();
|
||||
|
||||
const provider =
|
||||
req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider;
|
||||
const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined;
|
||||
|
||||
const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice });
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "audio",
|
||||
action: "speak",
|
||||
provider: provider ?? deps.audio.status().tts.defaultProvider,
|
||||
chars: text.trim().length
|
||||
});
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/stop",
|
||||
wrap(async (_req, res) => {
|
||||
deps.audio.stop();
|
||||
deps.getPuppet()?.appendAction({ type: "audio", action: "stop" });
|
||||
res.json({ ok: true, result: deps.audio.status() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/audio/capture",
|
||||
wrap(async (req, res) => {
|
||||
const start = req.body?.start !== false;
|
||||
const result = start ? await deps.audio.startCapture() : (deps.audio.stopCapture(), { stopped: true });
|
||||
|
||||
deps.getPuppet()?.appendAction({ type: "audio", action: start ? "capture-start" : "capture-stop" });
|
||||
res.json({ ok: true, result });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/transcript",
|
||||
wrap(async (req, res) => {
|
||||
const limit = clampLimit(req.query.limit, 100, 500);
|
||||
const sessionId = req.query.sessionId ? String(req.query.sessionId) : undefined;
|
||||
res.json({
|
||||
ok: true,
|
||||
result: {
|
||||
sessions: deps.transcripts.readSessions(20),
|
||||
segments: deps.transcripts.readSegments(limit, sessionId)
|
||||
}
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
app.get("/api/logs", (req, res) => {
|
||||
const puppet = requirePuppet();
|
||||
@@ -325,8 +649,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
});
|
||||
});
|
||||
|
||||
const server = app.listen(PORT, HOST, () => {
|
||||
console.log(`[webui] listening on http://${HOST}:${PORT} (no auth; LAN or Tailscale only)`);
|
||||
const tls = ensureTls(stateDir());
|
||||
const server = tls
|
||||
? https.createServer({ key: readFileSync(tls.key), cert: readFileSync(tls.cert) }, app)
|
||||
: http.createServer(app);
|
||||
|
||||
startAudioSocket(server, deps);
|
||||
|
||||
server.listen(PORT, HOST, () => {
|
||||
console.log(`[webui] listening on ${tls ? "https" : "http"}://${HOST}:${PORT} (no auth; LAN or Tailscale only)`);
|
||||
if (!tls && process.env.WEBUI_TLS !== "0") {
|
||||
console.warn("[webui] TLS is unavailable; browser media APIs need HTTPS or localhost");
|
||||
}
|
||||
});
|
||||
|
||||
server.on("error", error => {
|
||||
|
||||
@@ -42,6 +42,13 @@ const KNOWN_ACTIONS = new Set<string>([
|
||||
PuppetAction.StateGet,
|
||||
PuppetAction.VoiceGet,
|
||||
PuppetAction.VoiceOverview,
|
||||
PuppetAction.VoiceJoin,
|
||||
PuppetAction.VoiceLeave,
|
||||
PuppetAction.VoiceMove,
|
||||
PuppetAction.VoiceUsers,
|
||||
PuppetAction.UserStatus,
|
||||
PuppetAction.VoiceDevices,
|
||||
PuppetAction.VoiceSetDevices,
|
||||
PuppetAction.UserSearch,
|
||||
PuppetAction.DmList,
|
||||
PuppetAction.MessageSend,
|
||||
|
||||
+258
-14
@@ -20,6 +20,8 @@ import {
|
||||
GuildMemberStore,
|
||||
GuildStore,
|
||||
MessageStore,
|
||||
PresenceStore,
|
||||
ReadStateStore,
|
||||
UserStore,
|
||||
VoiceStateStore
|
||||
} from "@vencord/types/webpack/common";
|
||||
@@ -28,12 +30,21 @@ import { isSnowflake, PUPPET_LIMITS, PuppetAction, PuppetErrorCode, type PuppetR
|
||||
/** Channel types that can receive text messages. */
|
||||
const SENDABLE_CHANNEL_TYPES = new Set([0, 1, 3, 5, 10, 11, 12]);
|
||||
|
||||
/** Channel types that are voice channels. */
|
||||
const VOICE_CHANNEL_TYPES = new Set([2, 13]);
|
||||
|
||||
/** Discord's message fetch action, located lazily at runtime. */
|
||||
const MessageActions = findByPropsLazy("fetchMessages");
|
||||
|
||||
/** Discord's reaction actions, located lazily at runtime. */
|
||||
const ReactionActions = findByPropsLazy("addReaction", "removeReaction");
|
||||
|
||||
/** Discord's channel actions, located lazily at runtime. */
|
||||
const ChannelActions = findByPropsLazy("selectVoiceChannel");
|
||||
|
||||
/** Discord's media engine actions, located lazily at runtime. */
|
||||
const MediaEngineActions = findByPropsLazy("setSelfMute");
|
||||
|
||||
function compareIds(a: string, b: string): number {
|
||||
try {
|
||||
const left = BigInt(a);
|
||||
@@ -157,20 +168,53 @@ function buildState() {
|
||||
};
|
||||
}
|
||||
|
||||
function voiceMember(state: any) {
|
||||
function voiceMember(state: any, guildId?: string) {
|
||||
const user = UserStore.getUser(state.userId);
|
||||
return {
|
||||
id: state.userId,
|
||||
username: user?.username ?? state.userId,
|
||||
globalName: user?.globalName ?? null,
|
||||
nick: guildId ? GuildMemberStore.getNick(guildId, state.userId) : null,
|
||||
avatar: user?.avatar ?? null,
|
||||
mute: !!state.mute,
|
||||
deaf: !!state.deaf,
|
||||
selfMute: !!state.selfMute,
|
||||
selfDeaf: !!state.selfDeaf
|
||||
selfDeaf: !!state.selfDeaf,
|
||||
video: !!state.selfVideo,
|
||||
stream: !!state.selfStream || !!state.stream,
|
||||
suppress: !!state.suppress
|
||||
};
|
||||
}
|
||||
|
||||
/** Discord snowflakes carry a millisecond timestamp. */
|
||||
function snowflakeToIso(id: string | null | undefined): string | null {
|
||||
if (!id) return null;
|
||||
try {
|
||||
return new Date(Number(BigInt(id) >> 22n) + 1420070400000).toISOString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function unreadCounts(channelId: string): { unread: number; mentions: number } {
|
||||
try {
|
||||
return {
|
||||
unread: ReadStateStore.getUnreadCount(channelId) ?? 0,
|
||||
mentions: ReadStateStore.getMentionCount(channelId) ?? 0
|
||||
};
|
||||
} catch {
|
||||
return { unread: 0, mentions: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
function presenceOf(userId: string): string {
|
||||
try {
|
||||
return PresenceStore.getStatus(userId) ?? "offline";
|
||||
} catch {
|
||||
return "offline";
|
||||
}
|
||||
}
|
||||
|
||||
function toUserResult(user: any, guildId?: string) {
|
||||
const nick = guildId ? GuildMemberStore.getNick(guildId, user.id) : null;
|
||||
return {
|
||||
@@ -318,7 +362,66 @@ export async function handlePuppetAction(action: string, params: Record<string,
|
||||
guildId: channel?.guild_id ?? guildId ?? null,
|
||||
channelId: voiceState.channelId,
|
||||
channelName: channel?.name ?? null,
|
||||
members: Object.values(states).map(voiceMember)
|
||||
members: Object.values(states).map(state =>
|
||||
voiceMember(state, (channel?.guild_id as string | undefined) ?? (guildId as string | undefined))
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
case PuppetAction.VoiceUsers: {
|
||||
const { userIds } = params;
|
||||
assert(
|
||||
Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50,
|
||||
PuppetErrorCode.InvalidParam,
|
||||
"userIds must be an array of 1 to 50 user IDs"
|
||||
);
|
||||
assert(
|
||||
userIds.every(id => isSnowflake(id)),
|
||||
PuppetErrorCode.InvalidSnowflake,
|
||||
"every userId must be a snowflake"
|
||||
);
|
||||
|
||||
const users = (userIds as string[]).map(id => {
|
||||
const state = VoiceStateStore.getVoiceStateForUser(id) ?? VoiceStateStore.getVoiceState(null, id);
|
||||
const user = UserStore.getUser(id);
|
||||
const profile = {
|
||||
id,
|
||||
username: user?.username ?? id,
|
||||
globalName: user?.globalName ?? null,
|
||||
avatar: user?.avatar ?? null
|
||||
};
|
||||
|
||||
if (!state?.channelId) {
|
||||
return { ...profile, channelId: null, channelName: null, guildId: null, members: [] };
|
||||
}
|
||||
|
||||
const channel = ChannelStore.getChannel(state.channelId);
|
||||
const states = VoiceStateStore.getVoiceStatesForChannel(state.channelId);
|
||||
|
||||
return {
|
||||
...profile,
|
||||
channelId: state.channelId,
|
||||
channelName: channel?.name ?? null,
|
||||
guildId: (channel?.guild_id as string | undefined) ?? null,
|
||||
members: Object.values(states).map(member =>
|
||||
voiceMember(member, channel?.guild_id as string | undefined)
|
||||
)
|
||||
};
|
||||
});
|
||||
|
||||
return ok({ users });
|
||||
}
|
||||
|
||||
case PuppetAction.UserStatus: {
|
||||
const { userIds } = params;
|
||||
assert(
|
||||
Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50,
|
||||
PuppetErrorCode.InvalidParam,
|
||||
"userIds must be an array of 1 to 50 user IDs"
|
||||
);
|
||||
|
||||
return ok({
|
||||
users: (userIds as string[]).map(id => ({ id, status: presenceOf(id) }))
|
||||
});
|
||||
}
|
||||
|
||||
@@ -339,11 +442,127 @@ export async function handlePuppetAction(action: string, params: Record<string,
|
||||
channels: [...byChannel.entries()].map(([channelId, states]) => ({
|
||||
channelId,
|
||||
channelName: ChannelStore.getChannel(channelId)?.name ?? null,
|
||||
members: states.map(voiceMember)
|
||||
members: states.map(state => voiceMember(state, guildId as string))
|
||||
}))
|
||||
});
|
||||
}
|
||||
|
||||
case PuppetAction.VoiceJoin:
|
||||
case PuppetAction.VoiceMove: {
|
||||
const { channelId, mute = true } = params;
|
||||
assert(isSnowflake(channelId), PuppetErrorCode.InvalidSnowflake, "channelId must be a snowflake");
|
||||
|
||||
const channel = ChannelStore.getChannel(channelId);
|
||||
assert(channel, PuppetErrorCode.ChannelNotFound, `channel ${channelId} not found`);
|
||||
assert(
|
||||
VOICE_CHANNEL_TYPES.has(channel.type),
|
||||
PuppetErrorCode.InvalidParam,
|
||||
`channel type ${channel.type} is not a voice channel`
|
||||
);
|
||||
|
||||
const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel;
|
||||
assert(
|
||||
typeof selectVoiceChannel === "function",
|
||||
PuppetErrorCode.NotSupported,
|
||||
"voice actions are not available in this client"
|
||||
);
|
||||
|
||||
if (action === PuppetAction.VoiceJoin && mute !== false) {
|
||||
const setSelfMute = (MediaEngineActions as any)?.setSelfMute;
|
||||
if (typeof setSelfMute === "function") {
|
||||
try {
|
||||
setSelfMute(true);
|
||||
} catch {
|
||||
// Best effort: the join still works unmuted.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await selectVoiceChannel(channelId);
|
||||
} catch (error) {
|
||||
throw new ActionError(PuppetErrorCode.VoiceFailed, `voice join failed: ${String(error)}`);
|
||||
}
|
||||
|
||||
return ok({
|
||||
channelId,
|
||||
joined: true,
|
||||
muted: action === PuppetAction.VoiceJoin ? mute !== false : undefined
|
||||
});
|
||||
}
|
||||
|
||||
case PuppetAction.VoiceLeave: {
|
||||
const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel;
|
||||
assert(
|
||||
typeof selectVoiceChannel === "function",
|
||||
PuppetErrorCode.NotSupported,
|
||||
"voice actions are not available in this client"
|
||||
);
|
||||
|
||||
try {
|
||||
await selectVoiceChannel(null);
|
||||
} catch (error) {
|
||||
throw new ActionError(PuppetErrorCode.VoiceFailed, `voice leave failed: ${String(error)}`);
|
||||
}
|
||||
|
||||
return ok({ left: true });
|
||||
}
|
||||
|
||||
case PuppetAction.VoiceDevices: {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return ok({
|
||||
input: devices
|
||||
.filter(device => device.kind === "audioinput")
|
||||
.map(device => ({ id: device.deviceId, label: device.label || device.deviceId })),
|
||||
output: devices
|
||||
.filter(device => device.kind === "audiooutput")
|
||||
.map(device => ({ id: device.deviceId, label: device.label || device.deviceId }))
|
||||
});
|
||||
}
|
||||
|
||||
case PuppetAction.VoiceSetDevices: {
|
||||
const { inputDeviceId, outputDeviceId } = params;
|
||||
if (inputDeviceId !== undefined) {
|
||||
assert(
|
||||
typeof inputDeviceId === "string" && inputDeviceId.length <= 200,
|
||||
PuppetErrorCode.InvalidParam,
|
||||
"inputDeviceId must be a short string"
|
||||
);
|
||||
}
|
||||
if (outputDeviceId !== undefined) {
|
||||
assert(
|
||||
typeof outputDeviceId === "string" && outputDeviceId.length <= 200,
|
||||
PuppetErrorCode.InvalidParam,
|
||||
"outputDeviceId must be a short string"
|
||||
);
|
||||
}
|
||||
|
||||
const setInputDevice = (MediaEngineActions as any)?.setInputDevice;
|
||||
const setOutputDevice = (MediaEngineActions as any)?.setOutputDevice;
|
||||
|
||||
if (inputDeviceId !== undefined) {
|
||||
assert(
|
||||
typeof setInputDevice === "function",
|
||||
PuppetErrorCode.NotSupported,
|
||||
"device selection is not available in this client"
|
||||
);
|
||||
await setInputDevice(inputDeviceId);
|
||||
}
|
||||
if (outputDeviceId !== undefined) {
|
||||
assert(
|
||||
typeof setOutputDevice === "function",
|
||||
PuppetErrorCode.NotSupported,
|
||||
"device selection is not available in this client"
|
||||
);
|
||||
await setOutputDevice(outputDeviceId);
|
||||
}
|
||||
|
||||
return ok({
|
||||
inputDeviceId: inputDeviceId ?? null,
|
||||
outputDeviceId: outputDeviceId ?? null
|
||||
});
|
||||
}
|
||||
|
||||
case PuppetAction.UserSearch: {
|
||||
const { query, guildId, limit = 10 } = params;
|
||||
assert(
|
||||
@@ -379,20 +598,45 @@ export async function handlePuppetAction(action: string, params: Record<string,
|
||||
|
||||
const channels = ChannelStore.getSortedPrivateChannels()
|
||||
.slice(0, Number(limit))
|
||||
.map(channel => ({
|
||||
.map(channel => {
|
||||
const raw = (channel as any).recipients;
|
||||
const recipientIds: string[] = Array.isArray(raw)
|
||||
? raw
|
||||
: Object.values(raw ?? {})
|
||||
.map((user: any) => user?.id ?? user)
|
||||
.filter(Boolean);
|
||||
|
||||
// 1:1 DMs may store no recipient list; the channel knows the other user.
|
||||
if (!recipientIds.length && typeof (channel as any).getRecipientId === "function") {
|
||||
const id = (channel as any).getRecipientId();
|
||||
if (id) recipientIds.push(id);
|
||||
}
|
||||
|
||||
const lastMessageId = (channel as any).lastMessageId ?? null;
|
||||
const unread = unreadCounts(channel.id);
|
||||
|
||||
return {
|
||||
id: channel.id,
|
||||
type: channel.type,
|
||||
name: (channel as any).name ?? null,
|
||||
icon: (channel as any).icon ?? null,
|
||||
lastMessageId: (channel as any).lastMessageId ?? null,
|
||||
recipients: Object.values((channel as any).recipients ?? {}).map((user: any) => ({
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
globalName: user.globalName ?? null,
|
||||
avatar: user.avatar ?? null,
|
||||
bot: !!user.bot
|
||||
}))
|
||||
}));
|
||||
lastMessageId,
|
||||
lastMessageAt: snowflakeToIso(lastMessageId),
|
||||
unread: unread.unread,
|
||||
mentions: unread.mentions,
|
||||
recipients: recipientIds.map(id => {
|
||||
const user = UserStore.getUser(id);
|
||||
return {
|
||||
id,
|
||||
username: user?.username ?? id,
|
||||
globalName: user?.globalName ?? null,
|
||||
avatar: user?.avatar ?? null,
|
||||
status: presenceOf(id),
|
||||
bot: !!user?.bot
|
||||
};
|
||||
})
|
||||
};
|
||||
});
|
||||
|
||||
return ok({ channels });
|
||||
}
|
||||
|
||||
@@ -129,6 +129,20 @@ export function initPuppetEvents() {
|
||||
}
|
||||
});
|
||||
|
||||
dispatcher.subscribe("SPEAKING", (event: any) => {
|
||||
try {
|
||||
if (!event?.userId) return;
|
||||
VesktopNative.puppet.emit({
|
||||
type: "VOICE_SPEAKING",
|
||||
userId: event.userId,
|
||||
speaking: event.speaking !== false,
|
||||
channelId: event.channelId ?? event.channel_id ?? null
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[puppet] speaking normalization failed:", error);
|
||||
}
|
||||
});
|
||||
|
||||
console.log("[veskslop] puppet events active");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -18,6 +18,13 @@ export const enum PuppetAction {
|
||||
StateGet = "state.get",
|
||||
VoiceGet = "voice.get",
|
||||
VoiceOverview = "voice.overview",
|
||||
VoiceJoin = "voice.join",
|
||||
VoiceLeave = "voice.leave",
|
||||
VoiceMove = "voice.move",
|
||||
VoiceUsers = "voice.users",
|
||||
UserStatus = "user.status",
|
||||
VoiceDevices = "voice.devices",
|
||||
VoiceSetDevices = "voice.setDevices",
|
||||
UserSearch = "user.search",
|
||||
DmList = "dm.list",
|
||||
MessageSend = "message.send",
|
||||
@@ -33,6 +40,13 @@ export const PuppetActions: readonly PuppetAction[] = [
|
||||
PuppetAction.StateGet,
|
||||
PuppetAction.VoiceGet,
|
||||
PuppetAction.VoiceOverview,
|
||||
PuppetAction.VoiceJoin,
|
||||
PuppetAction.VoiceLeave,
|
||||
PuppetAction.VoiceMove,
|
||||
PuppetAction.VoiceUsers,
|
||||
PuppetAction.UserStatus,
|
||||
PuppetAction.VoiceDevices,
|
||||
PuppetAction.VoiceSetDevices,
|
||||
PuppetAction.UserSearch,
|
||||
PuppetAction.DmList,
|
||||
PuppetAction.MessageSend,
|
||||
@@ -56,6 +70,7 @@ export const enum PuppetErrorCode {
|
||||
NotSendable = "NOT_SENDABLE",
|
||||
SendFailed = "SEND_FAILED",
|
||||
ReactionFailed = "REACTION_FAILED",
|
||||
VoiceFailed = "VOICE_FAILED",
|
||||
DmUnavailable = "DM_UNAVAILABLE",
|
||||
NotSupported = "NOT_SUPPORTED",
|
||||
RateLimited = "RATE_LIMITED",
|
||||
|
||||
Reference in New Issue
Block a user