diff --git a/poc/.gitignore b/poc/.gitignore index 39b5695..68c7da3 100644 --- a/poc/.gitignore +++ b/poc/.gitignore @@ -4,5 +4,10 @@ node_modules/ .token .secrets/ .veskslop-state.json +preferences.json +tls/ +models/ actions.jsonl actions.jsonl.* +transcripts.jsonl +transcripts.jsonl.* diff --git a/poc/package.json b/poc/package.json index 3af3073..9a2db9d 100644 --- a/poc/package.json +++ b/poc/package.json @@ -14,7 +14,8 @@ "control:mock": "tsx src/mock-core.ts", "test:trigger": "tsx src/test-trigger.ts", "cdp:dispatch": "tsx src/cdp-dispatch.ts", - "test:cache": "tsx src/test-cache.ts" + "test:cache": "tsx src/test-cache.ts", + "test:audio": "tsx src/test-audio.ts" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", diff --git a/poc/public/app.js b/poc/public/app.js index 3604f59..b1b7cd8 100644 --- a/poc/public/app.js +++ b/poc/public/app.js @@ -23,7 +23,25 @@ const state = { eventRate: 0, events: [], visibleGuilds: null, + visibleGuildsSaved: null, + timeZoneOverride: localStorage.getItem("veskslop-timezone") ?? "", + speech: { provider: null, language: null, voice: null }, + scope: "guild", + speaking: new Set(), + people: new Map(), + favourites: { users: [], channels: [] }, + favouriteUsers: [], + speakingRenderAt: 0, + daemonDown: false, composerChannelId: null, + audioRecording: false, + clientAudio: { + on: false, + ws: null, + ctx: null, + nextTime: 0, + sinkId: localStorage.getItem("veskslop-browser-output") ?? "" + }, takeover: localStorage.getItem("veskslop-takeover") === "1", consoleOpen: false, consoleTab: "events", @@ -43,22 +61,56 @@ function visibleGuildsList() { return state.guilds.filter(guild => isGuildVisible(guild.id)); } -function loadVisibleGuilds() { - try { - const raw = localStorage.getItem("veskslop-visible-guilds"); - state.visibleGuilds = raw ? new Set(JSON.parse(raw)) : null; - } catch { - state.visibleGuilds = null; - } +/** All guilds visible is stored as null. */ +function normalizedVisibility(visibility) { + if (!visibility) return null; + if (state.guilds.length > 0 && visibility.size === state.guilds.length) return null; + return visibility; } -function saveVisibleGuilds() { - if (!state.visibleGuilds || state.visibleGuilds.size === state.guilds.length) { +function sameVisibility(a, b) { + if (!a || !b) return !a && !b; + if (a.size !== b.size) return false; + for (const id of a) if (!b.has(id)) return false; + return true; +} + +function updateGuildsSaveButton() { + const button = $("#guilds-save"); + if (button) button.disabled = sameVisibility(state.visibleGuilds, state.visibleGuildsSaved); +} + +async function loadPreferences() { + try { + const prefs = await VK.api.get("/api/preferences"); + const visibility = Array.isArray(prefs.visibleGuilds) ? new Set(prefs.visibleGuilds) : null; + state.visibleGuilds = visibility ? new Set(visibility) : null; + state.visibleGuildsSaved = visibility ? new Set(visibility) : null; + } catch { state.visibleGuilds = null; - localStorage.removeItem("veskslop-visible-guilds"); - return; + state.visibleGuildsSaved = null; + } + + updateGuildsSaveButton(); +} + +/** Persist the current checkbox selection to the daemon state. */ +async function saveGuildVisibility() { + const visibility = normalizedVisibility(state.visibleGuilds); + const visibleGuilds = visibility ? [...visibility] : null; + + try { + const prefs = await VK.api.post("/api/preferences", { visibleGuilds }); + state.visibleGuildsSaved = Array.isArray(prefs.visibleGuilds) ? new Set(prefs.visibleGuilds) : null; + state.visibleGuilds = state.visibleGuildsSaved ? new Set(state.visibleGuildsSaved) : null; + + renderVisibleGuilds(); + renderGuildPicker(); + renderGuildSwitcher(); + toast("Guild visibility saved", "ok"); + } catch (error) { + toast(error.message, "error"); } - localStorage.setItem("veskslop-visible-guilds", JSON.stringify([...state.visibleGuilds])); } /* Helpers */ @@ -174,9 +226,108 @@ function guildMarkContent(mark, guild) { mark.append(img); } -function formatTime(timestamp) { +function browserTimeZone() { + try { + return Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC"; + } catch { + return "UTC"; + } +} + +function formatDate(timestamp, options) { const date = new Date(timestamp); - return Number.isNaN(date.getTime()) ? "" : date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + if (Number.isNaN(date.getTime())) return ""; + + return new Intl.DateTimeFormat(undefined, { + ...options, + timeZone: state.timeZoneOverride || undefined + }).format(date); +} + +function formatTime(timestamp) { + return formatDate(timestamp, { hour: "2-digit", minute: "2-digit" }); +} + +function formatClock(timestamp) { + return formatDate(timestamp, { hour: "2-digit", minute: "2-digit", second: "2-digit" }); +} + +function formatDateTime(timestamp) { + return formatDate(timestamp, { dateStyle: "medium", timeStyle: "medium" }); +} + +function updateTimeZoneStatus() { + const status = $("#timezone-status"); + if (!status) return; + + status.textContent = state.timeZoneOverride + ? `override · ${state.timeZoneOverride}` + : `auto · ${browserTimeZone()}`; +} + +function renderTimeZones() { + renderMessages(); + renderEvents(); + if (state.consoleTab === "transcript") void refreshTranscript(); + if (!$("#settings-overlay").hidden) void refreshLogs(); +} + +const CURATED_ZONES = [ + { name: "UTC", abbr: "UTC", zone: "UTC" }, + { name: "UK", abbr: "GMT/BST", zone: "Europe/London" }, + { name: "Central Europe", abbr: "CET/CEST", zone: "Europe/Berlin" }, + { name: "Eastern Europe", abbr: "EET/EEST", zone: "Europe/Bucharest" }, + { name: "US East Coast", abbr: "ET", zone: "America/New_York" }, + { name: "Ohio", abbr: "ET", zone: "America/New_York" }, + { name: "US Central", abbr: "CT", zone: "America/Chicago" }, + { name: "US Mountain", abbr: "MT", zone: "America/Denver" }, + { name: "US West Coast", abbr: "PT", zone: "America/Los_Angeles" }, + { name: "Japan", abbr: "JST", zone: "Asia/Tokyo" }, + { name: "India", abbr: "IST", zone: "Asia/Kolkata" }, + { name: "Australia", abbr: "AET", zone: "Australia/Sydney" } +]; + +function initTimeZone() { + const select = $("#timezone-select"); + const zones = typeof Intl.supportedValuesOf === "function" ? Intl.supportedValuesOf("timeZone") : []; + const curated = new Set(CURATED_ZONES.map(entry => entry.zone)); + + select.replaceChildren(el("option", { value: "", text: `Auto (browser: ${browserTimeZone()})` })); + + select.append( + el( + "optgroup", + { label: "Common" }, + ...CURATED_ZONES.map(entry => + el("option", { value: entry.zone, text: `${entry.name} (${entry.abbr})` }) + ) + ) + ); + + select.append( + el( + "optgroup", + { label: "All timezones" }, + ...zones.filter(zone => !curated.has(zone)).map(zone => el("option", { value: zone, text: zone })) + ) + ); + + if (state.timeZoneOverride && !zones.includes(state.timeZoneOverride) && !curated.has(state.timeZoneOverride)) { + select.append(el("option", { value: state.timeZoneOverride, text: state.timeZoneOverride })); + } + + select.value = state.timeZoneOverride; + + updateTimeZoneStatus(); + + select.addEventListener("change", () => { + state.timeZoneOverride = select.value; + if (state.timeZoneOverride) localStorage.setItem("veskslop-timezone", state.timeZoneOverride); + else localStorage.removeItem("veskslop-timezone"); + + updateTimeZoneStatus(); + renderTimeZones(); + }); } function formatUptime(seconds) { @@ -233,7 +384,7 @@ async function loadState() { renderIdentity(); renderGuildPicker(); renderGuildSwitcher(); - return; + return false; } state.user = data.user ?? null; @@ -257,6 +408,8 @@ async function loadState() { } else if (state.channelId) { renderComposer(); } + + return true; } async function loadDms() { @@ -289,7 +442,10 @@ async function loadVoice() { state.voiceOverview = []; } + trackPeople((state.voiceOverview ?? []).flatMap(entry => entry.members ?? [])); + renderVoiceRail(); + renderPeople(); renderRightVoice(); renderCallStrip(); } @@ -328,8 +484,18 @@ async function refreshAll() { refreshing = true; try { - await loadState(); + await loadPreferences(); + + // The bridge can connect before the renderer answers state.get. + let loaded = false; + for (let attempt = 0; attempt < 15 && !loaded; attempt++) { + loaded = await loadState(); + if (!loaded) await new Promise(resolve => setTimeout(resolve, 2000)); + } + if (!loaded) return; + await loadDms(); + await refreshFavourites(); await loadVoice(); if (state.channelId) await loadMessages({ refresh: true }); } catch (error) { @@ -343,6 +509,9 @@ async function selectGuild(guildId) { state.guildId = guildId; state.channelId = null; state.channel = null; + state.scope = "guild"; + $("#scope-guild").classList.add("active"); + $("#scope-dm").classList.remove("active"); $("#guild-menu").hidden = true; renderGuildPicker(); @@ -423,7 +592,8 @@ function renderIdentity() { function renderGuildPicker() { const guild = currentGuild(); - $("#guild-picker-label").textContent = guild?.name ?? "Direct messages"; + $("#guild-picker-label").textContent = + state.scope === "dm" ? "Direct messages" : guild?.name ?? "Direct messages"; } function renderGuildSwitcher() { @@ -447,7 +617,6 @@ function setGuildVisible(guildId, visible) { if (visible) state.visibleGuilds.add(guildId); else state.visibleGuilds.delete(guildId); - saveVisibleGuilds(); if (state.guildId && !isGuildVisible(state.guildId)) { const next = visibleGuildsList()[0]; @@ -479,25 +648,79 @@ function renderVisibleGuilds() { root.append(el("label", { class: "vk-visibility-row" }, checkbox, mark, el("span", { text: guild.name }))); } + + updateGuildsSaveButton(); } function voiceMarks(member) { const marks = el("span", { class: "vk-marks" }); + const speaking = state.speaking.has(member.id); - if (member.selfMute || member.mute) { - marks.append(el("span", { class: "neutral", title: member.mute ? "muted by server" : "self-muted", html: VK.icon("micSlash") })); + // Fixed order: audio → video → share. + if (speaking) { + const levels = el("span", { class: "vk-levels", title: "speaking" }); + for (let index = 0; index < 4; index++) levels.append(el("span")); + marks.append(levels); } - if (member.selfDeaf || member.deaf) { + + if (member.suppress || member.mute || member.selfMute) { + const title = member.suppress ? "muted by server" : member.mute ? "server-muted" : "self-muted"; + marks.append(el("span", { class: "neutral", title, html: VK.icon("micSlash") })); + } + + if (member.deaf || member.selfDeaf) { marks.append(el("span", { class: "neutral", title: "deafened", html: VK.icon("speakerSlash") })); } + if (member.video) { + marks.append(el("span", { class: "accent", title: "camera on", html: VK.icon("camera") })); + } + + if (member.stream) { + marks.append(el("span", { class: "accent", title: "screen sharing", html: VK.icon("monitor") })); + } + return marks; } +function memberName(member) { + return member.nick ?? member.username ?? member.id; +} + +function memberNameNode(member, className = "vk-name") { + return el("span", { + class: `${className} ${state.speaking.has(member.id) ? "speaking" : ""}`.trim(), + text: memberName(member), + style: member.id === state.user?.id ? "color: var(--color-accent-300)" : "" + }); +} + +function trackPeople(members) { + for (const member of members ?? []) { + if (!member?.id || member.id === state.user?.id) continue; + + state.people.set(member.id, { + id: member.id, + username: member.username ?? member.id, + nick: member.nick ?? member.username ?? member.id, + avatar: member.avatar ?? null, + at: Date.now() + }); + } +} + function renderRail() { - renderTextChannels(); - renderVoiceRail(); - renderDirect(); + const guildScope = state.scope === "guild"; + $("#rail-guild").hidden = !guildScope; + $("#rail-dm").hidden = guildScope; + + if (guildScope) { + renderTextChannels(); + renderVoiceRail(); + renderPeople(); + } else { + renderDmRail(); + } } function renderTextChannels() { @@ -549,11 +772,18 @@ function renderVoiceRail() { const active = state.voiceSelf?.channelId === channel.id; if (call?.members?.length) { + trackPeople(call.members); + const block = el("div", { class: "vk-rail-voice" }); block.append( el( - "div", - { class: "vk-rail-voice-head" }, + "button", + { + class: "vk-rail-voice-head", + type: "button", + title: active ? "Leave this call" : "Join this voice channel", + onclick: () => void joinVoice(channel.id) + }, el("span", { class: "vk-icon", style: "color: var(--color-accent)", html: VK.icon("volume") }), el("span", { class: "vk-rail-name", style: "font-weight:500", text: channel.name }), el( @@ -570,16 +800,7 @@ function renderVoiceRail() { const members = el("div", { class: "vk-rail-voice-members" }); for (const member of call.members) { members.append( - el( - "span", - { class: "vk-voice-member" }, - el("span", { - class: "vk-name", - style: member.id === state.user?.id ? "color: var(--color-accent-300)" : "", - text: member.username - }), - voiceMarks(member) - ) + el("span", { class: "vk-voice-member" }, memberNameNode(member, "vk-name"), voiceMarks(member)) ); } block.append(members); @@ -591,6 +812,7 @@ function renderVoiceRail() { { class: `vk-rail-row ${active ? "active" : ""}`, type: "button", + title: "Join this voice channel", onclick: () => joinVoice(channel.id) }, el("span", { class: "vk-icon vk-dim", html: VK.icon("volume") }), @@ -602,8 +824,121 @@ function renderVoiceRail() { } } -function renderDirect() { - const root = $("#direct-list"); +function renderPeople() { + const root = $("#people-list"); + if (!root) return; + root.replaceChildren(); + + const people = [...state.people.values()].sort((a, b) => b.at - a.at).slice(0, 20); + if (!people.length) { + root.append(el("div", { class: "vk-empty", text: "No voice people seen yet." })); + return; + } + + for (const person of people) { + root.append( + el( + "button", + { + class: "vk-rail-row", + type: "button", + title: person.username, + onclick: () => openPersonDm(person.id) + }, + avatar(person, { size: "sm" }), + el("span", { + class: `vk-rail-name ${state.speaking.has(person.id) ? "speaking" : ""}`.trim(), + text: person.nick ?? person.username + }) + ) + ); + } +} + +function openPersonDm(userId) { + const dm = state.dms.find( + entry => entry.type === 1 && (entry.recipients ?? []).some(recipient => recipient.id === userId) + ); + + if (!dm) { + toast("No cached DM with that user", "error"); + return; + } + + void openDm(dm); +} + +function dmStarred(dm) { + if (dm.type === 3) return state.favourites.channels.includes(dm.id); + const recipient = dm.recipients?.[0]?.id; + return !!recipient && state.favourites.users.includes(recipient); +} + +function starButton(active, onToggle) { + const button = el("button", { + class: `vk-star ${active ? "on" : ""}`, + type: "button", + title: active ? "Remove from favourites" : "Add to favourites", + onclick: event => { + event.stopPropagation(); + onToggle(); + } + }); + button.innerHTML = VK.icon("star", 12); + return button; +} + +function unreadBadge(dm) { + const mentions = dm.mentions ?? 0; + const unread = dm.unread ?? 0; + if (!mentions && !unread) return null; + + const kind = !state.takeover ? "muted" : mentions ? "mention" : "unread"; + return el("span", { + class: `vk-dm-badge ${kind}`, + text: mentions ? String(mentions) : unread > 99 ? "99+" : String(unread) + }); +} + +function formatDmTime(iso) { + if (!iso) return ""; + + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return ""; + + const now = new Date(); + if (date.toDateString() === now.toDateString()) return formatTime(iso); + + const yesterday = new Date(now); + yesterday.setDate(now.getDate() - 1); + if (date.toDateString() === yesterday.toDateString()) return "yesterday"; + + return formatDate(iso, { month: "short", day: "numeric" }); +} + +function dmRow(dm) { + const active = dm.id === state.channelId; + const row = el("button", { + class: `vk-rail-row ${active ? "active" : ""}`, + type: "button", + onclick: () => void openDm(dm) + }); + + const badge = unreadBadge(dm); + if (badge) row.append(badge); + + row.append(dmAvatar(dm, { size: "sm" })); + row.append(el("span", { class: "vk-rail-name", text: dmLabel(dm) })); + + const time = formatDmTime(dm.lastMessageAt); + if (time) row.append(el("span", { class: "vk-dm-time", text: time })); + + row.append(starButton(dmStarred(dm), () => void toggleFavouriteDm(dm))); + return row; +} + +function renderDmRail() { + const root = $("#dm-rail-list"); if (!root) return; root.replaceChildren(); @@ -612,20 +947,72 @@ function renderDirect() { return; } - for (const dm of state.dms.slice(0, 12)) { - root.append( - el( - "button", - { - class: `vk-rail-row ${dm.id === state.channelId ? "active" : ""}`, - type: "button", - onclick: () => void selectChannel(dm.id) - }, - dmAvatar(dm, { size: "sm" }), - el("span", { class: "vk-rail-name", text: dmLabel(dm) }) - ) - ); + for (const dm of state.dms) root.append(dmRow(dm)); +} + +async function openDm(dm) { + await applyScope("dm"); + await selectChannel(dm.id); +} + +async function applyScope(scope) { + state.scope = scope; + $("#scope-guild").classList.toggle("active", scope === "guild"); + $("#scope-dm").classList.toggle("active", scope === "dm"); + renderGuildPicker(); + + if (scope === "dm") { + if (!state.channel?.dm) { + const first = state.dms[0]; + if (first) await selectChannel(first.id); + } + } else if (state.channel?.dm) { + const guild = currentGuild(); + const first = guild?.channels.find(channel => SENDABLE_TYPES.has(channel.type)); + if (first) await selectChannel(first.id); } + + renderRail(); +} + +async function saveFavourites(patch) { + try { + const prefs = await VK.api.post("/api/preferences", { + favourites: { ...state.favourites, ...patch } + }); + state.favourites = prefs.favourites ?? { users: [], channels: [] }; + + await refreshFavourites(); + renderDmRail(); + renderRightDms(); + } catch (error) { + toast(error.message, "error"); + } +} + +function toggleFavouriteDm(dm) { + if (dm.type === 3) { + const channels = state.favourites.channels.includes(dm.id) + ? state.favourites.channels.filter(id => id !== dm.id) + : [...state.favourites.channels, dm.id]; + void saveFavourites({ channels }); + return; + } + + const userId = dm.recipients?.[0]?.id; + if (!userId) { + toast("No cached recipient to favourite", "error"); + return; + } + + const users = state.favourites.users.includes(userId) + ? state.favourites.users.filter(id => id !== userId) + : [...state.favourites.users, userId]; + void saveFavourites({ users }); +} + +function removeFavouriteUser(userId) { + void saveFavourites({ users: state.favourites.users.filter(id => id !== userId) }); } /* Center rendering */ @@ -816,50 +1203,63 @@ function renderRightVoice() { if (!root) return; root.replaceChildren(); - const channels = (state.voiceOverview ?? []).filter(entry => entry.members?.length); - const rows = channels.flatMap(entry => entry.members.map(member => ({ member, entry }))); + const inCall = state.favouriteUsers.filter(user => user.channelId); + $("#voice-count").textContent = inCall.length ? `${inCall.length} in call` : ""; - $("#voice-count").textContent = rows.length ? `${Math.min(rows.length, 7)} of ${rows.length} shown` : ""; - - if (!rows.length) { - root.append(el("div", { class: "vk-empty", text: "No voice channels occupied." })); + if (!inCall.length) { + root.append(el("div", { class: "vk-empty", text: "No favourited users in a call." })); return; } - for (const { member, entry } of rows.slice(0, 7)) { - const isSelf = member.id === state.user?.id; + const groups = new Map(); + for (const user of inCall) { + const list = groups.get(user.channelId) ?? []; + list.push(user); + groups.set(user.channelId, list); + } - root.append( + for (const [channelId, users] of groups) { + const card = el( + "div", + { class: "vk-right-group" }, el( "div", - { class: `vk-right-row ${isSelf ? "highlight" : ""}` }, - avatar(member), - el( - "span", - { class: "vk-right-name" }, - el("span", { class: "vk-name", text: member.username }), - el("span", { class: "vk-sub", text: entry.channelName ?? entry.channelId }) - ), - el( - "span", - { class: "vk-right-trail" }, - voiceMarks(member), - isSelf - ? el("button", { - class: "btn btn-secondary vk-small-btn", - type: "button", - text: "View", - onclick: () => toast("Voice monitoring is not implemented yet", "error") - }) - : el("button", { - class: "btn btn-primary vk-small-btn", - type: "button", - text: "Join", - onclick: () => joinVoice(entry.channelId) - }) - ) + { class: "vk-right-group-head" }, + el("span", { class: "vk-icon", style: "color: var(--color-accent)", html: VK.icon("volume") }), + el("span", { class: "vk-name", text: users[0].channelName ?? channelId }) ) ); + + for (const user of users) { + card.append( + el( + "div", + { class: "vk-right-row" }, + avatar({ id: user.id, username: user.username, avatar: user.avatar }, { size: "sm" }), + el( + "span", + { class: "vk-right-name" }, + el("span", { + class: `vk-name ${state.speaking.has(user.id) ? "speaking" : ""}`.trim(), + text: user.username + }), + el("span", { class: "vk-sub", text: user.status }) + ), + el( + "span", + { class: "vk-right-trail" }, + el("button", { + class: "btn btn-primary vk-small-btn", + type: "button", + text: "Join", + onclick: () => joinVoice(channelId) + }) + ) + ) + ); + } + + root.append(card); } } @@ -875,20 +1275,7 @@ function renderRightDms() { return; } - for (const dm of state.dms.slice(0, 8)) { - root.append( - el( - "button", - { - class: `vk-rail-row ${dm.id === state.channelId ? "active" : ""}`, - type: "button", - onclick: () => void selectChannel(dm.id) - }, - dmAvatar(dm, { size: "sm" }), - el("span", { class: "vk-rail-name", text: dmLabel(dm) }) - ) - ); - } + for (const dm of state.dms.slice(0, 8)) root.append(dmRow(dm)); } function renderCallStrip() { @@ -917,7 +1304,7 @@ function renderCallStrip() { "span", { class: "vk-strip-pill" }, avatar(member, { size: "sm" }), - el("span", { class: "vk-strip-name", text: member.username }), + memberNameNode(member, "vk-strip-name"), voiceMarks(member) ) ) @@ -944,6 +1331,11 @@ function renderCallStrip() { } async function joinVoice(channelId) { + if (state.voiceSelf?.channelId === channelId) { + await voiceAction("/api/voice/leave"); + return; + } + await voiceAction("/api/voice/join", { channelId }); } @@ -959,23 +1351,30 @@ async function voiceAction(path, body) { /* Console */ +function eventLine(entry) { + return el( + "div", + { class: "vk-event-line" }, + el("span", { class: "vk-event-time", text: formatClock(entry.at) }), + el("span", { class: "vk-event-type", text: entry.event.type ?? "event" }), + el("span", { text: JSON.stringify(entry.event) }) + ); +} + +function renderEvents() { + const log = $("#event-log"); + if (!log) return; + log.replaceChildren(...state.events.slice(0, 200).map(eventLine)); +} + function addEvent(event) { - state.events.unshift(event); + state.events.unshift({ at: Date.now(), event }); if (state.events.length > 300) state.events.pop(); state.eventCount++; const log = $("#event-log"); if (log) { - log.prepend( - el( - "div", - { class: "vk-event-line" }, - el("span", { class: "vk-event-time", text: new Date().toLocaleTimeString() }), - el("span", { class: "vk-event-type", text: event.type ?? "event" }), - el("span", { text: JSON.stringify(event) }) - ) - ); - + log.prepend(eventLine(state.events[0])); while (log.childElementCount > 200) log.lastElementChild.remove(); } @@ -1001,6 +1400,7 @@ function setConsoleTab(tab) { for (const button of $$(".vk-console-tab")) button.classList.toggle("active", button.dataset.consoleTab === tab); for (const panel of $$(".vk-console-panel")) panel.hidden = panel.id !== `console-panel-${tab}`; if (!state.consoleOpen) setConsoleOpen(true); + if (tab === "transcript") void refreshTranscript(); } /* Settings */ @@ -1010,6 +1410,7 @@ function openSettings() { void refreshCoreStatus(); void refreshAuth(); void refreshLogs(); + void refreshAudio(); } function closeSettings() { @@ -1037,7 +1438,7 @@ function renderCoreStatus(status) { statusRow("process", badge(process.running ? "running" : "stopped", process.running ? "ok" : "")), statusRow("pid", el("code", { text: process.pid ?? "—" })), statusRow("managed", el("span", { text: process.managed ? "yes" : process.running ? "external" : "no" })), - statusRow("started", el("span", { text: process.startedAt ? new Date(process.startedAt).toLocaleString() : "—" })), + statusRow("started", el("span", { text: process.startedAt ? formatDateTime(process.startedAt) : "—" })), statusRow("command", el("code", { text: process.command ?? "—" })) ]; @@ -1118,6 +1519,7 @@ async function authAction(path, label) { async function refreshLogs() { const root = $("#action-log"); + try { const entries = await VK.api.get("/api/logs?limit=100"); root.replaceChildren(); @@ -1132,7 +1534,7 @@ async function refreshLogs() { el( "div", { class: "vk-log-line" }, - el("span", { class: "vk-log-time", text: entry.timestamp?.slice(11, 19) ?? "" }), + el("span", { class: "vk-log-time", text: entry.timestamp ? formatClock(entry.timestamp) : "" }), el("span", { class: "vk-log-type", text: entry.type ?? "?" }), el("span", { text: JSON.stringify(entry).slice(0, 220) }) ) @@ -1143,6 +1545,351 @@ async function refreshLogs() { } } +/* Audio */ + +function fillDeviceSelect(select, devices, defaultLabel) { + const current = select.value; + select.replaceChildren(); + + if (!devices.length) { + select.append(el("option", { value: "", text: "No devices reported", disabled: true })); + return; + } + + select.append(el("option", { value: "", text: defaultLabel })); + + for (const device of devices) { + select.append(el("option", { value: device.id, text: device.label })); + } + + if ([...select.options].some(option => option.value === current)) select.value = current; +} + +async function refreshAudio() { + try { + const status = await VK.api.get("/api/audio"); + state.audioRecording = !!status.recording; + + $("#audio-status").textContent = status.available + ? `ok · ${status.backend ?? "pulse"} · duck ${status.duckPercent}%${status.recording ? " · recording" : ""}` + : (status.error ?? "unavailable"); + + const hintRow = $("#audio-hint-row"); + hintRow.hidden = !status.hint; + $("#audio-hint").textContent = status.hint ?? ""; + + const tts = status.tts ?? {}; + $("#audio-tts").textContent = `local ${tts.local ? "yes" : "no"} · api ${ + tts.api ? "yes" : "no" + } · player ${status.player ? "yes" : "no"} · queue ${status.queue ?? 0}`; + $("#audio-capture").textContent = status.recording ? "Stop capture" : "Start capture"; + } catch (error) { + $("#audio-status").textContent = error.message; + } + + await refreshBrowserAudio(); + await refreshSpeech(); +} + +/* Speech settings (server-side preferences). */ + +async function refreshSpeech() { + try { + const status = await VK.api.get("/api/audio"); + const tts = status.tts ?? {}; + const stt = status.stt ?? {}; + + $("#speech-status").textContent = `${tts.engine ?? "no local engine"}${tts.api ? " + api" : ""}`; + $("#speech-stt").textContent = stt.local + ? `local ready${stt.engine ? ` (${stt.engine})` : ""}` + : stt.api + ? "API configured" + : "not configured"; + } catch { + // the daemon status is optional here + } + + let groups = []; + try { + groups = await VK.api.get("/api/audio/tts/voices"); + } catch { + // voices need a local engine + } + + try { + const prefs = await VK.api.get("/api/preferences"); + state.speech = prefs.tts ?? { provider: null, language: null, voice: null }; + + const languageSelect = $("#speech-language"); + languageSelect.replaceChildren(el("option", { value: "", text: "Auto" })); + for (const group of groups) { + languageSelect.append(el("option", { value: group.id, text: `${group.name} (${group.id})` })); + } + languageSelect.value = state.speech.language ?? ""; + + fillVoiceOptions(groups, state.speech.language, state.speech.voice); + + $("#speech-provider").value = state.speech.provider ?? ""; + $("#tts-provider").value = state.speech.provider ?? ""; + } catch { + // preferences are optional + } +} + +function fillVoiceOptions(groups, language, selected) { + const select = $("#speech-voice"); + const voices = groups.find(group => group.id === language)?.voices ?? []; + + select.replaceChildren(el("option", { value: "", text: "Default" })); + for (const voice of voices) { + const label = `${voice.name}${voice.gender ? ` · ${voice.gender}` : ""}`; + select.append(el("option", { value: voice.id, text: label })); + } + + select.value = selected && voices.some(voice => voice.id === selected) ? selected : ""; +} + +async function saveSpeech() { + const provider = $("#speech-provider").value || null; + const language = $("#speech-language").value || null; + const voice = $("#speech-voice").value || null; + + try { + const prefs = await VK.api.post("/api/preferences", { tts: { provider, language, voice } }); + state.speech = prefs.tts ?? { provider: null, language: null, voice: null }; + $("#tts-provider").value = state.speech.provider ?? ""; + await refreshSpeech(); + toast("Speech settings saved", "ok"); + } catch (error) { + toast(error.message, "error"); + } +} + +/* Browser audio (puppeting) — separate from the bot's own audio devices. */ + +async function refreshBrowserAudio() { + const hint = $("#browser-hint"); + const output = $("#browser-output"); + + if (!navigator.mediaDevices?.enumerateDevices) { + hint.textContent = "Browser audio needs HTTPS or localhost."; + output.disabled = true; + return; + } + + hint.textContent = "The microphone will be used for talk-through later."; + + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const outputs = devices + .filter(device => device.kind === "audiooutput") + .map(device => ({ id: device.deviceId, label: device.label || "Output device" })); + + fillDeviceSelect(output, outputs, "Browser default"); + output.value = state.clientAudio.sinkId; + + if (!outputs.length) { + hint.textContent = "No devices listed. Press Enable browser audio and allow the microphone prompt."; + } + } catch (error) { + hint.textContent = error.message; + } +} + +async function grantBrowserAudio() { + if (!navigator.mediaDevices?.getUserMedia) { + toast("Browser audio needs HTTPS or localhost", "error"); + return; + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + for (const track of stream.getTracks()) track.stop(); + toast("Browser audio enabled", "ok"); + await refreshBrowserAudio(); + } catch (error) { + toast(`Could not enable browser audio: ${error.message}`, "error"); + } +} + +function applyBrowserSink() { + const context = state.clientAudio.ctx; + if (!context || typeof context.setSinkId !== "function") return; + + context.setSinkId(state.clientAudio.sinkId || "").catch(error => { + console.warn("[client-audio] could not set the output device:", error); + }); +} + +function playPcm(arrayBuffer) { + if (!state.clientAudio.ctx) { + state.clientAudio.ctx = new AudioContext({ sampleRate: 48000 }); + applyBrowserSink(); + } + + const context = state.clientAudio.ctx; + const samples = new Int16Array(arrayBuffer); + const buffer = context.createBuffer(1, samples.length, 48000); + const data = buffer.getChannelData(0); + + for (let index = 0; index < samples.length; index++) data[index] = samples[index] / 32768; + + const source = context.createBufferSource(); + source.buffer = buffer; + source.connect(context.destination); + + const now = context.currentTime; + if (state.clientAudio.nextTime < now + 0.05) state.clientAudio.nextTime = now + 0.05; + + source.start(state.clientAudio.nextTime); + state.clientAudio.nextTime += buffer.duration; +} + +function updateClientAudioButton() { + const button = $("#client-audio"); + if (!button) return; + + button.classList.toggle("active", state.clientAudio.on); + button.textContent = state.clientAudio.on ? "Client audio on" : "Client audio"; + + const status = $("#browser-audio-status"); + if (status) status.textContent = state.clientAudio.on ? "on" : "off"; +} + +function startClientAudio() { + const socket = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/audio`); + socket.binaryType = "arraybuffer"; + state.clientAudio.ws = socket; + + socket.onopen = () => socket.send(JSON.stringify({ type: "listen", on: true })); + + socket.onmessage = event => { + if (typeof event.data === "string") { + let message; + try { + message = JSON.parse(event.data); + } catch { + return; + } + + if (message.type === "error") { + toast(message.error, "error"); + stopClientAudio(); + } else if (message.type === "status" && message.listening) { + state.clientAudio.on = true; + updateClientAudioButton(); + } else if (message.type === "status" && !message.listening) { + stopClientAudio(); + } + return; + } + + if (state.clientAudio.on) playPcm(event.data); + }; + + socket.onclose = () => { + state.clientAudio.ws = null; + state.clientAudio.on = false; + updateClientAudioButton(); + }; +} + +function stopClientAudio() { + const socket = state.clientAudio.ws; + state.clientAudio.ws = null; + state.clientAudio.on = false; + + if (socket) { + try { + socket.send(JSON.stringify({ type: "listen", on: false })); + } catch { + // already closing + } + try { + socket.close(); + } catch { + // already closed + } + } + + updateClientAudioButton(); +} + +function toggleClientAudio() { + if (state.clientAudio.on || state.clientAudio.ws) stopClientAudio(); + else startClientAudio(); +} + +function syncTtsControls() { + const speak = $("#tts-speak"); + if (speak) speak.disabled = !state.takeover; +} + +async function speakText() { + if (!state.takeover) { + toast("Composer is read-only — turn Takeover on to speak", "error"); + return; + } + + const text = $("#tts-input").value.trim(); + if (!text) return; + + try { + const provider = $("#tts-provider").value; + await VK.api.post("/api/audio/speak", { text, provider }); + $("#tts-status").textContent = "queued"; + toast("Queued for speech", "ok"); + void refreshAudio(); + } catch (error) { + toast(error.message, "error"); + } +} + +/* Transcript */ + +function transcriptLine(segment) { + const time = formatClock(segment.startedAt); + const speaker = segment.speakerName ?? segment.speakerId ?? "call"; + const text = segment.text ?? `(audio ${(segment.durationMs / 1000).toFixed(1)}s, no STT)`; + + return el( + "div", + { class: "vk-transcript-line", "data-segment": segment.id }, + el("span", { class: "vk-transcript-time", text: time }), + el("span", { class: "vk-transcript-speaker", text: speaker }), + el("span", { class: "vk-transcript-text", text }), + segment.wav ? el("span", { class: "vk-transcript-wav", text: segment.wav.split("/").pop() }) : null + ); +} + +async function refreshTranscript() { + try { + const result = await VK.api.get("/api/transcript?limit=200"); + const list = $("#transcript-list"); + list.replaceChildren(); + + const segments = result.segments ?? []; + if (!segments.length) { + list.append( + el("div", { + class: "vk-empty", + text: "No captured segments yet. Start capture in Settings → Client → Bot audio." + }) + ); + } else { + for (const segment of segments) list.append(transcriptLine(segment)); + list.scrollTop = list.scrollHeight; + } + + const active = [...(result.sessions ?? [])].reverse().find(session => !session.endedAt); + $("#transcript-status").textContent = active + ? `session ${active.id.slice(-8)} · ${segments.length} segments` + : `${segments.length} segments`; + } catch (error) { + toast(error.message, "error"); + } +} + /* Health */ async function refreshHealth() { @@ -1167,9 +1914,39 @@ async function refreshHealth() { const mcpDot = $("#mcp-dot"); mcpDot.className = `vk-dot ${health.coreConnected ? "online" : "offline"}`; $("#mcp-state").textContent = health.coreConnected ? "mcp connected" : "mcp offline"; + + const process = health.process ?? {}; + const startButton = $("#core-start-banner"); + + state.daemonDown = false; + document.body.classList.toggle("vk-offline", !health.coreConnected); + + if (health.coreConnected) { + startButton.hidden = true; + } else if (process.running) { + startButton.hidden = false; + startButton.disabled = true; + startButton.textContent = "Starting…"; + } else { + startButton.hidden = false; + startButton.disabled = false; + startButton.textContent = "Start server"; + } + + // A missed CORE_STATUS, or a renderer that was not ready yet: keep trying. + if (health.coreConnected && !state.user) void refreshAll(); } catch { + state.daemonDown = true; + document.body.classList.add("vk-offline"); $("#health-dot").className = "vk-dot offline"; $("#health-bridge").textContent = "daemon down"; + + const startButton = $("#core-start-banner"); + if (startButton) { + startButton.hidden = false; + startButton.disabled = false; + startButton.textContent = "Refresh"; + } } } @@ -1189,7 +1966,54 @@ function applyHello(event) { function renderRightFavourites() { const root = $("#right-favourites"); if (!root) return; - root.replaceChildren(el("div", { class: "vk-empty", text: "Favourites are not implemented yet." })); + root.replaceChildren(); + + const users = state.favouriteUsers; + const groups = state.dms.filter(dm => dm.type === 3 && state.favourites.channels.includes(dm.id)); + + if (!users.length && !groups.length) { + root.append(el("div", { class: "vk-empty", text: "No favourites yet. Star a DM." })); + return; + } + + for (const user of users) { + const status = user.channelId ? `in ${user.channelName ?? "a call"}` : user.status; + root.append( + el( + "div", + { class: "vk-right-row" }, + avatar({ id: user.id, username: user.username, avatar: user.avatar }, { size: "sm" }), + el( + "span", + { class: "vk-right-name" }, + el("span", { + class: `vk-name ${state.speaking.has(user.id) ? "speaking" : ""}`.trim(), + text: user.username + }), + el("span", { class: "vk-sub", text: status }) + ), + el("span", { class: "vk-right-trail" }, starButton(true, () => removeFavouriteUser(user.id))) + ) + ); + } + + for (const dm of groups) root.append(dmRow(dm)); +} + +async function refreshFavourites() { + try { + const result = await VK.api.get("/api/favourites"); + state.favourites = { + users: (result.users ?? []).map(user => user.id), + channels: result.channels ?? [] + }; + state.favouriteUsers = result.users ?? []; + } catch { + // the core may be disconnected + } + + renderRightFavourites(); + renderRightVoice(); } function handleLiveEvent(event) { @@ -1197,6 +2021,43 @@ function handleLiveEvent(event) { void loadMessages(); } + if (event.type === "TRANSCRIPT_SEGMENT") { + const list = $("#transcript-list"); + if (list) { + list.append(transcriptLine(event.segment)); + list.scrollTop = list.scrollHeight; + $("#transcript-status").textContent = `live · ${list.childElementCount} segments`; + } + return; + } + + if (event.type === "TRANSCRIPT_TEXT") { + const line = document.querySelector(`[data-segment="${event.id}"] .vk-transcript-text`); + if (line) line.textContent = event.text; + return; + } + + if (event.type === "TRANSCRIPT_ERROR") { + toast(event.error, "error"); + return; + } + + if (event.type === "VOICE_SPEAKING") { + if (event.speaking) state.speaking.add(event.userId); + else state.speaking.delete(event.userId); + + const now = Date.now(); + if (now - state.speakingRenderAt > 500) { + state.speakingRenderAt = now; + renderVoiceRail(); + renderPeople(); + renderRightVoice(); + renderRightFavourites(); + renderCallStrip(); + } + return; + } + if (event.type === "VOICE_STATE_UPDATE") { const now = Date.now(); if (now - state.voiceReloadAt > 1000) { @@ -1220,7 +2081,7 @@ function handleLiveEvent(event) { function init() { hydrateIcons(); - loadVisibleGuilds(); + initTimeZone(); const takeover = $("#takeover"); takeover.checked = state.takeover; @@ -1230,6 +2091,7 @@ function init() { localStorage.setItem("veskslop-takeover", state.takeover ? "1" : "0"); $("#takeover-label").textContent = state.takeover ? "Takeover on" : "Takeover"; renderComposer(); + syncTtsControls(); }); $("#guild-picker").addEventListener("click", () => { @@ -1259,6 +2121,27 @@ function init() { $("#core-stop").addEventListener("click", () => void coreAction("/api/core/stop", "Stop")); $("#core-restart").addEventListener("click", () => void coreAction("/api/core/restart", "Restart")); + $("#core-start-banner").addEventListener("click", async event => { + if (state.daemonDown) { + location.reload(); + return; + } + + const button = event.currentTarget; + button.disabled = true; + button.textContent = "Starting…"; + + try { + await VK.api.post("/api/core/start"); + toast("Starting the client…", "ok"); + } catch (error) { + toast(error.message, "error"); + } + + setTimeout(() => void refreshHealth(), 1000); + setTimeout(() => void refreshHealth(), 5000); + }); + $("#auth-start").addEventListener("click", () => void authAction("/api/auth/qr", "QR login started")); $("#auth-cancel").addEventListener("click", () => void authAction("/api/auth/qr/cancel", "Login cancelled")); $("#auth-restart").addEventListener("click", async () => { @@ -1293,24 +2176,85 @@ function init() { } $("#raw-events").addEventListener("click", () => setConsoleTab("events")); + $("#transcript-refresh").addEventListener("click", () => void refreshTranscript()); $("#channel-refresh").addEventListener("click", () => void loadMessages({ refresh: true })); $("#guilds-show-all").addEventListener("click", () => { state.visibleGuilds = null; - localStorage.removeItem("veskslop-visible-guilds"); renderVisibleGuilds(); renderGuildSwitcher(); renderGuildPicker(); }); + $("#guilds-hide-all").addEventListener("click", () => { + state.visibleGuilds = new Set(); + renderVisibleGuilds(); + renderGuildSwitcher(); + renderGuildPicker(); + }); + + $("#guilds-save").addEventListener("click", () => void saveGuildVisibility()); + const ttsInput = $("#tts-input"); ttsInput.value = localStorage.getItem(TTS_DRAFT_KEY) ?? ""; ttsInput.addEventListener("input", () => localStorage.setItem(TTS_DRAFT_KEY, ttsInput.value)); + const ttsProvider = $("#tts-provider"); + ttsProvider.addEventListener("change", () => { + $("#speech-provider").value = ttsProvider.value; + void saveSpeech(); + }); + + $("#speech-provider").addEventListener("change", () => void saveSpeech()); + $("#speech-language").addEventListener("change", () => { + $("#speech-voice").value = ""; + void saveSpeech(); + }); + $("#speech-voice").addEventListener("change", () => void saveSpeech()); + + $("#scope-guild").addEventListener("click", () => void applyScope("guild")); + $("#scope-dm").addEventListener("click", () => void applyScope("dm")); + $("#tts-close").addEventListener("click", () => { $("#tts-panel").hidden = true; }); - $("#tts-speak").addEventListener("click", () => toast("TTS is not implemented yet", "error")); + $("#tts-speak").addEventListener("click", () => void speakText()); + $("#tts-stop").addEventListener("click", async () => { + try { + await VK.api.post("/api/audio/stop"); + $("#tts-status").textContent = "stopped"; + } catch (error) { + toast(error.message, "error"); + } + }); + + $("#audio-refresh").addEventListener("click", () => void refreshAudio()); + $("#audio-connect").addEventListener("click", async () => { + try { + const result = await VK.api.post("/api/audio/connect"); + toast(result.message ?? (result.connected ? "Devices connected" : "Could not connect"), result.connected ? "ok" : "error"); + void refreshAudio(); + } catch (error) { + toast(error.message, "error"); + } + }); + $("#client-audio").addEventListener("click", toggleClientAudio); + $("#browser-grant").addEventListener("click", () => void grantBrowserAudio()); + $("#browser-output").addEventListener("change", event => { + state.clientAudio.sinkId = event.currentTarget.value; + localStorage.setItem("veskslop-browser-output", state.clientAudio.sinkId); + applyBrowserSink(); + }); + $("#audio-capture").addEventListener("click", async () => { + try { + const start = !state.audioRecording; + await VK.api.post("/api/audio/capture", { start }); + toast(start ? "Capture started" : "Capture stopped", "ok"); + void refreshAudio(); + } catch (error) { + toast(error.message, "error"); + } + }); document.addEventListener("keydown", event => { if (event.key !== "Escape") return; @@ -1335,12 +2279,14 @@ function init() { syncConsoleCount(); renderRightFavourites(); - void loadState(); - void loadDms(); - void loadVoice(); + void refreshAll(); void refreshHealth(); + syncTtsControls(); setInterval(refreshHealth, 5000); + setInterval(() => { + if (!$("#tts-panel").hidden) void refreshAudio(); + }, 3000); setInterval(() => { if (!$("#settings-overlay").hidden) { if (!$("#auth-qr").hidden || $("#auth-captcha").hidden === false) void refreshAuth(); diff --git a/poc/public/index.html b/poc/public/index.html index ce973c9..a2c66c1 100644 --- a/poc/public/index.html +++ b/poc/public/index.html @@ -15,6 +15,11 @@ debug +
+ + +
+
+ +
@@ -186,9 +212,12 @@
-
- roadmap -

Voice transcription (STT/TTS) is not implemented yet. Transcript segments will appear here once voice audio lands.

+
+
+ no capture session + +
+
@@ -205,50 +234,174 @@
-
-
- Core process - - - - - -
-
-
- -
-
- Account login - - - - - -
-
- - -
-
Visible guilds - + + + + +
-
-
- Action log - +
+ + + Client + + +
+
+
+ Bot audio + + + + + +
+
+
+ Audio engine + checking… +
+ +
+ TTS + — +
+
+
+ +
+
+ Speech + +
+
+
+ TTS engine + +
+
+ Language + +
+
+ Voice + +
+
+ STT + — +
+
+
+ +
+
+ Display + +
+
+
+ Timezone + +
+
+
+ +
+
+ Browser audio + + + +
+
+
+ Output device + +
+
+ Input device + +
+
+ Call audio + off +
+
+ Note + Browser audio needs HTTPS or localhost. +
+
+
-
-
+ + +
+ + + Server + + +
+
+
+ Core process + + + + + +
+
+
+ +
+
+ Account login + + + + + +
+
+ + +
+ +
+
+ Action log + +
+
+
+
+
diff --git a/poc/public/style.css b/poc/public/style.css index 62ecca1..4cef528 100644 --- a/poc/public/style.css +++ b/poc/public/style.css @@ -166,6 +166,12 @@ button { border-radius: var(--radius-sm); cursor: pointer; font-size: 13px; + width: 100%; + background: none; + border: none; + color: var(--color-text); + text-align: left; + font-family: inherit; } .vk-menu-item:hover { @@ -268,6 +274,151 @@ button { font-size: 12.5px; } +.vk-start-server { + padding: 5px 12px; + font-size: 12px; + animation: vk-bloom 2.4s ease-in-out infinite; +} + +@keyframes vk-bloom { + 0%, + 100% { + box-shadow: 0 0 0 0 transparent; + } + 50% { + box-shadow: 0 0 12px 2px color-mix(in srgb, var(--color-accent) 45%, transparent); + } +} + +@media (prefers-reduced-motion: reduce) { + .vk-start-server { + animation: none; + } +} + +.vk-scope { + display: inline-flex; + border: 1px solid var(--color-divider); + border-radius: var(--radius-md); + overflow: hidden; +} + +.vk-scope-btn { + padding: 5px 12px; + font-size: 12.5px; + background: none; + border: none; + color: var(--color-neutral-400); + cursor: pointer; +} + +.vk-scope-btn + .vk-scope-btn { + border-left: 1px solid var(--color-divider); +} + +.vk-scope-btn:hover { + color: var(--color-text); +} + +.vk-scope-btn.active { + color: var(--color-accent); + box-shadow: inset 0 0 0 1px var(--color-accent); +} + +/* The main areas dim and stop taking input while the bridge is down. */ +.vk-offline .vk-body, +.vk-offline .vk-console { + opacity: 0.5; + filter: saturate(0.6); + pointer-events: none; +} + +.vk-rail-scope { + display: flex; + flex-direction: column; + gap: 14px; +} + +.vk-dm-row { + position: relative; +} + +.vk-dm-badge { + display: grid; + place-items: center; + min-width: 17px; + height: 16px; + padding: 0 4px; + margin-right: 2px; + border-radius: 8px; + font: 500 10px/1 ui-monospace, Menlo, monospace; + flex: none; +} + +.vk-dm-badge.unread { + background: var(--color-accent-800); + color: var(--color-accent-100); +} + +.vk-dm-badge.mention { + background: color-mix(in srgb, var(--color-mention) 35%, transparent); + color: var(--color-mention); +} + +.vk-dm-badge.muted { + background: var(--color-neutral-800); + color: var(--color-neutral-300); +} + +.vk-dm-time { + margin-left: auto; + font: 400 10px/1 ui-monospace, Menlo, monospace; + color: var(--color-neutral-400); + flex: none; +} + +.vk-star { + display: inline-flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + padding: 0; + border: none; + background: none; + color: var(--color-neutral-500); + cursor: pointer; + flex: none; +} + +.vk-star:hover { + color: var(--color-accent-300); +} + +.vk-star.on { + color: var(--color-accent-400); +} + +.vk-person-name.speaking, +.vk-voice-member .speaking, +.vk-right-name .speaking { + font-weight: 700; +} + +.vk-marks .unavailable { + opacity: 0.35; +} + +:root { + --color-mention: #d29922; +} + +.vk-client-audio.active { + color: var(--color-accent); + border-color: var(--color-accent); + background: color-mix(in srgb, var(--color-accent) 12%, transparent); +} + /* Call strip */ .vk-call-strip { @@ -484,6 +635,18 @@ button { display: flex; align-items: center; gap: 7px; + width: 100%; + padding: 0; + background: none; + border: none; + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} + +.vk-rail-voice-head:hover .vk-rail-name { + color: var(--color-accent-300); } .vk-rail-voice-members { @@ -989,6 +1152,24 @@ a.vk-attachment:hover { padding: 10px 6px 0; } +.vk-right-group { + display: flex; + flex-direction: column; + gap: 1px; + margin-bottom: 6px; +} + +.vk-right-group-head { + display: flex; + align-items: center; + gap: 7px; + padding: 4px 6px; + font-size: 12.5px; + color: var(--color-neutral-300); + background: var(--color-neutral-900); + border-radius: var(--radius-sm); +} + .vk-roadmap { display: flex; flex-direction: column; @@ -1002,6 +1183,63 @@ a.vk-attachment:hover { color: var(--color-neutral-500); } +.vk-transcript { + display: flex; + flex-direction: column; + gap: 8px; + height: 100%; + min-height: 0; +} + +.vk-transcript-toolbar { + display: flex; + align-items: center; + gap: 10px; +} + +.vk-transcript-toolbar .vk-small-btn { + margin-left: auto; +} + +.vk-transcript-list { + flex: 1; + min-height: 0; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 4px; + font-family: ui-monospace, Menlo, monospace; + font-size: 11px; +} + +.vk-transcript-line { + display: flex; + gap: 8px; + align-items: baseline; + color: var(--color-neutral-400); +} + +.vk-transcript-line .vk-transcript-time { + flex: none; + color: var(--color-neutral-500); +} + +.vk-transcript-line .vk-transcript-speaker { + flex: none; + min-width: 84px; + color: var(--color-accent-300); +} + +.vk-transcript-line .vk-transcript-text { + color: var(--color-neutral-200); + word-break: break-word; +} + +.vk-transcript-line .vk-transcript-wav { + flex: none; + color: var(--color-neutral-500); +} + /* Voice state marks */ .vk-marks { @@ -1280,6 +1518,56 @@ a.vk-attachment:hover { flex-direction: column; } +.vk-settings-group { + display: flex; + flex-direction: column; +} + +.vk-settings-summary { + display: flex; + align-items: center; + gap: 8px; + cursor: pointer; + list-style: none; + padding: 6px 0; + font: 500 12px/1 var(--font-heading); + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--color-neutral-300); +} + +.vk-settings-summary::-webkit-details-marker { + display: none; +} + +.vk-settings-summary:hover { + color: var(--color-accent-300); +} + +.vk-settings-group:not([open]) .vk-caret { + transform: rotate(-90deg); +} + +.vk-settings-group-body { + display: flex; + flex-direction: column; + gap: 22px; + padding-top: 12px; +} + +.vk-select { + background: var(--color-bg); + color: var(--color-text); + border: 1px solid var(--color-divider); + border-radius: var(--radius-sm); + padding: 4px 8px; + font-size: 12.5px; +} + +.vk-select:disabled { + opacity: 0.55; +} + .vk-settings-actions { margin-left: auto; display: flex; diff --git a/poc/scripts/stt-vosk.py b/poc/scripts/stt-vosk.py new file mode 100644 index 0000000..b78d4cc --- /dev/null +++ b/poc/scripts/stt-vosk.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Veskslop STT helper — transcribe a 16-bit mono WAV with Vosk on the CPU.""" + +import json +import os +import sys +import wave + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: stt-vosk.py ", file=sys.stderr) + return 2 + + model_path = os.environ.get("VESKSLOP_VOSK_MODEL") + if not model_path: + print("VESKSLOP_VOSK_MODEL is not set", file=sys.stderr) + return 2 + + try: + from vosk import KaldiRecognizer, Model, SetLogLevel + except ImportError: + print("vosk is not installed (pip install vosk)", file=sys.stderr) + return 2 + + SetLogLevel(-1) + model = Model(model_path) + + with wave.open(sys.argv[1], "rb") as wav: + if wav.getnchannels() != 1 or wav.getsampwidth() != 2: + print("input must be 16-bit mono WAV", file=sys.stderr) + return 2 + + recognizer = KaldiRecognizer(model, wav.getframerate()) + parts = [] + + while True: + data = wav.readframes(4000) + if not data: + break + if recognizer.AcceptWaveform(data): + parts.append(json.loads(recognizer.Result()).get("text", "")) + + parts.append(json.loads(recognizer.FinalResult()).get("text", "")) + + print(" ".join(part for part in parts if part).strip()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/poc/src/audio-service.ts b/poc/src/audio-service.ts new file mode 100644 index 0000000..d1304e4 --- /dev/null +++ b/poc/src/audio-service.ts @@ -0,0 +1,344 @@ +/* + * Veskslop daemon — TTS queue, ducking, capture, and transcription. + * + * Lines play one at a time. While another person is speaking, the virtual sink + * is ducked. Capture reads the client output, splits it into utterances, stores + * them in the transcript, and runs speech to text when a provider is set. + */ + +import { AudioManager } from "./audio.js"; +import type { CaptureSegment } from "./capture.js"; +import { CapturePipeline } from "./capture.js"; +import type { Stt } from "./stt.js"; +import { Tts, type TtsOptions, type TtsVoice } from "./tts.js"; +import type { TranscriptSegment, TranscriptSession, TranscriptStore } from "./transcript.js"; +import { DEFAULT_VAD, type VadConfig } from "./vad.js"; + +export interface AudioServiceOptions { + cacheDir?: string; + transcripts: TranscriptStore; + stt: Stt; + vad?: VadConfig; + autoCapture?: boolean; + getSelfId: () => Promise; + getVoiceContext?: () => Promise<{ guildId?: string | null; channelId?: string | null } | undefined>; + getSpeakerName?: (userId: string) => Promise; + onTranscript?: (event: Record) => void; +} + +export interface SpeakResult { + queued: boolean; + id: number; + position: number; +} + +interface SpeakerInterval { + userId: string; + name?: string; + start: number; + end?: number; +} + +export class AudioService { + readonly manager = new AudioManager(); + private readonly tts: Tts; + private readonly transcripts: TranscriptStore; + private readonly stt: Stt; + private readonly vad: VadConfig; + private readonly queue: Array<{ id: number; text: string; options: TtsOptions }> = []; + private readonly speakers = new Set(); + private readonly speakerIntervals: SpeakerInterval[] = []; + private readonly speakerNames = new Map(); + private playing = false; + private nextId = 1; + private ducked = false; + private selfId?: string; + private capture?: CapturePipeline; + private session?: TranscriptSession; + private segmentCount = 0; + private initialized = false; + + constructor(private readonly options: AudioServiceOptions) { + this.tts = new Tts(options.cacheDir); + this.transcripts = options.transcripts; + this.stt = options.stt; + this.vad = options.vad ?? DEFAULT_VAD; + } + + async init(): Promise { + if (this.initialized) return; + this.initialized = true; + + await this.manager.detect(); + await this.tts.detect(); + await this.stt.detect(); + if (!this.manager.status().available) { + console.warn(`[audio] unavailable: ${this.manager.status().error}`); + return; + } + + try { + await this.manager.ensureDevices(); + console.log( + `[audio] devices ready: sink=${this.manager.status().sink} source=${this.manager.status().source}` + ); + } catch (error) { + console.warn("[audio] device setup failed:", (error as Error).message); + } + } + + status() { + return { + ...this.manager.status(), + tts: this.tts.status(), + stt: this.stt.status(), + queue: this.queue.length, + speaking: this.playing, + capturing: !!this.capture, + sessionId: this.session?.id ?? null, + segments: this.segmentCount, + transcript: this.transcripts.stats() + }; + } + + async speak(text: string, options: TtsOptions = {}): Promise { + await this.init(); + + const manager = this.manager.status(); + if (!manager.available) throw new Error(manager.error ?? "audio is not available"); + + const tts = this.tts.status(); + const provider = options.provider ?? tts.defaultProvider; + if (provider === "local" && !tts.local) { + throw new Error("local TTS is not configured (VESKSLOP_TTS_CMD)"); + } + if (provider === "api" && !tts.api) { + throw new Error("API TTS is not configured (VESKSLOP_TTS_API_URL)"); + } + + const trimmed = text.trim(); + if (!trimmed) throw new Error("text is required"); + if (trimmed.length > 1000) throw new Error("text is too long (max 1000 characters)"); + + const id = this.nextId++; + this.queue.push({ id, text: trimmed, options: { ...options, provider } }); + void this.pump(); + + return { queued: true, id, position: this.queue.length }; + } + + stop(): void { + this.queue.length = 0; + this.manager.stopPlayback(); + } + + /** Track speakers for ducking and attribution, and auto-capture calls. */ + async onEvent(event: any): Promise { + if (event?.type === "VOICE_STATE_UPDATE") { + if (this.options.autoCapture) await this.handleVoiceState(event); + return; + } + + if (event?.type !== "VOICE_SPEAKING") return; + + if (!this.selfId) this.selfId = await this.options.getSelfId(); + if (this.selfId && event.userId === this.selfId) return; + + const userId = String(event.userId); + + if (event.speaking) { + this.speakers.add(userId); + this.speakerIntervals.push({ userId, name: await this.speakerName(userId), start: Date.now() }); + } else { + this.speakers.delete(userId); + const interval = [...this.speakerIntervals].reverse().find(item => item.userId === userId && !item.end); + if (interval) interval.end = Date.now(); + } + + this.pruneIntervals(); + await this.applyDuck(this.playing && this.speakers.size > 0); + } + + async startCapture(): Promise<{ sessionId: string }> { + if (this.capture) throw new Error("capture is already running"); + await this.init(); + + const status = this.manager.status(); + if (!status.available) throw new Error(status.hint ?? status.error ?? "audio is not available"); + if (!status.ffmpeg) throw new Error("ffmpeg is missing; install it to capture audio"); + + const context = (await this.options.getVoiceContext?.()) ?? {}; + const session = this.transcripts.startSession(context); + this.segmentCount = 0; + + this.capture = new CapturePipeline({ + ffmpeg: process.env.VESKSLOP_FFMPEG ?? "ffmpeg", + target: `${status.captureSink}.monitor`, + directory: `${this.tts.directory}/captures`, + sessionId: session.id, + vad: this.vad, + onSegment: segment => this.handleSegment(session, segment), + onError: error => { + console.error("[capture] stream error:", error.message); + this.options.onTranscript?.({ type: "TRANSCRIPT_ERROR", error: error.message }); + } + }); + + this.capture.start(); + this.session = session; + this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION", session }); + + return { sessionId: session.id }; + } + + stopCapture(): void { + this.capture?.stop(); + this.capture = undefined; + + if (this.session) { + this.transcripts.endSession(this.session.id); + this.options.onTranscript?.({ type: "TRANSCRIPT_SESSION_END", id: this.session.id }); + this.session = undefined; + } + + this.speakerIntervals.length = 0; + } + + /** Stream call audio to the browser. Returns a stop function. */ + startMonitor(onData: (chunk: Buffer) => void, onError?: (error: Error) => void): () => void { + return this.manager.startMonitor(onData, onError); + } + + ttsVoices(): Promise { + return this.tts.voices(); + } + + async dispose(): Promise { + this.queue.length = 0; + this.stopCapture(); + await this.manager.dispose(); + } + + private async handleVoiceState(event: any): Promise { + if (!this.selfId) this.selfId = await this.options.getSelfId(); + if (!this.selfId || event.userId !== this.selfId) return; + + if (event.channelId && !this.capture) { + console.log("[capture] auto-start for", event.channelId); + await this.startCapture().catch(error => console.warn("[capture] auto-start failed:", (error as Error).message)); + } else if (!event.channelId && this.capture) { + console.log("[capture] auto-stop"); + this.stopCapture(); + } + } + + private handleSegment(session: TranscriptSession, segment: CaptureSegment) { + this.segmentCount++; + + const speaker = this.dominantSpeaker(segment.startedAt, segment.endedAt); + const record: TranscriptSegment = { + id: `segment-${Date.now()}-${this.segmentCount}`, + sessionId: session.id, + startedAt: segment.startedAt, + endedAt: segment.endedAt, + durationMs: segment.durationMs, + speakerId: speaker?.userId ?? null, + speakerName: speaker?.name ?? null, + wav: segment.file, + text: null + }; + + this.transcripts.addSegment(record); + this.options.onTranscript?.({ type: "TRANSCRIPT_SEGMENT", segment: record }); + + void this.transcribe(record); + } + + private async transcribe(record: TranscriptSegment) { + if (!record.wav) return; + + try { + const text = await this.stt.transcribe(record.wav); + if (!text) return; + + this.transcripts.setSegmentText(record.id, text); + this.options.onTranscript?.({ type: "TRANSCRIPT_TEXT", id: record.id, text }); + } catch (error) { + console.error("[capture] transcription failed:", (error as Error).message); + this.options.onTranscript?.({ type: "TRANSCRIPT_ERROR", error: (error as Error).message }); + } + } + + private dominantSpeaker(startedAt: string, endedAt: string): SpeakerInterval | undefined { + const start = Date.parse(startedAt); + const end = Date.parse(endedAt); + const totals = new Map(); + + for (const interval of this.speakerIntervals) { + const overlap = Math.min(end, interval.end ?? end) - Math.max(start, interval.start); + if (overlap <= 0) continue; + + const entry = totals.get(interval.userId) ?? { ms: 0, interval }; + entry.ms += overlap; + totals.set(interval.userId, entry); + } + + let best: SpeakerInterval | undefined; + let bestMs = 0; + for (const entry of totals.values()) { + if (entry.ms > bestMs) { + best = entry.interval; + bestMs = entry.ms; + } + } + + return best; + } + + private async speakerName(userId: string): Promise { + if (this.speakerNames.has(userId)) return this.speakerNames.get(userId); + const name = await this.options.getSpeakerName?.(userId).catch(() => undefined); + if (name) this.speakerNames.set(userId, name); + return name; + } + + private pruneIntervals() { + const cutoff = Date.now() - 60_000; + while (this.speakerIntervals.length && this.speakerIntervals[0].end && this.speakerIntervals[0].end < cutoff) { + this.speakerIntervals.shift(); + } + } + + private async pump(): Promise { + if (this.playing) return; + + const item = this.queue.shift(); + if (!item) { + await this.applyDuck(false); + return; + } + + this.playing = true; + try { + const file = await this.tts.synthesize(item.text, item.options); + await this.applyDuck(this.speakers.size > 0); + await this.manager.play(file); + } catch (error) { + console.error("[audio] speak failed:", (error as Error).message); + } finally { + this.playing = false; + await this.applyDuck(false); + void this.pump(); + } + } + + private async applyDuck(on: boolean): Promise { + if (on === this.ducked) return; + try { + await this.manager.setDuck(on); + this.ducked = on; + } catch (error) { + console.warn("[audio] duck failed:", (error as Error).message); + } + } +} diff --git a/poc/src/audio.ts b/poc/src/audio.ts new file mode 100644 index 0000000..958581b --- /dev/null +++ b/poc/src/audio.ts @@ -0,0 +1,326 @@ +/* + * Veskslop daemon — audio devices, playback, ducking, and capture. + * + * The daemon plays audio into a virtual sink. A remap source of that sink's + * monitor becomes the client microphone. Client output can be routed to a + * second sink that the daemon records. Every system command is optional: when + * the host has no PulseAudio/PipeWire, the manager reports unavailable. + */ + +import { spawn, type ChildProcess } from "node:child_process"; + +const SINK = process.env.VESKSLOP_AUDIO_SINK ?? "veskslop_tts"; +const SOURCE = process.env.VESKSLOP_AUDIO_SOURCE ?? "veskslop_mic"; +const CAPTURE_SINK = process.env.VESKSLOP_AUDIO_CAPTURE_SINK ?? "veskslop_capture"; +const DUCK_PERCENT = Number(process.env.VESKSLOP_AUDIO_DUCK ?? 30); + +export class AudioUnavailableError extends Error {} + +export interface AudioManagerStatus { + available: boolean; + backend: "pulse" | null; + error?: string; + hint?: string; + sink: string; + source: string; + captureSink: string; + ducked: boolean; + duckPercent: number; + playing: boolean; + recording: boolean; + player: boolean; + ffmpeg: boolean; +} + +interface CommandResult { + code: number | null; + stdout: string; + stderr: string; + errorCode?: string; +} + +function run(command: string, args: string[]): Promise { + return new Promise(resolve => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", chunk => (stdout += chunk)); + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => + resolve({ code: null, stdout, stderr: String(error), errorCode: (error as NodeJS.ErrnoException).code }) + ); + child.on("close", code => resolve({ code, stdout, stderr })); + }); +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export class AudioManager { + private modules: number[] = []; + private player?: ChildProcess; + private recorder?: ChildProcess; + private ducked = false; + private available = false; + private error?: string; + private hint?: string; + private backend: "pulse" | null = null; + private playerAvailable = false; + private ffmpegAvailable = false; + + async detect(): Promise { + const ffplay = await run(process.env.VESKSLOP_FFPLAY ?? "ffplay", ["-version"]); + const ffmpeg = await run(process.env.VESKSLOP_FFMPEG ?? "ffmpeg", ["-version"]); + this.playerAvailable = ffplay.code === 0; + this.ffmpegAvailable = ffmpeg.code === 0; + + let info = await run("pactl", ["info"]); + if (info.code !== 0 && info.errorCode !== "ENOENT" && process.env.VESKSLOP_AUDIO_MANAGE_SERVER !== "0") { + await this.tryStartServer(); + info = await run("pactl", ["info"]); + } + + this.available = info.code === 0; + this.backend = this.available ? "pulse" : null; + + if (this.available) { + this.error = undefined; + this.hint = this.playerAvailable ? undefined : "ffplay is missing; install ffmpeg to play audio."; + return; + } + + if (info.errorCode === "ENOENT") { + this.error = "pactl is not installed"; + this.hint = + "Install a sound server and its tools, for example: apt install pulseaudio pulseaudio-utils ffmpeg"; + } else { + this.error = info.stderr.trim() || "the audio server is not reachable"; + this.hint = + "The audio server is not running. Start PulseAudio or PipeWire-Pulse. The daemon starts pulseaudio automatically when it is installed."; + } + } + + /** Start a private PulseAudio daemon when the server is missing. */ + private async tryStartServer(): Promise { + const start = await run("pulseaudio", ["--start", "--exit-idle-time=-1"]); + if (start.errorCode === "ENOENT") { + this.hint = "pulseaudio is not installed; install it or start PipeWire-Pulse."; + return; + } + + for (let attempt = 0; attempt < 6; attempt++) { + await sleep(500); + const info = await run("pactl", ["info"]); + if (info.code === 0) return; + } + } + + status(): AudioManagerStatus { + return { + available: this.available, + backend: this.backend, + error: this.error, + hint: this.hint, + sink: SINK, + source: SOURCE, + captureSink: CAPTURE_SINK, + ducked: this.ducked, + duckPercent: DUCK_PERCENT, + playing: !!this.player, + recording: !!this.recorder, + player: this.playerAvailable, + ffmpeg: this.ffmpegAvailable + }; + } + + /** Create the virtual devices when they are missing. Safe to call twice. */ + async ensureDevices(): Promise { + if (!this.available) await this.detect(); + if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available"); + + const sinks = (await this.pactl(["list", "short", "sinks"])).split("\n"); + const sources = (await this.pactl(["list", "short", "sources"])).split("\n"); + + if (!sinks.some(line => line.includes(SINK))) { + const id = await this.pactl([ + "load-module", + "module-null-sink", + `sink_name=${SINK}`, + "sink_properties=device.description=VeskslopTTS" + ]); + this.modules.push(Number(id)); + } + + if (!sources.some(line => line.includes(SOURCE))) { + const id = await this.pactl([ + "load-module", + "module-remap-source", + `master=${SINK}.monitor`, + `source_name=${SOURCE}`, + "source_properties=device.description=VeskslopMic" + ]); + this.modules.push(Number(id)); + } + + if (!sinks.some(line => line.includes(CAPTURE_SINK))) { + const id = await this.pactl([ + "load-module", + "module-null-sink", + `sink_name=${CAPTURE_SINK}`, + "sink_properties=device.description=VeskslopCapture" + ]); + this.modules.push(Number(id)); + } + } + + /** Stop children and unload the modules this daemon created. */ + async dispose(): Promise { + this.stopPlayback(); + this.stopCapture(); + if (this.ducked) await this.setDuck(false).catch(() => undefined); + + for (const id of [...this.modules].reverse()) { + await run("pactl", ["unload-module", String(id)]); + } + this.modules = []; + } + + async play(file: string): Promise { + if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available"); + + const binary = process.env.VESKSLOP_FFPLAY ?? "ffplay"; + await new Promise((resolve, reject) => { + const child = spawn(binary, ["-nodisp", "-autoexit", "-loglevel", "error", file], { + env: { ...process.env, PULSE_SINK: SINK }, + stdio: ["ignore", "ignore", "pipe"] + }); + + this.player = child; + let stderr = ""; + + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => { + this.player = undefined; + reject(new AudioUnavailableError(String(error))); + }); + child.on("close", code => { + this.player = undefined; + if (code === 0 || code === null) resolve(); + else reject(new AudioUnavailableError(stderr.trim() || `the player exited with ${code}`)); + }); + }); + } + + stopPlayback(): void { + try { + this.player?.kill("SIGTERM"); + } catch { + // already gone + } + this.player = undefined; + } + + /** Lower the virtual sink volume while other people speak. */ + async setDuck(on: boolean): Promise { + if (this.ducked === on) return; + await this.pactl(["set-sink-volume", SINK, on ? `${DUCK_PERCENT}%` : "100%"]); + this.ducked = on; + } + + async startCapture(file: string): Promise { + if (this.recorder) throw new Error("capture is already running"); + if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available"); + + const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg"; + const child = spawn( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "pulse", + "-i", + `${CAPTURE_SINK}.monitor`, + "-ac", + "1", + "-ar", + "16000", + "-y", + file + ], + { stdio: ["ignore", "ignore", "pipe"] } + ); + + this.recorder = child; + child.on("close", () => { + if (this.recorder === child) this.recorder = undefined; + }); + } + + stopCapture(): void { + const child = this.recorder; + this.recorder = undefined; + try { + child?.kill("SIGINT"); + } catch { + // already gone + } + } + + /** Stream the capture sink as raw 48 kHz mono PCM. Returns a stop function. */ + startMonitor(onData: (chunk: Buffer) => void, onError?: (error: Error) => void): () => void { + if (!this.available) throw new AudioUnavailableError(this.error ?? "audio is not available"); + if (!this.ffmpegAvailable) { + throw new AudioUnavailableError("ffmpeg is missing; install it to hear the call"); + } + + const binary = process.env.VESKSLOP_FFMPEG ?? "ffmpeg"; + const child = spawn( + binary, + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "pulse", + "-i", + `${CAPTURE_SINK}.monitor`, + "-ac", + "1", + "-ar", + "48000", + "-f", + "s16le", + "-" + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + + child.stdout?.on("data", onData); + let stderr = ""; + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => onError?.(new AudioUnavailableError(String(error)))); + child.on("close", code => { + if (code) onError?.(new AudioUnavailableError(stderr.trim() || `ffmpeg exited with ${code}`)); + }); + + return () => { + try { + child.kill("SIGTERM"); + } catch { + // already gone + } + }; + } + + private async pactl(args: string[]): Promise { + const result = await run("pactl", args); + if (result.code !== 0) { + throw new AudioUnavailableError(result.stderr.trim() || `pactl ${args[0]} failed`); + } + return result.stdout.trim(); + } +} diff --git a/poc/src/capture.ts b/poc/src/capture.ts new file mode 100644 index 0000000..a2cfb49 --- /dev/null +++ b/poc/src/capture.ts @@ -0,0 +1,103 @@ +/* + * Veskslop daemon — capture pipeline. + * + * Reads the client output monitor as raw PCM, splits it into utterances with + * the VAD, and writes one WAV per utterance. The caller stores the metadata. + */ + +import { spawn, type ChildProcess } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { pcmToWav, Vad, type VadConfig, type VadSegment } from "./vad.js"; + +export interface CaptureSegment { + file: string; + durationMs: number; + startedAt: string; + endedAt: string; +} + +export interface CapturePipelineOptions { + ffmpeg: string; + target: string; + directory: string; + sessionId: string; + vad: VadConfig; + onSegment: (segment: CaptureSegment) => void; + onError: (error: Error) => void; +} + +export class CapturePipeline { + private child?: ChildProcess; + private vad?: Vad; + private counter = 0; + private stopped = false; + + constructor(private readonly options: CapturePipelineOptions) {} + + start() { + mkdirSync(this.options.directory, { recursive: true }); + this.vad = new Vad(this.options.vad, segment => this.emit(segment)); + + this.child = spawn( + this.options.ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "pulse", + "-i", + this.options.target, + "-ac", + "1", + "-ar", + String(this.options.vad.sampleRate), + "-f", + "s16le", + "-" + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + + let stderr = ""; + this.child.stdout?.on("data", chunk => this.vad?.push(chunk)); + this.child.stderr?.on("data", chunk => (stderr += chunk)); + this.child.on("error", error => this.options.onError(new Error(String(error)))); + this.child.on("close", code => { + if (!this.stopped && code) { + this.options.onError(new Error(stderr.trim() || `ffmpeg exited with ${code}`)); + } + }); + } + + stop() { + this.stopped = true; + this.vad?.flush(); + this.vad = undefined; + + try { + this.child?.kill("SIGTERM"); + } catch { + // already gone + } + this.child = undefined; + } + + private emit(segment: VadSegment) { + this.counter++; + const endedAt = new Date(); + const startedAt = new Date(endedAt.getTime() - segment.durationMs); + const file = join(this.options.directory, `${this.options.sessionId}-${this.counter}.wav`); + + writeFileSync(file, pcmToWav(segment.pcm, this.options.vad?.sampleRate ?? 16000)); + + this.options.onSegment({ + file, + durationMs: segment.durationMs, + startedAt: startedAt.toISOString(), + endedAt: endedAt.toISOString() + }); + } +} diff --git a/poc/src/control-bridge.ts b/poc/src/control-bridge.ts index c4b1168..b2e2b97 100644 --- a/poc/src/control-bridge.ts +++ b/poc/src/control-bridge.ts @@ -15,10 +15,16 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/ import * as z from "zod/v4"; import { AuthService } from "./auth-service.js"; +import { AudioService } from "./audio-service.js"; import { tokenFile } from "./config.js"; +import { dirname, join } from "node:path"; +import { Stt } from "./stt.js"; +import { TranscriptStore } from "./transcript.js"; +import { DEFAULT_VAD } from "./vad.js"; import { CoreClient, coreSocketPath } from "./core-client.js"; import { CoreProcess } from "./core-process.js"; -import { dedupe, getQueryCount, paced, pacedSend, sleep } from "./pacing.js"; +import { dedupe, getQueryCount, paced, pacedSend, pacedVoice, sleep } from "./pacing.js"; +import { PreferencesStore } from "./preferences.js"; import { Puppet } from "./puppet.js"; import { startWebUi } from "./webui.js"; @@ -40,10 +46,11 @@ async function connectLoop(): Promise { const client = await CoreClient.connect(); core = client; - puppet = new Puppet(client); + puppet = new Puppet(client, audio); client.onEvent(event => { void puppet!.enqueueEvent(event); + void audio.onEvent(event); webui.broadcast(event); }); @@ -59,6 +66,7 @@ async function connectLoop(): Promise { coreConnected = true; console.log(`[daemon] connected to the core bridge at ${coreSocketPath}`); webui.broadcast({ type: "CORE_STATUS", coreConnected: true }); + void autoConnectAudio(client); return; } catch (error) { @@ -83,6 +91,81 @@ function requirePuppet(): Puppet { const coreProcess = new CoreProcess({ tokenFile }); +const transcripts = new TranscriptStore( + process.env.VESKSLOP_TRANSCRIPT ?? join(dirname(tokenFile), "transcripts.jsonl") +); +const preferences = new PreferencesStore( + process.env.VESKSLOP_PREFERENCES ?? join(dirname(tokenFile), "preferences.json"), + { legacyFile: process.env.VESKSLOP_STATE_FILE ?? join(dirname(tokenFile), ".veskslop-state.json") } +); +const stt = new Stt(); + +const audio = new AudioService({ + cacheDir: process.env.VESKSLOP_CACHE_DIR, + transcripts, + stt, + autoCapture: process.env.VESKSLOP_CAPTURE_AUTO === "1", + vad: { + ...DEFAULT_VAD, + thresholdDb: Number(process.env.VESKSLOP_VAD_THRESHOLD_DB ?? DEFAULT_VAD.thresholdDb), + minSpeechMs: Number(process.env.VESKSLOP_VAD_MIN_SPEECH_MS ?? DEFAULT_VAD.minSpeechMs), + silenceMs: Number(process.env.VESKSLOP_VAD_SILENCE_MS ?? DEFAULT_VAD.silenceMs), + preRollMs: Number(process.env.VESKSLOP_VAD_PRE_ROLL_MS ?? DEFAULT_VAD.preRollMs), + maxSegmentMs: Number(process.env.VESKSLOP_VAD_MAX_SEGMENT_MS ?? DEFAULT_VAD.maxSegmentMs) + }, + getSelfId: async () => (core ? await core.getSelfId() : undefined), + getVoiceContext: async () => { + if (!core) return {}; + const state = (await core.voiceGet()) as { guildId?: string | null; channelId?: string | null }; + return { guildId: state?.guildId ?? null, channelId: state?.channelId ?? null }; + }, + getSpeakerName: async userId => { + if (!core) return undefined; + const state = (await core.voiceGet(userId)) as { + members?: Array<{ id: string; username?: string }>; + }; + return state?.members?.find(member => member.id === userId)?.username; + }, + onTranscript: event => webui.broadcast(event) +}); + +void audio.init(); + +/** Point the client at the virtual microphone and capture sink when present. */ +async function autoConnectAudio(client: CoreClient): Promise<{ connected: boolean; message: string }> { + if (process.env.VESKSLOP_AUDIO_AUTOCONNECT === "0") { + return { connected: false, message: "auto-connect is disabled" }; + } + + const status = audio.status(); + if (!status.available) { + return { connected: false, message: status.hint ?? status.error ?? "audio is not available" }; + } + + try { + await audio.manager.ensureDevices(); + + const devices = (await client.voiceDevices()) as { + input?: Array<{ id: string; label: string }>; + output?: Array<{ id: string; label: string }>; + }; + + const input = devices.input?.find(device => device.label.includes("VeskslopMic")); + const output = devices.output?.find(device => device.label.includes("VeskslopCapture")); + + if (!input && !output) { + return { connected: false, message: "the client does not see the virtual devices yet" }; + } + + const result = await client.voiceSetDevices(input?.id, output?.id); + console.log("[audio] client devices set:", JSON.stringify(result)); + return { connected: true, message: "the client now uses the virtual devices" }; + } catch (error) { + console.warn("[audio] device auto-connect failed:", (error as Error).message); + return { connected: false, message: (error as Error).message }; + } +} + const auth = new AuthService(tokenFile, update => { webui.broadcast({ type: "AUTH_QR", update }); @@ -100,7 +183,11 @@ const webui = startWebUi({ getPuppet: () => puppet, isCoreConnected: () => coreConnected, coreProcess, - auth + auth, + audio, + transcripts, + preferences, + connectAudioDevices: () => autoConnectAudio(requireCore()) }); coreProcess.onExit(() => { @@ -118,6 +205,9 @@ const READ_ONLY_TOOLS: Record Promise | unknown> get_voice_overview: args => requireCore().voiceOverview(args?.guildId), search_users: args => requireCore().userSearch(args?.query, args?.guildId, args?.limit), list_dms: args => requireCore().dmList(args?.limit), + get_audio_status: () => audio.status(), + list_audio_devices: () => requireCore().voiceDevices(), + get_transcript: args => transcripts.readSegments(args?.limit ?? 100, args?.sessionId), get_context: args => requirePuppet().getContext(args?.channelId, { maxMessages: args?.maxMessages, @@ -268,6 +358,110 @@ function buildServer(): McpServer { async ({ guildId }) => jsonText(await requireCore().voiceOverview(guildId)) ); + server.registerTool( + "voice_join", + { + description: "Join a voice channel as the logged-in user. Joins muted by default.", + inputSchema: { + channelId: z.string().describe("Voice channel ID"), + mute: z.boolean().optional().describe("Join muted. Defaults to true.") + } + }, + async ({ channelId, mute }) => + jsonText(await pacedVoice(() => requireCore().voiceJoin(channelId, mute ?? true))) + ); + + server.registerTool( + "voice_leave", + { + description: "Leave the current voice channel." + }, + async () => jsonText(await pacedVoice(() => requireCore().voiceLeave())) + ); + + server.registerTool( + "voice_move", + { + description: "Move the client to another voice channel.", + inputSchema: { + channelId: z.string().describe("Voice channel ID") + } + }, + async ({ channelId }) => jsonText(await pacedVoice(() => requireCore().voiceMove(channelId))) + ); + + server.registerTool( + "get_audio_status", + { + description: "Audio stack, virtual devices, TTS providers, queue, ducking, and capture state." + }, + async () => jsonText(audio.status()) + ); + + server.registerTool( + "speak", + { + description: "Queue text to speech into the bot's virtual microphone.", + inputSchema: { + text: z.string().min(1).max(1000).describe("Text to speak"), + provider: z.enum(["local", "api"]).optional().describe("TTS provider"), + voice: z.string().optional().describe("Voice name for the provider") + } + }, + async ({ text, provider, voice }) => jsonText(await audio.speak(text, { provider, voice })) + ); + + server.registerTool( + "stop_audio", + { + description: "Stop speech and clear the queue." + }, + async () => { + audio.stop(); + return jsonText(audio.status()); + } + ); + + server.registerTool( + "start_capture", + { + description: "Record the client output sink to a WAV file on the host." + }, + async () => jsonText(await audio.startCapture()) + ); + + server.registerTool( + "stop_capture", + { + description: "Stop the current capture." + }, + async () => { + audio.stopCapture(); + return jsonText(audio.status()); + } + ); + + server.registerTool( + "list_audio_devices", + { + description: "List the client's input and output audio devices." + }, + async () => jsonText(await requireCore().voiceDevices()) + ); + + server.registerTool( + "set_audio_devices", + { + description: "Set the client's input and output audio devices.", + inputSchema: { + inputDeviceId: z.string().optional().describe("Input device ID"), + outputDeviceId: z.string().optional().describe("Output device ID") + } + }, + async ({ inputDeviceId, outputDeviceId }) => + jsonText(await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId)) + ); + server.registerTool( "search_users", { @@ -303,6 +497,18 @@ function buildServer(): McpServer { async ({ limit }) => jsonText(requirePuppet().readActionLog(limit)) ); + server.registerTool( + "get_transcript", + { + description: "Read captured call segments: time, speaker, WAV path, and text when STT is configured.", + inputSchema: { + limit: z.number().int().min(1).max(500).default(100).describe("How many segments"), + sessionId: z.string().optional().describe("Only this capture session") + } + }, + async ({ limit, sessionId }) => jsonText(transcripts.readSegments(limit, sessionId)) + ); + server.registerTool( "run_commands", { @@ -427,10 +633,7 @@ app.listen(PORT, HOST, (error?: Error) => { function shutdown() { core?.close(); - void coreProcess - .dispose(4000) - .catch(() => undefined) - .finally(() => process.exit(0)); + void Promise.allSettled([coreProcess.dispose(4000), audio.dispose()]).finally(() => process.exit(0)); } process.on("SIGINT", shutdown); diff --git a/poc/src/core-client.ts b/poc/src/core-client.ts index b6cf44f..12ef5e0 100644 --- a/poc/src/core-client.ts +++ b/poc/src/core-client.ts @@ -25,8 +25,11 @@ export type CoreEventListener = (event: any) => void; interface Pending { resolve: (value: any) => void; reject: (error: Error) => void; + timer: ReturnType; } +const REQUEST_TIMEOUT_MS = Number(process.env.VESKSLOP_REQUEST_TIMEOUT_MS ?? 30000); + export class CoreClient { private nextId = 1; private readonly pending = new Map(); @@ -50,6 +53,7 @@ export class CoreClient { socket.on("close", () => { for (const request of this.pending.values()) { + clearTimeout(request.timer); request.reject(new CoreError("DISCONNECTED", "core disconnected")); } this.pending.clear(); @@ -93,6 +97,7 @@ export class CoreClient { const request = this.pending.get(message.id); if (!request) return; this.pending.delete(message.id); + clearTimeout(request.timer); if (message.ok) request.resolve(message.result); else request.reject(new CoreError(message.error?.code ?? "UNKNOWN", message.error?.message ?? "core error")); @@ -102,7 +107,24 @@ export class CoreClient { const id = this.nextId++; return new Promise((resolve, reject) => { - this.pending.set(id, { resolve, reject }); + const timer = setTimeout(() => { + if (this.pending.delete(id)) { + reject(new CoreError("TIMEOUT", `the core did not answer ${action}`)); + } + }, REQUEST_TIMEOUT_MS); + + this.pending.set(id, { + resolve: value => { + clearTimeout(timer); + resolve(value); + }, + reject: error => { + clearTimeout(timer); + reject(error); + }, + timer + }); + this.socket.write(JSON.stringify({ id, action, params }) + "\n"); }); } @@ -158,6 +180,37 @@ export class CoreClient { return this.request("voice.overview", { guildId }); } + async voiceJoin(channelId: string, mute = true) { + return this.request("voice.join", { channelId, mute }); + } + + async voiceLeave() { + return this.request("voice.leave"); + } + + async voiceMove(channelId: string) { + return this.request("voice.move", { channelId }); + } + + async voiceUsers(userIds: string[]) { + return this.request("voice.users", { userIds }); + } + + async userStatus(userIds: string[]) { + return this.request("user.status", { userIds }); + } + + async voiceDevices() { + return this.request("voice.devices"); + } + + async voiceSetDevices(inputDeviceId?: string, outputDeviceId?: string) { + return this.request("voice.setDevices", { + ...(inputDeviceId ? { inputDeviceId } : {}), + ...(outputDeviceId ? { outputDeviceId } : {}) + }); + } + async userSearch(query: string, guildId?: string, limit?: number) { return this.request("user.search", { query, diff --git a/poc/src/mock-core.ts b/poc/src/mock-core.ts index c3c233b..cf14933 100644 --- a/poc/src/mock-core.ts +++ b/poc/src/mock-core.ts @@ -94,36 +94,76 @@ const FAKE_HISTORY = Array.from({ length: 8 }, (_, index) => ({ : [] })); +const MOCK_VOICE_CHANNEL = "200000000000000002"; +let botVoiceChannel: string | null = MOCK_VOICE_CHANNEL; + +const STATE_FAIL_MS = Number(process.env.VESKSLOP_MOCK_STATE_FAIL_MS ?? 0); +const startedAt = Date.now(); + +function mockVoiceMembers() { + return [ + { id: "999", username: "mockuser", avatar: null, mute: false, deaf: false }, + { id: "177117296140091393", username: "zzib", avatar: null, mute: false, deaf: false } + ]; +} + function handle(action: string | undefined, params: any) { switch (action) { case "ping": return "pong"; case "state.get": + if (STATE_FAIL_MS && Date.now() - startedAt < STATE_FAIL_MS) { + throw new Error("renderer is not ready"); + } return { user: { id: "999", username: "mockuser", avatar: null }, guilds: FAKE_GUILDS }; case "voice.get": + if (!botVoiceChannel) { + return { guildId: params?.guildId ?? null, channelId: null, channelName: null, members: [] }; + } return { guildId: params?.guildId ?? "100000000000000001", - channelId: "200000000000000002", + channelId: botVoiceChannel, channelName: "voice", - members: [ - { id: params?.userId ?? "999", username: "mockuser", mute: false, deaf: false }, - { id: "177117296140091393", username: "zzib", mute: false, deaf: false } - ] + members: mockVoiceMembers() }; case "voice.overview": return { guildId: params?.guildId, - channels: [ - { - channelId: "200000000000000002", - channelName: "voice", - members: [ - { id: "999", username: "mockuser", mute: false, deaf: false }, - { id: "177117296140091393", username: "zzib", mute: false, deaf: false } - ] - } - ] + channels: botVoiceChannel + ? [{ channelId: botVoiceChannel, channelName: "voice", members: mockVoiceMembers() }] + : [] }; + case "voice.join": + case "voice.move": + botVoiceChannel = params?.channelId ?? null; + return { channelId: params?.channelId, joined: true, muted: params?.mute !== false }; + case "voice.leave": + botVoiceChannel = null; + return { left: true }; + case "voice.devices": + return { + input: [{ id: "mock-mic", label: "VeskslopMic (mock)" }], + output: [{ id: "mock-out", label: "VeskslopCapture (mock)" }] + }; + case "voice.users": + return { + users: (params?.userIds ?? []).map((id: string) => ({ + id, + channelId: id === "999" ? MOCK_VOICE_CHANNEL : null, + channelName: id === "999" ? "voice" : null, + guildId: id === "999" ? "100000000000000001" : null, + members: id === "999" ? mockVoiceMembers() : [] + })) + }; + case "user.status": + return { + users: (params?.userIds ?? []).map((id: string) => ({ + id, + status: id === "999" ? "online" : id === "177117296140091393" ? "idle" : "offline" + })) + }; + case "voice.setDevices": + return { inputDeviceId: params?.inputDeviceId ?? null, outputDeviceId: params?.outputDeviceId ?? null }; case "user.search": { const needle = String(params?.query ?? "") .replace(/^@/, "") @@ -144,8 +184,18 @@ function handle(action: string | undefined, params: any) { name: null, icon: null, lastMessageId: "710000000000000001", + lastMessageAt: new Date(Date.now() - 42 * 60_000).toISOString(), + unread: 2, + mentions: 0, recipients: [ - { id: "177117296140091393", username: "zzib", globalName: "Zzib", avatar: null, bot: false } + { + id: "177117296140091393", + username: "zzib", + globalName: "Zzib", + avatar: null, + status: "idle", + bot: false + } ] }, { @@ -154,9 +204,12 @@ function handle(action: string | undefined, params: any) { name: "oboys + 2", icon: null, lastMessageId: "710000000000000002", + lastMessageAt: new Date(Date.now() - 26 * 60 * 60_000).toISOString(), + unread: 0, + mentions: 3, recipients: [ - { id: "888", username: "testbot", globalName: null, avatar: null, bot: true }, - { id: "999", username: "mockuser", globalName: "Mock User", avatar: null, bot: false } + { id: "888", username: "testbot", globalName: null, avatar: null, status: "offline", bot: true }, + { id: "999", username: "mockuser", globalName: "Mock User", avatar: null, status: "online", bot: false } ] } ] @@ -208,9 +261,15 @@ const server = createServer(socket => { buffer = buffer.slice(newline + 1); if (!line) continue; + let request: any; try { - const request = JSON.parse(line); + request = JSON.parse(line); + } catch { + socket.write(JSON.stringify({ ok: false, error: { code: "INVALID_REQUEST", message: "invalid JSON" } }) + "\n"); + continue; + } + try { if (request.action === "events.subscribe") { subscribers.add(socket); socket.write(JSON.stringify({ id: request.id, ok: true, result: { subscribed: true } }) + "\n"); @@ -225,7 +284,7 @@ const server = createServer(socket => { } catch (error) { const message = error instanceof Error ? error.message : String(error); console.log(`[mock] ${new Date().toISOString()} ERROR ${message}`); - socket.write(JSON.stringify({ ok: false, error: { code: "UNKNOWN_ACTION", message } }) + "\n"); + socket.write(JSON.stringify({ id: request.id, ok: false, error: { code: "UNKNOWN_ACTION", message } }) + "\n"); } } }); diff --git a/poc/src/pacing.ts b/poc/src/pacing.ts index 090220a..16c427a 100644 --- a/poc/src/pacing.ts +++ b/poc/src/pacing.ts @@ -21,7 +21,9 @@ export const PACING = { idleMinMs: 5000, idleMaxMs: 20000, readMinMs: 100, - readMaxMs: 300 + readMaxMs: 300, + voiceMinMs: 250, + voiceMaxMs: 750 }; function randomBetween(min: number, max: number): number { @@ -95,6 +97,23 @@ export function paced(task: () => Promise): Promise { return run; } +/** Serialize a voice action with a short gap. Voice state changes are not sends. */ +export function pacedVoice(task: () => Promise): Promise { + const delay = randomBetween(PACING.voiceMinMs, PACING.voiceMaxMs); + const run = queue.then(async () => { + await sleep(delay); + if (process.env.DEBUG === "1") { + console.error(`[voice] action (+${delay}ms)`); + } + return task(); + }); + queue = run.then( + () => undefined, + () => undefined + ); + return run; +} + export function getQueryCount(): number { return queryCount; } diff --git a/poc/src/preferences.ts b/poc/src/preferences.ts new file mode 100644 index 0000000..daf8c24 --- /dev/null +++ b/poc/src/preferences.ts @@ -0,0 +1,138 @@ +/* + * Veskslop daemon — web UI preferences. + * + * Stored next to the token file, not in the puppet state, so the web UI can + * read and write them while the Discord client is disconnected. + */ + +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +export interface TtsPreferences { + provider: "local" | "api" | null; + language: string | null; + voice: string | null; +} + +export interface FavouritesPreferences { + users: string[]; + channels: string[]; +} + +export interface WebPreferences { + /** null means every guild is visible. An empty array hides all. */ + visibleGuilds: string[] | null; + tts: TtsPreferences; + favourites: FavouritesPreferences; +} + +const DEFAULTS: WebPreferences = { + visibleGuilds: null, + tts: { provider: null, language: null, voice: null }, + favourites: { users: [], channels: [] } +}; + +export interface PreferencesOptions { + /** The old state file that may still hold preferences. */ + legacyFile?: string; +} + +export class PreferencesStore { + private preferences: WebPreferences = structuredClone(DEFAULTS); + + constructor( + private readonly file = process.env.VESKSLOP_PREFERENCES ?? ".veskslop-preferences.json", + options: PreferencesOptions = {} + ) { + try { + if (existsSync(this.file)) { + this.preferences = this.normalize(JSON.parse(readFileSync(this.file, "utf8"))); + } else { + this.migrate(options.legacyFile); + } + } catch (error) { + console.warn("[preferences] could not read", this.file, (error as Error).message); + } + } + + get(): WebPreferences { + return structuredClone(this.preferences); + } + + update(patch: Partial): WebPreferences { + if (patch.visibleGuilds !== undefined) { + this.preferences.visibleGuilds = Array.isArray(patch.visibleGuilds) + ? [...new Set(patch.visibleGuilds.map(String))] + : null; + } + + if (patch.tts !== undefined) { + this.preferences.tts = { + provider: patch.tts?.provider === "api" ? "api" : patch.tts?.provider === "local" ? "local" : null, + language: patch.tts?.language ? String(patch.tts.language).slice(0, 16) : null, + voice: patch.tts?.voice ? String(patch.tts.voice).slice(0, 64) : null + }; + } + + if (patch.favourites !== undefined) { + this.preferences.favourites = { + users: Array.isArray(patch.favourites?.users) + ? [...new Set(patch.favourites.users.map(String))].slice(0, 100) + : [], + channels: Array.isArray(patch.favourites?.channels) + ? [...new Set(patch.favourites.channels.map(String))].slice(0, 100) + : [] + }; + } + + this.save(); + return this.get(); + } + + private migrate(legacyFile?: string) { + if (!legacyFile || !existsSync(legacyFile)) return; + + try { + const legacy = JSON.parse(readFileSync(legacyFile, "utf8")); + if (legacy?.preferences) { + this.preferences = this.normalize(legacy.preferences); + this.save(); + console.log("[preferences] migrated from", legacyFile); + } + } catch { + // nothing to migrate + } + } + + private normalize(raw: any): WebPreferences { + const provider = raw?.tts?.provider; + + return { + visibleGuilds: Array.isArray(raw?.visibleGuilds) + ? [...new Set(raw.visibleGuilds.map((id: unknown) => String(id)))] + : null, + tts: { + provider: provider === "api" || provider === "local" ? provider : null, + language: raw?.tts?.language ? String(raw.tts.language).slice(0, 16) : null, + voice: raw?.tts?.voice ? String(raw.tts.voice).slice(0, 64) : null + }, + favourites: { + users: Array.isArray(raw?.favourites?.users) + ? [...new Set(raw.favourites.users.map((id: unknown) => String(id)))].slice(0, 100) + : [], + channels: Array.isArray(raw?.favourites?.channels) + ? [...new Set(raw.favourites.channels.map((id: unknown) => String(id)))].slice(0, 100) + : [] + } + }; + } + + private save() { + try { + mkdirSync(dirname(this.file), { recursive: true }); + writeFileSync(this.file, JSON.stringify(this.preferences, null, 4)); + } catch (error) { + console.error("[preferences] write failed:", error); + } + } +} diff --git a/poc/src/puppet.ts b/poc/src/puppet.ts index 3abda22..cb40f19 100644 --- a/poc/src/puppet.ts +++ b/poc/src/puppet.ts @@ -99,6 +99,12 @@ export interface PuppetTransport { reactionAdd(channelId: string, messageId: string, emoji: string): Promise; } +/** The audio actions a chat command can trigger. */ +export interface PuppetAudio { + speak(text: string, options?: { provider?: "local" | "api"; voice?: string }): Promise; + stop(): void; +} + interface PollState { channelId: string; authorId: string; @@ -142,7 +148,10 @@ export class Puppet { private selfId?: string; private queue: Promise = Promise.resolve(); - constructor(private readonly core: PuppetTransport) { + constructor( + private readonly core: PuppetTransport, + private readonly audio?: PuppetAudio + ) { try { const state = JSON.parse(readFileSync(STATE_FILE, "utf8")) as PersistedState; if (state.ownerId) this.ownerId = state.ownerId; @@ -159,8 +168,7 @@ export class Puppet { if (state.config) { this.config = { - contextMessages: clamp(state.config.contextMessages ?? DEFAULT_CONFIG.contextMessages, 1, 50), - contextChars: clamp(state.config.contextChars ?? DEFAULT_CONFIG.contextChars, 200, 20000), + contextMessages: clamp(state.config.contextMessages ?? DEFAULT_CONFIG.contextMessages, 1, 50), contextChars: clamp(state.config.contextChars ?? DEFAULT_CONFIG.contextChars, 200, 20000), fetchCooldownSeconds: clamp( state.config.fetchCooldownSeconds ?? DEFAULT_CONFIG.fetchCooldownSeconds, 0, @@ -181,6 +189,7 @@ export class Puppet { } }; } + } catch { // first run: defaults apply } @@ -263,7 +272,11 @@ export class Puppet { writeFileSync( STATE_FILE, JSON.stringify( - { ownerId: this.ownerId, whitelist: this.whitelist, config: this.config }, + { + ownerId: this.ownerId, + whitelist: this.whitelist, + config: this.config + }, null, 4 ) @@ -446,6 +459,8 @@ export class Puppet { if (command?.name === "whitelist") { await this.handleWhitelistCommand(command, author.id, tier, channelId); + } else if (command?.name === "speak") { + await this.handleSpeakCommand(command, author.id, channelId); } else if (POLL_ENABLED && command?.name === "poll") { await this.handlePollCommand(author.id, channelId, text); } @@ -541,6 +556,51 @@ export class Puppet { } } + /* Text to speech */ + + private async handleSpeakCommand(command: ParsedCommand, authorId: string, channelId: string) { + if (!this.audio) { + return this.respond(authorId, channelId, "speak", "Voice audio is not available."); + } + + const text = command.args.join(" ").trim(); + if (!text || text === "help") { + return this.respond( + authorId, + channelId, + "speak", + ["speak — text to speech.", "Usage:", " speak ", " speak stop"].join("\n") + ); + } + + if (text === "stop") { + this.audio.stop(); + this.actionLog.append({ type: "command", command: "speak stop", requesterId: authorId, result: "ok" }); + return this.respond(authorId, channelId, "speak", "Stopped."); + } + + try { + await this.audio.speak(text, {}); + this.actionLog.append({ + type: "command", + command: "speak", + requesterId: authorId, + result: "queued", + chars: text.length + }); + return this.respond(authorId, channelId, "speak", "Speaking."); + } catch (error) { + this.actionLog.append({ + type: "command", + command: "speak", + requesterId: authorId, + result: "error", + error: String(error) + }); + return this.respond(authorId, channelId, "speak", `Could not speak: ${(error as Error).message}`); + } + } + /* Polls */ private async handlePollCommand(authorId: string, channelId: string, text: string) { diff --git a/poc/src/stt.ts b/poc/src/stt.ts new file mode 100644 index 0000000..0b77a5d --- /dev/null +++ b/poc/src/stt.ts @@ -0,0 +1,135 @@ +/* + * Veskslop daemon — speech to text. + * + * Providers, in order: VESKSLOP_STT_CMD, a local Vosk model + * (VESKSLOP_VOSK_MODEL or a `vosk-model*` folder under poc/models), and an + * HTTP API. The local command receives the WAV path ({wav}) and prints text. + */ + +import { spawn } from "node:child_process"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export class SttError extends Error {} + +export interface SttStatus { + local: boolean; + api: boolean; + engine: string | null; + hint?: string; +} + +export class Stt { + private voskModel?: string; + + get localCommand(): string | undefined { + return process.env.VESKSLOP_STT_CMD; + } + + get apiUrl(): string | undefined { + return process.env.VESKSLOP_STT_API_URL; + } + + async detect(): Promise { + this.voskModel = this.findVoskModel(); + } + + status(): SttStatus { + const local = !!(this.localCommand || this.voskModel); + + return { + local, + api: !!this.apiUrl, + engine: this.localCommand ? "command" : this.voskModel ? "vosk" : null, + hint: + local || this.apiUrl + ? undefined + : "Install vosk (pip install vosk) and set VESKSLOP_VOSK_MODEL, or set VESKSLOP_STT_CMD." + }; + } + + /** Returns undefined when no provider is configured. */ + async transcribe(wavPath: string): Promise { + if (this.localCommand) return this.viaLocalCommand(wavPath, this.localCommand); + + if (this.voskModel) { + const script = fileURLToPath(new URL("../scripts/stt-vosk.py", import.meta.url)); + return this.viaLocalCommand(wavPath, `python3 ${script} {wav}`, { + VESKSLOP_VOSK_MODEL: this.voskModel + }); + } + + if (this.apiUrl) return this.viaApi(wavPath); + return undefined; + } + + private findVoskModel(): string | undefined { + const explicit = process.env.VESKSLOP_VOSK_MODEL; + if (explicit) return explicit; + + const base = process.env.VESKSLOP_VOSK_DIR ?? fileURLToPath(new URL("../models", import.meta.url)); + + try { + const model = readdirSync(base, { withFileTypes: true }).find( + entry => entry.isDirectory() && entry.name.startsWith("vosk-model") + ); + return model ? join(base, model.name) : undefined; + } catch { + return undefined; + } + } + + private async viaLocalCommand( + wavPath: string, + command: string, + extraEnv: Record = {} + ): Promise { + if (!command.includes("{wav}")) throw new SttError("the STT command must contain {wav}"); + + const parts = command + .split(/\s+/) + .filter(Boolean) + .map(part => part.replace("{wav}", wavPath)); + + return new Promise((resolve, reject) => { + const child = spawn(parts[0], parts.slice(1), { + env: { ...process.env, ...extraEnv }, + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", chunk => (stdout += chunk)); + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => reject(new SttError(String(error)))); + child.on("close", code => { + if (code === 0) resolve(stdout.trim()); + else reject(new SttError(stderr.trim() || `the STT command exited with ${code}`)); + }); + }); + } + + private async viaApi(wavPath: string): Promise { + const response = await fetch(this.apiUrl!, { + method: "POST", + headers: { + "Content-Type": "audio/wav", + ...(process.env.VESKSLOP_STT_API_KEY + ? { Authorization: `Bearer ${process.env.VESKSLOP_STT_API_KEY}` } + : {}) + }, + body: readFileSync(wavPath) + }); + + if (!response.ok) throw new SttError(`the STT API returned ${response.status}`); + + const type = response.headers.get("content-type") ?? ""; + if (type.includes("json")) { + const data: any = await response.json(); + return String(data.text ?? data.transcription ?? "").trim(); + } + + return (await response.text()).trim(); + } +} diff --git a/poc/src/test-audio.ts b/poc/src/test-audio.ts new file mode 100644 index 0000000..53b6604 --- /dev/null +++ b/poc/src/test-audio.ts @@ -0,0 +1,123 @@ +/* + * Veskslop — unit tests for the capture pipeline logic. + * + * Usage: pnpm test:audio + */ + +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Stt } from "./stt.js"; +import { TranscriptStore } from "./transcript.js"; +import { DEFAULT_VAD, pcmToWav, rms, Vad, type VadConfig } from "./vad.js"; + +// rms +{ + const silence = Buffer.alloc(640); + const loud = Buffer.alloc(640); + for (let index = 0; index < 320; index++) { + loud.writeInt16LE(Math.round(Math.sin(index / 5) * 0.4 * 32767), index * 2); + } + + assert.equal(rms(silence), 0); + assert.ok(rms(loud) > 0.2, `rms ${rms(loud)}`); +} + +// VAD segments speech, keeps the pre-roll, and buffers partial frames +{ + const config: VadConfig = { ...DEFAULT_VAD, minSpeechMs: 40, silenceMs: 100, preRollMs: 40, maxSegmentMs: 5000 }; + const frame = (loud: boolean) => { + const buffer = Buffer.alloc(((config.sampleRate * config.frameMs) / 1000) * 2); + if (loud) { + for (let index = 0; index < buffer.length / 2; index++) { + buffer.writeInt16LE(Math.round(Math.sin(index / 4) * 0.5 * 32767), index * 2); + } + } + return buffer; + }; + + const durations: number[] = []; + const vad = new Vad(config, segment => durations.push(segment.durationMs)); + + for (let index = 0; index < 25; index++) vad.push(frame(false)); + for (let index = 0; index < 25; index++) vad.push(frame(true)); + for (let index = 0; index < 10; index++) vad.push(frame(false)); + + assert.equal(durations.length, 1); + assert.ok(durations[0] >= 500 && durations[0] <= 700, `duration ${durations[0]}`); + + const splitDurations: number[] = []; + const split = new Vad(config, segment => splitDurations.push(segment.durationMs)); + const pcm = Buffer.concat([ + ...Array(25).fill(frame(false)), + ...Array(25).fill(frame(true)), + ...Array(10).fill(frame(false)) + ]); + for (let offset = 0; offset < pcm.length; offset += 333) { + split.push(pcm.subarray(offset, offset + 333)); + } + assert.equal(splitDurations.length, 1); +} + +// WAV header +{ + const wav = pcmToWav(Buffer.alloc(320), 16000); + assert.equal(wav.length, 44 + 320); + assert.equal(wav.toString("ascii", 0, 4), "RIFF"); + assert.equal(wav.toString("ascii", 8, 12), "WAVE"); + assert.equal(wav.readUInt32LE(24), 16000); +} + +// Transcript store: segments gain text later +{ + const dir = mkdtempSync(join(tmpdir(), "veskslop-test-")); + const store = new TranscriptStore(join(dir, "transcript.jsonl")); + const session = store.startSession({ guildId: "g1", channelId: "c1" }); + + store.addSegment({ + id: "s1", + sessionId: session.id, + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + durationMs: 1200, + speakerId: "u1", + speakerName: "Zibbe", + wav: "/tmp/x.wav", + text: null + }); + store.setSegmentText("s1", "hello world"); + store.endSession(session.id); + + const sessions = store.readSessions(); + assert.equal(sessions.length, 1); + assert.ok(sessions[0].endedAt); + + const segments = store.readSegments(); + assert.equal(segments.length, 1); + assert.equal(segments[0].text, "hello world"); + assert.equal(segments[0].speakerName, "Zibbe"); + assert.deepEqual(store.stats(), { sessions: 1, segments: 1, transcribed: 1 }); + + rmSync(dir, { recursive: true, force: true }); +} + +// STT local command +{ + const dir = mkdtempSync(join(tmpdir(), "veskslop-stt-")); + const wav = join(dir, "a.wav"); + writeFileSync(wav, pcmToWav(Buffer.alloc(320), 16000)); + + const script = join(dir, "fake-stt.js"); + writeFileSync(script, 'process.stdout.write("hello from stt\\n");'); + + process.env.VESKSLOP_STT_CMD = `node ${script} {wav}`; + const stt = new Stt(); + assert.equal(await stt.transcribe(wav), "hello from stt"); + delete process.env.VESKSLOP_STT_CMD; + + rmSync(dir, { recursive: true, force: true }); +} + +console.log("all audio tests passed"); diff --git a/poc/src/test-trigger.ts b/poc/src/test-trigger.ts index 423c80b..fa0425c 100644 --- a/poc/src/test-trigger.ts +++ b/poc/src/test-trigger.ts @@ -40,7 +40,15 @@ const reactionEvent = { emoji: { name: process.env.TEST_REACTION_EMOJI ?? "1️⃣", id: null } }; -const event = isReaction ? reactionEvent : messageEvent; +const speakingUser = process.env.TEST_TRIGGER_SPEAKING; +const speakingEvent = { + type: "VOICE_SPEAKING", + userId: speakingUser ?? "999", + speaking: process.env.TEST_TRIGGER_SPEAKING_OFF !== "1", + channelId: process.env.TEST_TRIGGER_CHANNEL ?? "200000000000000002" +}; + +const event = isReaction ? reactionEvent : speakingUser ? speakingEvent : messageEvent; await new Promise((resolve, reject) => { const socket = net.connect(coreSocketPath); diff --git a/poc/src/tls.ts b/poc/src/tls.ts new file mode 100644 index 0000000..cda2104 --- /dev/null +++ b/poc/src/tls.ts @@ -0,0 +1,75 @@ +/* + * Veskslop daemon — self-signed TLS for the web UI. + * + * Browser media APIs need a secure context. The daemon generates one + * self-signed certificate (openssl) covering localhost and the host's LAN + * addresses, and serves the UI over HTTPS. WEBUI_TLS=0 disables this. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync } from "node:fs"; +import { networkInterfaces } from "node:os"; +import { join } from "node:path"; + +export interface TlsPaths { + key: string; + cert: string; +} + +function localAddresses(): string[] { + const names = ["DNS:localhost", "IP:127.0.0.1"]; + + for (const list of Object.values(networkInterfaces())) { + for (const info of list ?? []) { + if (info.family === "IPv4" && !info.internal) names.push(`IP:${info.address}`); + } + } + + return names; +} + +export function ensureTls(storeDir: string): TlsPaths | undefined { + if (process.env.WEBUI_TLS === "0") return undefined; + + const keyOverride = process.env.WEBUI_TLS_KEY; + const certOverride = process.env.WEBUI_TLS_CERT; + if (keyOverride && certOverride) { + return existsSync(keyOverride) && existsSync(certOverride) + ? { key: keyOverride, cert: certOverride } + : undefined; + } + + const dir = join(storeDir, "tls"); + const key = join(dir, "key.pem"); + const cert = join(dir, "cert.pem"); + if (existsSync(key) && existsSync(cert)) return { key, cert }; + + try { + mkdirSync(dir, { recursive: true }); + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + key, + "-out", + cert, + "-days", + "825", + "-subj", + "/CN=veskslop", + "-addext", + `subjectAltName=${localAddresses().join(",")}` + ], + { stdio: "ignore" } + ); + return { key, cert }; + } catch (error) { + console.warn("[webui] self-signed certificate generation failed:", (error as Error).message); + return undefined; + } +} diff --git a/poc/src/transcript.ts b/poc/src/transcript.ts new file mode 100644 index 0000000..bbd24cd --- /dev/null +++ b/poc/src/transcript.ts @@ -0,0 +1,146 @@ +/* + * Veskslop daemon — transcript store. + * + * Sessions and segments append to a JSONL file. Text can arrive after a + * segment, so a text update appends a patch line instead of rewriting the file. + */ + +import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync } from "node:fs"; +import { dirname } from "node:path"; + +const MAX_BYTES = Number(process.env.VESKSLOP_TRANSCRIPT_MAX_BYTES ?? 20 * 1024 * 1024); + +export interface TranscriptSession { + id: string; + startedAt: string; + endedAt?: string; + guildId?: string | null; + channelId?: string | null; +} + +export interface TranscriptSegment { + id: string; + sessionId: string; + startedAt: string; + endedAt: string; + durationMs: number; + speakerId?: string | null; + speakerName?: string | null; + wav?: string | null; + text?: string | null; +} + +export class TranscriptStore { + private counter = 0; + + constructor(private readonly file = process.env.VESKSLOP_TRANSCRIPT ?? ".veskslop-transcripts.jsonl") { + try { + mkdirSync(dirname(this.file), { recursive: true }); + } catch { + // the current directory works + } + } + + get path(): string { + return this.file; + } + + startSession(meta: { guildId?: string | null; channelId?: string | null } = {}): TranscriptSession { + const session: TranscriptSession = { + id: this.nextId("session"), + startedAt: new Date().toISOString(), + guildId: meta.guildId ?? null, + channelId: meta.channelId ?? null + }; + this.append({ type: "session", session }); + return session; + } + + endSession(id: string) { + this.append({ type: "session-end", id, endedAt: new Date().toISOString() }); + } + + addSegment(segment: TranscriptSegment) { + this.append({ type: "segment", segment }); + } + + setSegmentText(id: string, text: string) { + this.append({ type: "segment-text", id, text }); + } + + readSessions(limit = 20): TranscriptSession[] { + const sessions = new Map(); + + for (const line of this.lines()) { + if (line.type === "session" && line.session) sessions.set(line.session.id, line.session); + if (line.type === "session-end") { + const session = sessions.get(line.id); + if (session) session.endedAt = line.endedAt; + } + } + + return [...sessions.values()].slice(-limit); + } + + readSegments(limit = 100, sessionId?: string): TranscriptSegment[] { + const segments = new Map(); + + for (const line of this.lines()) { + if (line.type === "segment" && line.segment) { + if (!sessionId || line.segment.sessionId === sessionId) segments.set(line.segment.id, line.segment); + } + if (line.type === "segment-text") { + const segment = segments.get(line.id); + if (segment) segment.text = line.text; + } + } + + return [...segments.values()].slice(-limit); + } + + stats(): { sessions: number; segments: number; transcribed: number } { + const sessions = this.readSessions(1000); + const segments = this.readSegments(10000); + return { + sessions: sessions.length, + segments: segments.length, + transcribed: segments.filter(segment => !!segment.text).length + }; + } + + private lines(): any[] { + try { + if (!existsSync(this.file)) return []; + return readFileSync(this.file, "utf8") + .split("\n") + .filter(Boolean) + .map(line => { + try { + return JSON.parse(line); + } catch { + return {}; + } + }); + } catch { + return []; + } + } + + private append(entry: Record & { type: string }) { + const line = JSON.stringify({ timestamp: new Date().toISOString(), ...entry }) + "\n"; + + try { + if (existsSync(this.file) && statSync(this.file).size + line.length > MAX_BYTES) { + renameSync(this.file, `${this.file}.1`); + } + appendFileSync(this.file, line); + } catch (error) { + console.error("[transcript] write failed:", error); + } + } + + private nextId(prefix: string): string { + this.counter++; + return `${prefix}-${Date.now()}-${this.counter}`; + } +} diff --git a/poc/src/tts.ts b/poc/src/tts.ts new file mode 100644 index 0000000..623c574 --- /dev/null +++ b/poc/src/tts.ts @@ -0,0 +1,225 @@ +/* + * Veskslop daemon — text to speech. + * + * Providers: a local engine (espeak-ng when installed, or VESKSLOP_TTS_CMD) + * and an HTTP API. The local command receives an output path ({out}), a text + * file ({text_file}), and optionally a voice ({voice}). + */ + +import { spawn } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export class TtsError extends Error {} + +export type TtsProvider = "local" | "api"; + +export interface TtsOptions { + provider?: TtsProvider; + voice?: string; +} + +export interface TtsStatus { + local: boolean; + api: boolean; + defaultProvider: TtsProvider; + engine: string | null; +} + +export interface TtsVoice { + id: string; + name: string; + language: string; + gender: "male" | "female" | null; +} + +interface CommandResult { + code: number | null; + stdout: string; + stderr: string; +} + +function run(command: string, args: string[]): Promise { + return new Promise(resolve => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", chunk => (stdout += chunk)); + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => resolve({ code: null, stdout, stderr: String(error) })); + child.on("close", code => resolve({ code, stdout, stderr })); + }); +} + +function parseVoices(output: string): TtsVoice[] { + const lines = output.split("\n"); + const header = lines[0] ?? ""; + const languageIndex = header.indexOf("Language"); + const ageIndex = header.indexOf("Age/Gender"); + const nameIndex = header.indexOf("VoiceName"); + const fileIndex = header.indexOf("File"); + + if (languageIndex < 0 || ageIndex < 0 || nameIndex < 0 || fileIndex < 0) return []; + + const voices: TtsVoice[] = []; + for (const line of lines.slice(1)) { + const language = line.slice(languageIndex, ageIndex).trim(); + const ageGender = line.slice(ageIndex, nameIndex).trim(); + const name = line.slice(nameIndex, fileIndex).trim(); + if (!language || !name) continue; + + voices.push({ + id: language, + name, + language, + gender: ageGender.endsWith("F") ? "female" : ageGender.endsWith("M") ? "male" : null + }); + } + + return voices; +} + +export class Tts { + private readonly dir: string; + private counter = 0; + private espeak = false; + private voiceCache?: TtsVoice[]; + + constructor(cacheDir?: string) { + const base = cacheDir ?? process.env.VESKSLOP_CACHE_DIR ?? join(tmpdir(), "veskslop-cache"); + this.dir = join(base, "audio"); + try { + mkdirSync(this.dir, { recursive: true }); + } catch { + // the fallback directory works + } + } + + get localCommand(): string | undefined { + return process.env.VESKSLOP_TTS_CMD; + } + + get apiUrl(): string | undefined { + return process.env.VESKSLOP_TTS_API_URL; + } + + get directory(): string { + return this.dir; + } + + async detect(): Promise { + const result = await run("espeak-ng", ["--version"]); + this.espeak = result.code === 0; + } + + status(): TtsStatus { + const local = !!this.localCommand || this.espeak; + const override = process.env.VESKSLOP_TTS_PROVIDER; + + return { + local, + api: !!this.apiUrl, + defaultProvider: override === "local" || override === "api" ? override : local ? "local" : "api", + engine: this.localCommand ? "command" : this.espeak ? "espeak-ng" : null + }; + } + + async voices(): Promise { + if (this.voiceCache) return this.voiceCache; + if (!this.espeak) return (this.voiceCache = []); + + const result = await run("espeak-ng", ["--voices"]); + this.voiceCache = result.code === 0 ? parseVoices(result.stdout) : []; + return this.voiceCache; + } + + async synthesize(text: string, options: TtsOptions = {}): Promise { + const provider = options.provider ?? this.status().defaultProvider; + return provider === "api" ? this.viaApi(text, options) : this.viaLocalCommand(text, options); + } + + private async viaLocalCommand(text: string, options: TtsOptions): Promise { + const command = this.localCommand; + if (!command && !this.espeak) { + throw new TtsError("no local TTS engine is available (install espeak-ng or set VESKSLOP_TTS_CMD)"); + } + if (command && !command.includes("{out}")) { + throw new TtsError("VESKSLOP_TTS_CMD must contain {out}"); + } + + const out = this.nextPath("wav"); + const textFile = this.nextPath("txt"); + writeFileSync(textFile, text); + + const parts = command + ? command + .split(/\s+/) + .filter(Boolean) + .map(part => + part + .replace("{out}", out) + .replace("{text_file}", textFile) + .replace("{voice}", options.voice ?? "") + ) + : ["espeak-ng", ...(options.voice ? ["-v", options.voice] : []), "-w", out, "-f", textFile]; + + const args = parts.filter(Boolean); + + await new Promise((resolve, reject) => { + const child = spawn(args[0], args.slice(1), { + env: { ...process.env, VESKSLOP_TTS_VOICE: options.voice ?? "" }, + stdio: ["ignore", "ignore", "pipe"] + }); + let stderr = ""; + + child.stderr?.on("data", chunk => (stderr += chunk)); + child.on("error", error => reject(new TtsError(String(error)))); + child.on("close", code => { + if (code === 0) resolve(); + else reject(new TtsError(stderr.trim() || `the TTS command exited with ${code}`)); + }); + }); + + return out; + } + + private async viaApi(text: string, options: TtsOptions): Promise { + const url = this.apiUrl; + if (!url) throw new TtsError("VESKSLOP_TTS_API_URL is not set"); + + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(process.env.VESKSLOP_TTS_API_KEY + ? { Authorization: `Bearer ${process.env.VESKSLOP_TTS_API_KEY}` } + : {}) + }, + body: JSON.stringify({ text, ...(options.voice ? { voice: options.voice } : {}) }) + }); + + if (!response.ok) throw new TtsError(`the TTS API returned ${response.status}`); + + const buffer = Buffer.from(await response.arrayBuffer()); + const type = response.headers.get("content-type") ?? ""; + const extension = + type.includes("mpeg") || type.includes("mp3") + ? "mp3" + : type.includes("ogg") + ? "ogg" + : type.includes("wav") + ? "wav" + : "mp3"; + + const out = this.nextPath(extension); + writeFileSync(out, buffer); + return out; + } + + private nextPath(extension: string): string { + this.counter++; + return join(this.dir, `tts-${Date.now()}-${this.counter}.${extension}`); + } +} diff --git a/poc/src/vad.ts b/poc/src/vad.ts new file mode 100644 index 0000000..9973215 --- /dev/null +++ b/poc/src/vad.ts @@ -0,0 +1,165 @@ +/* + * Veskslop daemon — voice activity detection and WAV writing (pure). + * + * The capture pipeline feeds 16-bit mono PCM frames into the detector. It emits + * one segment per utterance: speech plus a small pre-roll and a trailing gap. + */ + +export interface VadConfig { + sampleRate: number; + frameMs: number; + /** Below this level a frame counts as silence. */ + thresholdDb: number; + /** Speech must last this long before a segment opens. */ + minSpeechMs: number; + /** Silence for this long closes an open segment. */ + silenceMs: number; + /** Audio kept from before the speech started. */ + preRollMs: number; + /** A segment is force-closed at this length. */ + maxSegmentMs: number; +} + +export const DEFAULT_VAD: VadConfig = { + sampleRate: 16000, + frameMs: 20, + thresholdDb: -35, + minSpeechMs: 120, + silenceMs: 600, + preRollMs: 300, + maxSegmentMs: 30000 +}; + +export interface VadSegment { + pcm: Buffer; + durationMs: number; +} + +export function rms(frame: Buffer): number { + if (frame.length < 2) return 0; + + let sum = 0; + const samples = Math.floor(frame.length / 2); + for (let index = 0; index < samples; index++) { + const sample = frame.readInt16LE(index * 2) / 32768; + sum += sample * sample; + } + + return Math.sqrt(sum / samples); +} + +export function pcmToWav(pcm: Buffer, sampleRate: number): Buffer { + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(1, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(sampleRate * 2, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + return Buffer.concat([header, pcm]); +} + +export class Vad { + private readonly frameBytes: number; + private readonly threshold: number; + private readonly minSpeechFrames: number; + private readonly silenceFrameLimit: number; + private readonly preRollFrameLimit: number; + private readonly maxSegmentFrames: number; + + private pending: Buffer = Buffer.alloc(0); + private preRoll: Buffer[] = []; + private segment: Buffer[] = []; + private speechFrames = 0; + private silenceFrames = 0; + private open = false; + + constructor( + private readonly config: VadConfig, + private readonly onSegment: (segment: VadSegment) => void + ) { + this.frameBytes = Math.round((config.sampleRate * config.frameMs) / 1000) * 2; + this.threshold = 10 ** (config.thresholdDb / 20); + this.minSpeechFrames = Math.max(1, Math.ceil(config.minSpeechMs / config.frameMs)); + this.silenceFrameLimit = Math.max(1, Math.ceil(config.silenceMs / config.frameMs)); + this.preRollFrameLimit = Math.max(0, Math.ceil(config.preRollMs / config.frameMs)); + this.maxSegmentFrames = Math.max(1, Math.ceil(config.maxSegmentMs / config.frameMs)); + } + + get inSpeech(): boolean { + return this.open; + } + + push(chunk: Buffer) { + this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk; + + while (this.pending.length >= this.frameBytes) { + const frame = this.pending.subarray(0, this.frameBytes); + this.pending = this.pending.subarray(this.frameBytes); + this.consumeFrame(frame); + } + } + + /** Close an open segment. Call when the capture stops. */ + flush() { + if (!this.open) return; + this.closeSegment(); + } + + private consumeFrame(frame: Buffer) { + const loud = rms(frame) >= this.threshold; + + if (!this.open) { + if (this.preRollFrameLimit > 0) { + this.preRoll.push(frame); + if (this.preRoll.length > this.preRollFrameLimit) this.preRoll.shift(); + } + + if (loud) { + this.speechFrames++; + if (this.speechFrames >= this.minSpeechFrames) { + this.open = true; + this.segment = [...this.preRoll]; + this.preRoll = []; + this.silenceFrames = 0; + } + } else { + this.speechFrames = 0; + } + + return; + } + + this.segment.push(frame); + + if (loud) { + this.silenceFrames = 0; + } else { + this.silenceFrames++; + } + + if (this.silenceFrames >= this.silenceFrameLimit || this.segment.length >= this.maxSegmentFrames) { + this.closeSegment(); + } + } + + private closeSegment() { + const frames = this.silenceFrames > 0 ? this.segment.slice(0, -this.silenceFrames) : this.segment; + this.open = false; + this.speechFrames = 0; + this.silenceFrames = 0; + this.segment = []; + + if (!frames.length) return; + + const pcm = Buffer.concat(frames); + this.onSegment({ pcm, durationMs: (frames.length * this.config.frameMs) }); + } +} diff --git a/poc/src/webui.ts b/poc/src/webui.ts index f5b171c..36a996c 100644 --- a/poc/src/webui.ts +++ b/poc/src/webui.ts @@ -8,14 +8,57 @@ */ import express, { type Request, type Response } from "express"; +import { readFileSync } from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import { dirname } from "node:path"; import { fileURLToPath } from "node:url"; import QRCode from "qrcode"; +import { WebSocketServer } from "ws"; import type { AuthService, AuthStatus } from "./auth-service.js"; +import type { AudioService } from "./audio-service.js"; import { CoreError, type CoreClient } from "./core-client.js"; import type { CoreProcess } from "./core-process.js"; -import { dedupe, getQueryCount, paced, pacedSend } from "./pacing.js"; +import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js"; +import type { PreferencesStore, WebPreferences } from "./preferences.js"; import type { Puppet } from "./puppet.js"; +import { ensureTls } from "./tls.js"; +import type { TranscriptStore } from "./transcript.js"; +import type { TtsVoice } from "./tts.js"; + +const PINNED_LANGUAGES = ["en", "sv", "de", "da", "nb", "no", "ru", "chr"]; +const LANGUAGE_NAMES: Record = { + en: "English", + sv: "Swedish", + de: "German", + da: "Danish", + nb: "Norwegian", + no: "Norwegian", + ru: "Russian", + chr: "Cherokee" +}; + +function groupVoices(voices: TtsVoice[]) { + const groups = new Map(); + + for (const voice of voices) { + const base = voice.language.split("-")[0]; + const group = groups.get(base) ?? { id: base, name: LANGUAGE_NAMES[base] ?? voice.language, voices: [] }; + group.voices.push(voice); + groups.set(base, group); + } + + const all = [...groups.values()]; + const pinned = PINNED_LANGUAGES.map(id => all.find(group => group.id === id)).filter( + (group): group is { id: string; name: string; voices: TtsVoice[] } => !!group + ); + const rest = all + .filter(group => !PINNED_LANGUAGES.includes(group.id)) + .sort((a, b) => a.name.localeCompare(b.name)); + + return [...pinned, ...rest]; +} const HOST = process.env.WEBUI_HOST ?? "0.0.0.0"; const PORT = Number(process.env.WEBUI_PORT ?? 8789); @@ -26,6 +69,10 @@ export interface WebUiDeps { isCoreConnected: () => boolean; coreProcess: CoreProcess; auth: AuthService; + audio: AudioService; + transcripts: TranscriptStore; + preferences: PreferencesStore; + connectAudioDevices: () => Promise<{ connected: boolean; message: string }>; } export interface WebUiHandle { @@ -54,9 +101,84 @@ async function qrDataUrl(status: AuthStatus): Promise { return QRCode.toDataURL(session.url, { margin: 1, width: 280 }); } +function stateDir(): string { + const stateFile = + process.env.VESKSLOP_STATE_FILE ?? fileURLToPath(new URL("../.veskslop-state.json", import.meta.url)); + return dirname(stateFile); +} + +/** Live browser audio: the call output streamed as raw PCM over a WebSocket. */ +function startAudioSocket(server: http.Server | https.Server, deps: WebUiDeps): void { + const wss = new WebSocketServer({ server, path: "/audio" }); + + wss.on("connection", socket => { + let stop: (() => void) | undefined; + + const send = (message: unknown) => { + if (socket.readyState === socket.OPEN) socket.send(JSON.stringify(message)); + }; + + socket.on("message", raw => { + let message: any; + try { + message = JSON.parse(raw.toString()); + } catch { + return; + } + + if (message.type !== "listen") return; + + if (!message.on) { + stop?.(); + stop = undefined; + send({ type: "status", listening: false }); + return; + } + + if (stop) return; + + try { + const status = deps.audio.status(); + if (!status.available) { + throw new Error(status.hint ?? status.error ?? "audio is not available"); + } + if (!status.ffmpeg) { + throw new Error("ffmpeg is missing; install it to hear the call"); + } + + stop = deps.audio.startMonitor( + chunk => { + if (socket.readyState === socket.OPEN) socket.send(chunk); + }, + error => { + send({ type: "error", error: error.message }); + stop?.(); + stop = undefined; + } + ); + send({ type: "status", listening: true }); + } catch (error) { + send({ type: "error", error: (error as Error).message }); + } + }); + + const cleanup = () => { + stop?.(); + stop = undefined; + }; + + socket.on("close", cleanup); + socket.on("error", cleanup); + }); +} + export function startWebUi(deps: WebUiDeps): WebUiHandle { const app = express(); app.use(express.json({ limit: "64kb" })); + app.use("/api", (_req, res, next) => { + res.setHeader("Cache-Control", "no-store"); + next(); + }); app.use(express.static(fileURLToPath(new URL("../public", import.meta.url)))); const clients = new Set(); @@ -269,6 +391,84 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); + app.get( + "/api/preferences", + wrap(async (_req, res) => { + res.json({ ok: true, result: deps.preferences.get() }); + }) + ); + + app.post( + "/api/preferences", + wrap(async (req, res) => { + const { visibleGuilds, tts, favourites } = req.body ?? {}; + + if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) { + throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null"); + } + + const patch: Partial = {}; + if (visibleGuilds !== undefined) patch.visibleGuilds = visibleGuilds; + + if (tts !== undefined) { + patch.tts = { + provider: tts?.provider ?? null, + language: tts?.language ?? null, + voice: tts?.voice ?? null + }; + } + + if (favourites !== undefined) { + patch.favourites = { + users: Array.isArray(favourites?.users) ? favourites.users : [], + channels: Array.isArray(favourites?.channels) ? favourites.channels : [] + }; + } + + res.json({ ok: true, result: deps.preferences.update(patch) }); + }) + ); + + app.get( + "/api/favourites", + wrap(async (_req, res) => { + const prefs = deps.preferences.get(); + const userIds = prefs.favourites.users; + + if (!userIds.length) { + res.json({ ok: true, result: { users: [], channels: prefs.favourites.channels } }); + return; + } + + const [statuses, voices] = await Promise.all([ + requireCore().userStatus(userIds) as Promise<{ users?: Array<{ id: string; status: string }> }>, + requireCore().voiceUsers(userIds) as Promise<{ users?: any[] }> + ]); + + const statusMap = new Map((statuses.users ?? []).map(user => [user.id, user.status])); + const voiceMap = new Map((voices.users ?? []).map(user => [user.id, user])); + + res.json({ + ok: true, + result: { + users: userIds.map(id => { + const voice = voiceMap.get(id); + return { + id, + username: voice?.username ?? id, + avatar: voice?.avatar ?? null, + status: statusMap.get(id) ?? "offline", + channelId: voice?.channelId ?? null, + channelName: voice?.channelName ?? null, + guildId: voice?.guildId ?? null + }; + }), + channels: prefs.favourites.channels + } + }); + }) + ); + app.get( "/api/voice", wrap(async (req, res) => { @@ -286,13 +486,137 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }) ); - app.post("/api/voice/join", (_req, res) => { - res.status(501).json({ ok: false, error: "voice.join is not implemented yet" }); - }); + app.post( + "/api/voice/join", + wrap(async (req, res) => { + const channelId = String(req.body?.channelId ?? ""); + const mute = req.body?.mute !== false; - app.post("/api/voice/leave", (_req, res) => { - res.status(501).json({ ok: false, error: "voice.leave is not implemented yet" }); - }); + const result = await pacedVoice(() => requireCore().voiceJoin(channelId, mute)); + deps.getPuppet()?.appendAction({ type: "voice", action: "join", channelId, mute }); + res.json({ ok: true, result }); + }) + ); + + app.post( + "/api/voice/leave", + wrap(async (_req, res) => { + const result = await pacedVoice(() => requireCore().voiceLeave()); + deps.getPuppet()?.appendAction({ type: "voice", action: "leave" }); + res.json({ ok: true, result }); + }) + ); + + app.post( + "/api/voice/move", + wrap(async (req, res) => { + const channelId = String(req.body?.channelId ?? ""); + const result = await pacedVoice(() => requireCore().voiceMove(channelId)); + deps.getPuppet()?.appendAction({ type: "voice", action: "move", channelId }); + res.json({ ok: true, result }); + }) + ); + + app.get( + "/api/audio", + wrap(async (_req, res) => { + res.json({ ok: true, result: deps.audio.status() }); + }) + ); + + app.get( + "/api/audio/tts/voices", + wrap(async (_req, res) => { + res.json({ ok: true, result: groupVoices(await deps.audio.ttsVoices()) }); + }) + ); + + app.get( + "/api/audio/devices", + wrap(async (_req, res) => { + res.json({ ok: true, result: await requireCore().voiceDevices() }); + }) + ); + + app.post( + "/api/audio/connect", + wrap(async (_req, res) => { + res.json({ ok: true, result: await deps.connectAudioDevices() }); + }) + ); + + app.post( + "/api/audio/devices", + wrap(async (req, res) => { + const inputDeviceId = req.body?.inputDeviceId ? String(req.body.inputDeviceId) : undefined; + const outputDeviceId = req.body?.outputDeviceId ? String(req.body.outputDeviceId) : undefined; + + const result = await requireCore().voiceSetDevices(inputDeviceId, outputDeviceId); + deps.getPuppet()?.appendAction({ + type: "audio", + action: "devices", + inputDeviceId: inputDeviceId ?? null, + outputDeviceId: outputDeviceId ?? null + }); + res.json({ ok: true, result }); + }) + ); + + app.post( + "/api/audio/speak", + wrap(async (req, res) => { + const text = String(req.body?.text ?? ""); + const prefs = deps.preferences.get(); + + const provider = + req.body?.provider === "api" || req.body?.provider === "local" ? req.body.provider : prefs.tts.provider; + const voice = req.body?.voice ? String(req.body.voice) : prefs.tts.voice ?? undefined; + + const result = await deps.audio.speak(text, { provider: provider ?? undefined, voice }); + deps.getPuppet()?.appendAction({ + type: "audio", + action: "speak", + provider: provider ?? deps.audio.status().tts.defaultProvider, + chars: text.trim().length + }); + res.json({ ok: true, result }); + }) + ); + + app.post( + "/api/audio/stop", + wrap(async (_req, res) => { + deps.audio.stop(); + deps.getPuppet()?.appendAction({ type: "audio", action: "stop" }); + res.json({ ok: true, result: deps.audio.status() }); + }) + ); + + app.post( + "/api/audio/capture", + wrap(async (req, res) => { + const start = req.body?.start !== false; + const result = start ? await deps.audio.startCapture() : (deps.audio.stopCapture(), { stopped: true }); + + deps.getPuppet()?.appendAction({ type: "audio", action: start ? "capture-start" : "capture-stop" }); + res.json({ ok: true, result }); + }) + ); + + app.get( + "/api/transcript", + wrap(async (req, res) => { + const limit = clampLimit(req.query.limit, 100, 500); + const sessionId = req.query.sessionId ? String(req.query.sessionId) : undefined; + res.json({ + ok: true, + result: { + sessions: deps.transcripts.readSessions(20), + segments: deps.transcripts.readSegments(limit, sessionId) + } + }); + }) + ); app.get("/api/logs", (req, res) => { const puppet = requirePuppet(); @@ -325,8 +649,18 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle { }); }); - const server = app.listen(PORT, HOST, () => { - console.log(`[webui] listening on http://${HOST}:${PORT} (no auth; LAN or Tailscale only)`); + const tls = ensureTls(stateDir()); + const server = tls + ? https.createServer({ key: readFileSync(tls.key), cert: readFileSync(tls.cert) }, app) + : http.createServer(app); + + startAudioSocket(server, deps); + + server.listen(PORT, HOST, () => { + console.log(`[webui] listening on ${tls ? "https" : "http"}://${HOST}:${PORT} (no auth; LAN or Tailscale only)`); + if (!tls && process.env.WEBUI_TLS !== "0") { + console.warn("[webui] TLS is unavailable; browser media APIs need HTTPS or localhost"); + } }); server.on("error", error => { diff --git a/src/main/puppetBridge.ts b/src/main/puppetBridge.ts index c84929c..27a3832 100644 --- a/src/main/puppetBridge.ts +++ b/src/main/puppetBridge.ts @@ -42,6 +42,13 @@ const KNOWN_ACTIONS = new Set([ PuppetAction.StateGet, PuppetAction.VoiceGet, PuppetAction.VoiceOverview, + PuppetAction.VoiceJoin, + PuppetAction.VoiceLeave, + PuppetAction.VoiceMove, + PuppetAction.VoiceUsers, + PuppetAction.UserStatus, + PuppetAction.VoiceDevices, + PuppetAction.VoiceSetDevices, PuppetAction.UserSearch, PuppetAction.DmList, PuppetAction.MessageSend, diff --git a/src/renderer/puppet/actions.ts b/src/renderer/puppet/actions.ts index 19696dc..c0e421e 100644 --- a/src/renderer/puppet/actions.ts +++ b/src/renderer/puppet/actions.ts @@ -20,6 +20,8 @@ import { GuildMemberStore, GuildStore, MessageStore, + PresenceStore, + ReadStateStore, UserStore, VoiceStateStore } from "@vencord/types/webpack/common"; @@ -28,12 +30,21 @@ import { isSnowflake, PUPPET_LIMITS, PuppetAction, PuppetErrorCode, type PuppetR /** Channel types that can receive text messages. */ const SENDABLE_CHANNEL_TYPES = new Set([0, 1, 3, 5, 10, 11, 12]); +/** Channel types that are voice channels. */ +const VOICE_CHANNEL_TYPES = new Set([2, 13]); + /** Discord's message fetch action, located lazily at runtime. */ const MessageActions = findByPropsLazy("fetchMessages"); /** Discord's reaction actions, located lazily at runtime. */ const ReactionActions = findByPropsLazy("addReaction", "removeReaction"); +/** Discord's channel actions, located lazily at runtime. */ +const ChannelActions = findByPropsLazy("selectVoiceChannel"); + +/** Discord's media engine actions, located lazily at runtime. */ +const MediaEngineActions = findByPropsLazy("setSelfMute"); + function compareIds(a: string, b: string): number { try { const left = BigInt(a); @@ -157,20 +168,53 @@ function buildState() { }; } -function voiceMember(state: any) { +function voiceMember(state: any, guildId?: string) { const user = UserStore.getUser(state.userId); return { id: state.userId, username: user?.username ?? state.userId, globalName: user?.globalName ?? null, + nick: guildId ? GuildMemberStore.getNick(guildId, state.userId) : null, avatar: user?.avatar ?? null, mute: !!state.mute, deaf: !!state.deaf, selfMute: !!state.selfMute, - selfDeaf: !!state.selfDeaf + selfDeaf: !!state.selfDeaf, + video: !!state.selfVideo, + stream: !!state.selfStream || !!state.stream, + suppress: !!state.suppress }; } +/** Discord snowflakes carry a millisecond timestamp. */ +function snowflakeToIso(id: string | null | undefined): string | null { + if (!id) return null; + try { + return new Date(Number(BigInt(id) >> 22n) + 1420070400000).toISOString(); + } catch { + return null; + } +} + +function unreadCounts(channelId: string): { unread: number; mentions: number } { + try { + return { + unread: ReadStateStore.getUnreadCount(channelId) ?? 0, + mentions: ReadStateStore.getMentionCount(channelId) ?? 0 + }; + } catch { + return { unread: 0, mentions: 0 }; + } +} + +function presenceOf(userId: string): string { + try { + return PresenceStore.getStatus(userId) ?? "offline"; + } catch { + return "offline"; + } +} + function toUserResult(user: any, guildId?: string) { const nick = guildId ? GuildMemberStore.getNick(guildId, user.id) : null; return { @@ -318,7 +362,66 @@ export async function handlePuppetAction(action: string, params: Record + voiceMember(state, (channel?.guild_id as string | undefined) ?? (guildId as string | undefined)) + ) + }); + } + + case PuppetAction.VoiceUsers: { + const { userIds } = params; + assert( + Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50, + PuppetErrorCode.InvalidParam, + "userIds must be an array of 1 to 50 user IDs" + ); + assert( + userIds.every(id => isSnowflake(id)), + PuppetErrorCode.InvalidSnowflake, + "every userId must be a snowflake" + ); + + const users = (userIds as string[]).map(id => { + const state = VoiceStateStore.getVoiceStateForUser(id) ?? VoiceStateStore.getVoiceState(null, id); + const user = UserStore.getUser(id); + const profile = { + id, + username: user?.username ?? id, + globalName: user?.globalName ?? null, + avatar: user?.avatar ?? null + }; + + if (!state?.channelId) { + return { ...profile, channelId: null, channelName: null, guildId: null, members: [] }; + } + + const channel = ChannelStore.getChannel(state.channelId); + const states = VoiceStateStore.getVoiceStatesForChannel(state.channelId); + + return { + ...profile, + channelId: state.channelId, + channelName: channel?.name ?? null, + guildId: (channel?.guild_id as string | undefined) ?? null, + members: Object.values(states).map(member => + voiceMember(member, channel?.guild_id as string | undefined) + ) + }; + }); + + return ok({ users }); + } + + case PuppetAction.UserStatus: { + const { userIds } = params; + assert( + Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50, + PuppetErrorCode.InvalidParam, + "userIds must be an array of 1 to 50 user IDs" + ); + + return ok({ + users: (userIds as string[]).map(id => ({ id, status: presenceOf(id) })) }); } @@ -339,11 +442,127 @@ export async function handlePuppetAction(action: string, params: Record ({ channelId, channelName: ChannelStore.getChannel(channelId)?.name ?? null, - members: states.map(voiceMember) + members: states.map(state => voiceMember(state, guildId as string)) })) }); } + case PuppetAction.VoiceJoin: + case PuppetAction.VoiceMove: { + const { channelId, mute = true } = params; + assert(isSnowflake(channelId), PuppetErrorCode.InvalidSnowflake, "channelId must be a snowflake"); + + const channel = ChannelStore.getChannel(channelId); + assert(channel, PuppetErrorCode.ChannelNotFound, `channel ${channelId} not found`); + assert( + VOICE_CHANNEL_TYPES.has(channel.type), + PuppetErrorCode.InvalidParam, + `channel type ${channel.type} is not a voice channel` + ); + + const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel; + assert( + typeof selectVoiceChannel === "function", + PuppetErrorCode.NotSupported, + "voice actions are not available in this client" + ); + + if (action === PuppetAction.VoiceJoin && mute !== false) { + const setSelfMute = (MediaEngineActions as any)?.setSelfMute; + if (typeof setSelfMute === "function") { + try { + setSelfMute(true); + } catch { + // Best effort: the join still works unmuted. + } + } + } + + try { + await selectVoiceChannel(channelId); + } catch (error) { + throw new ActionError(PuppetErrorCode.VoiceFailed, `voice join failed: ${String(error)}`); + } + + return ok({ + channelId, + joined: true, + muted: action === PuppetAction.VoiceJoin ? mute !== false : undefined + }); + } + + case PuppetAction.VoiceLeave: { + const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel; + assert( + typeof selectVoiceChannel === "function", + PuppetErrorCode.NotSupported, + "voice actions are not available in this client" + ); + + try { + await selectVoiceChannel(null); + } catch (error) { + throw new ActionError(PuppetErrorCode.VoiceFailed, `voice leave failed: ${String(error)}`); + } + + return ok({ left: true }); + } + + case PuppetAction.VoiceDevices: { + const devices = await navigator.mediaDevices.enumerateDevices(); + return ok({ + input: devices + .filter(device => device.kind === "audioinput") + .map(device => ({ id: device.deviceId, label: device.label || device.deviceId })), + output: devices + .filter(device => device.kind === "audiooutput") + .map(device => ({ id: device.deviceId, label: device.label || device.deviceId })) + }); + } + + case PuppetAction.VoiceSetDevices: { + const { inputDeviceId, outputDeviceId } = params; + if (inputDeviceId !== undefined) { + assert( + typeof inputDeviceId === "string" && inputDeviceId.length <= 200, + PuppetErrorCode.InvalidParam, + "inputDeviceId must be a short string" + ); + } + if (outputDeviceId !== undefined) { + assert( + typeof outputDeviceId === "string" && outputDeviceId.length <= 200, + PuppetErrorCode.InvalidParam, + "outputDeviceId must be a short string" + ); + } + + const setInputDevice = (MediaEngineActions as any)?.setInputDevice; + const setOutputDevice = (MediaEngineActions as any)?.setOutputDevice; + + if (inputDeviceId !== undefined) { + assert( + typeof setInputDevice === "function", + PuppetErrorCode.NotSupported, + "device selection is not available in this client" + ); + await setInputDevice(inputDeviceId); + } + if (outputDeviceId !== undefined) { + assert( + typeof setOutputDevice === "function", + PuppetErrorCode.NotSupported, + "device selection is not available in this client" + ); + await setOutputDevice(outputDeviceId); + } + + return ok({ + inputDeviceId: inputDeviceId ?? null, + outputDeviceId: outputDeviceId ?? null + }); + } + case PuppetAction.UserSearch: { const { query, guildId, limit = 10 } = params; assert( @@ -379,20 +598,45 @@ export async function handlePuppetAction(action: string, params: Record ({ - id: channel.id, - type: channel.type, - name: (channel as any).name ?? null, - icon: (channel as any).icon ?? null, - lastMessageId: (channel as any).lastMessageId ?? null, - recipients: Object.values((channel as any).recipients ?? {}).map((user: any) => ({ - id: user.id, - username: user.username, - globalName: user.globalName ?? null, - avatar: user.avatar ?? null, - bot: !!user.bot - })) - })); + .map(channel => { + const raw = (channel as any).recipients; + const recipientIds: string[] = Array.isArray(raw) + ? raw + : Object.values(raw ?? {}) + .map((user: any) => user?.id ?? user) + .filter(Boolean); + + // 1:1 DMs may store no recipient list; the channel knows the other user. + if (!recipientIds.length && typeof (channel as any).getRecipientId === "function") { + const id = (channel as any).getRecipientId(); + if (id) recipientIds.push(id); + } + + const lastMessageId = (channel as any).lastMessageId ?? null; + const unread = unreadCounts(channel.id); + + return { + id: channel.id, + type: channel.type, + name: (channel as any).name ?? null, + icon: (channel as any).icon ?? null, + lastMessageId, + lastMessageAt: snowflakeToIso(lastMessageId), + unread: unread.unread, + mentions: unread.mentions, + recipients: recipientIds.map(id => { + const user = UserStore.getUser(id); + return { + id, + username: user?.username ?? id, + globalName: user?.globalName ?? null, + avatar: user?.avatar ?? null, + status: presenceOf(id), + bot: !!user?.bot + }; + }) + }; + }); return ok({ channels }); } diff --git a/src/renderer/puppet/events.ts b/src/renderer/puppet/events.ts index 4a335dc..331e3da 100644 --- a/src/renderer/puppet/events.ts +++ b/src/renderer/puppet/events.ts @@ -129,6 +129,20 @@ export function initPuppetEvents() { } }); + dispatcher.subscribe("SPEAKING", (event: any) => { + try { + if (!event?.userId) return; + VesktopNative.puppet.emit({ + type: "VOICE_SPEAKING", + userId: event.userId, + speaking: event.speaking !== false, + channelId: event.channelId ?? event.channel_id ?? null + }); + } catch (error) { + console.error("[puppet] speaking normalization failed:", error); + } + }); + console.log("[veskslop] puppet events active"); }); } diff --git a/src/shared/puppetProtocol.ts b/src/shared/puppetProtocol.ts index 1555139..8965177 100644 --- a/src/shared/puppetProtocol.ts +++ b/src/shared/puppetProtocol.ts @@ -18,6 +18,13 @@ export const enum PuppetAction { StateGet = "state.get", VoiceGet = "voice.get", VoiceOverview = "voice.overview", + VoiceJoin = "voice.join", + VoiceLeave = "voice.leave", + VoiceMove = "voice.move", + VoiceUsers = "voice.users", + UserStatus = "user.status", + VoiceDevices = "voice.devices", + VoiceSetDevices = "voice.setDevices", UserSearch = "user.search", DmList = "dm.list", MessageSend = "message.send", @@ -33,6 +40,13 @@ export const PuppetActions: readonly PuppetAction[] = [ PuppetAction.StateGet, PuppetAction.VoiceGet, PuppetAction.VoiceOverview, + PuppetAction.VoiceJoin, + PuppetAction.VoiceLeave, + PuppetAction.VoiceMove, + PuppetAction.VoiceUsers, + PuppetAction.UserStatus, + PuppetAction.VoiceDevices, + PuppetAction.VoiceSetDevices, PuppetAction.UserSearch, PuppetAction.DmList, PuppetAction.MessageSend, @@ -56,6 +70,7 @@ export const enum PuppetErrorCode { NotSendable = "NOT_SENDABLE", SendFailed = "SEND_FAILED", ReactionFailed = "REACTION_FAILED", + VoiceFailed = "VOICE_FAILED", DmUnavailable = "DM_UNAVAILABLE", NotSupported = "NOT_SUPPORTED", RateLimited = "RATE_LIMITED",