zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add voice audio, capture and transcript, browser monitor, and DM UI

Browse Source
This commit is contained in:
Fisto
2026-09-14 11:19:06 +02:00
parent 65f0788502
commit c003d9532f
27 changed files with 4479 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/puppetBridge.ts
+7
View File
@@ -42,6 +42,13 @@ const KNOWN_ACTIONS = new Set<string>([
PuppetAction.StateGet,
PuppetAction.VoiceGet,
PuppetAction.VoiceOverview,
PuppetAction.VoiceJoin,
PuppetAction.VoiceLeave,
PuppetAction.VoiceMove,
PuppetAction.VoiceUsers,
PuppetAction.UserStatus,
PuppetAction.VoiceDevices,
PuppetAction.VoiceSetDevices,
PuppetAction.UserSearch,
PuppetAction.DmList,
PuppetAction.MessageSend,
src/renderer/puppet/actions.ts
+262 -18
View File
@@ -20,6 +20,8 @@ import {
GuildMemberStore,
GuildStore,
MessageStore,
PresenceStore,
ReadStateStore,
UserStore,
VoiceStateStore
} from "@vencord/types/webpack/common";
@@ -28,12 +30,21 @@ import { isSnowflake, PUPPET_LIMITS, PuppetAction, PuppetErrorCode, type PuppetR
/** Channel types that can receive text messages. */
const SENDABLE_CHANNEL_TYPES = new Set([0, 1, 3, 5, 10, 11, 12]);
/** Channel types that are voice channels. */
const VOICE_CHANNEL_TYPES = new Set([2, 13]);
/** Discord's message fetch action, located lazily at runtime. */
const MessageActions = findByPropsLazy("fetchMessages");
/** Discord's reaction actions, located lazily at runtime. */
const ReactionActions = findByPropsLazy("addReaction", "removeReaction");
/** Discord's channel actions, located lazily at runtime. */
const ChannelActions = findByPropsLazy("selectVoiceChannel");
/** Discord's media engine actions, located lazily at runtime. */
const MediaEngineActions = findByPropsLazy("setSelfMute");
function compareIds(a: string, b: string): number {
try {
const left = BigInt(a);
@@ -157,20 +168,53 @@ function buildState() {
};
}
function voiceMember(state: any) {
function voiceMember(state: any, guildId?: string) {
const user = UserStore.getUser(state.userId);
return {
id: state.userId,
username: user?.username ?? state.userId,
globalName: user?.globalName ?? null,
nick: guildId ? GuildMemberStore.getNick(guildId, state.userId) : null,
avatar: user?.avatar ?? null,
mute: !!state.mute,
deaf: !!state.deaf,
selfMute: !!state.selfMute,
selfDeaf: !!state.selfDeaf
selfDeaf: !!state.selfDeaf,
video: !!state.selfVideo,
stream: !!state.selfStream || !!state.stream,
suppress: !!state.suppress
};
}
/** Discord snowflakes carry a millisecond timestamp. */
function snowflakeToIso(id: string | null | undefined): string | null {
if (!id) return null;
try {
return new Date(Number(BigInt(id) >> 22n) + 1420070400000).toISOString();
} catch {
return null;
}
}
function unreadCounts(channelId: string): { unread: number; mentions: number } {
try {
return {
unread: ReadStateStore.getUnreadCount(channelId) ?? 0,
mentions: ReadStateStore.getMentionCount(channelId) ?? 0
};
} catch {
return { unread: 0, mentions: 0 };
}
}
function presenceOf(userId: string): string {
try {
return PresenceStore.getStatus(userId) ?? "offline";
} catch {
return "offline";
}
}
function toUserResult(user: any, guildId?: string) {
const nick = guildId ? GuildMemberStore.getNick(guildId, user.id) : null;
return {
@@ -318,7 +362,66 @@ export async function handlePuppetAction(action: string, params: Record<string,
guildId: channel?.guild_id ?? guildId ?? null,
channelId: voiceState.channelId,
channelName: channel?.name ?? null,
members: Object.values(states).map(voiceMember)
members: Object.values(states).map(state =>
voiceMember(state, (channel?.guild_id as string | undefined) ?? (guildId as string | undefined))
)
});
}
case PuppetAction.VoiceUsers: {
const { userIds } = params;
assert(
Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50,
PuppetErrorCode.InvalidParam,
"userIds must be an array of 1 to 50 user IDs"
);
assert(
userIds.every(id => isSnowflake(id)),
PuppetErrorCode.InvalidSnowflake,
"every userId must be a snowflake"
);
const users = (userIds as string[]).map(id => {
const state = VoiceStateStore.getVoiceStateForUser(id) ?? VoiceStateStore.getVoiceState(null, id);
const user = UserStore.getUser(id);
const profile = {
id,
username: user?.username ?? id,
globalName: user?.globalName ?? null,
avatar: user?.avatar ?? null
};
if (!state?.channelId) {
return { ...profile, channelId: null, channelName: null, guildId: null, members: [] };
}
const channel = ChannelStore.getChannel(state.channelId);
const states = VoiceStateStore.getVoiceStatesForChannel(state.channelId);
return {
...profile,
channelId: state.channelId,
channelName: channel?.name ?? null,
guildId: (channel?.guild_id as string | undefined) ?? null,
members: Object.values(states).map(member =>
voiceMember(member, channel?.guild_id as string | undefined)
)
};
});
return ok({ users });
}
case PuppetAction.UserStatus: {
const { userIds } = params;
assert(
Array.isArray(userIds) && userIds.length >= 1 && userIds.length <= 50,
PuppetErrorCode.InvalidParam,
"userIds must be an array of 1 to 50 user IDs"
);
return ok({
users: (userIds as string[]).map(id => ({ id, status: presenceOf(id) }))
});
}
@@ -339,11 +442,127 @@ export async function handlePuppetAction(action: string, params: Record<string,
channels: [...byChannel.entries()].map(([channelId, states]) => ({
channelId,
channelName: ChannelStore.getChannel(channelId)?.name ?? null,
members: states.map(voiceMember)
members: states.map(state => voiceMember(state, guildId as string))
}))
});
}
case PuppetAction.VoiceJoin:
case PuppetAction.VoiceMove: {
const { channelId, mute = true } = params;
assert(isSnowflake(channelId), PuppetErrorCode.InvalidSnowflake, "channelId must be a snowflake");
const channel = ChannelStore.getChannel(channelId);
assert(channel, PuppetErrorCode.ChannelNotFound, `channel ${channelId} not found`);
assert(
VOICE_CHANNEL_TYPES.has(channel.type),
PuppetErrorCode.InvalidParam,
`channel type ${channel.type} is not a voice channel`
);
const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel;
assert(
typeof selectVoiceChannel === "function",
PuppetErrorCode.NotSupported,
"voice actions are not available in this client"
);
if (action === PuppetAction.VoiceJoin && mute !== false) {
const setSelfMute = (MediaEngineActions as any)?.setSelfMute;
if (typeof setSelfMute === "function") {
try {
setSelfMute(true);
} catch {
// Best effort: the join still works unmuted.
}
}
}
try {
await selectVoiceChannel(channelId);
} catch (error) {
throw new ActionError(PuppetErrorCode.VoiceFailed, `voice join failed: ${String(error)}`);
}
return ok({
channelId,
joined: true,
muted: action === PuppetAction.VoiceJoin ? mute !== false : undefined
});
}
case PuppetAction.VoiceLeave: {
const selectVoiceChannel = (ChannelActions as any)?.selectVoiceChannel;
assert(
typeof selectVoiceChannel === "function",
PuppetErrorCode.NotSupported,
"voice actions are not available in this client"
);
try {
await selectVoiceChannel(null);
} catch (error) {
throw new ActionError(PuppetErrorCode.VoiceFailed, `voice leave failed: ${String(error)}`);
}
return ok({ left: true });
}
case PuppetAction.VoiceDevices: {
const devices = await navigator.mediaDevices.enumerateDevices();
return ok({
input: devices
.filter(device => device.kind === "audioinput")
.map(device => ({ id: device.deviceId, label: device.label || device.deviceId })),
output: devices
.filter(device => device.kind === "audiooutput")
.map(device => ({ id: device.deviceId, label: device.label || device.deviceId }))
});
}
case PuppetAction.VoiceSetDevices: {
const { inputDeviceId, outputDeviceId } = params;
if (inputDeviceId !== undefined) {
assert(
typeof inputDeviceId === "string" && inputDeviceId.length <= 200,
PuppetErrorCode.InvalidParam,
"inputDeviceId must be a short string"
);
}
if (outputDeviceId !== undefined) {
assert(
typeof outputDeviceId === "string" && outputDeviceId.length <= 200,
PuppetErrorCode.InvalidParam,
"outputDeviceId must be a short string"
);
}
const setInputDevice = (MediaEngineActions as any)?.setInputDevice;
const setOutputDevice = (MediaEngineActions as any)?.setOutputDevice;
if (inputDeviceId !== undefined) {
assert(
typeof setInputDevice === "function",
PuppetErrorCode.NotSupported,
"device selection is not available in this client"
);
await setInputDevice(inputDeviceId);
}
if (outputDeviceId !== undefined) {
assert(
typeof setOutputDevice === "function",
PuppetErrorCode.NotSupported,
"device selection is not available in this client"
);
await setOutputDevice(outputDeviceId);
}
return ok({
inputDeviceId: inputDeviceId ?? null,
outputDeviceId: outputDeviceId ?? null
});
}
case PuppetAction.UserSearch: {
const { query, guildId, limit = 10 } = params;
assert(
@@ -379,20 +598,45 @@ export async function handlePuppetAction(action: string, params: Record<string,
const channels = ChannelStore.getSortedPrivateChannels()
.slice(0, Number(limit))
.map(channel => ({
id: channel.id,
type: channel.type,
name: (channel as any).name ?? null,
icon: (channel as any).icon ?? null,
lastMessageId: (channel as any).lastMessageId ?? null,
recipients: Object.values((channel as any).recipients ?? {}).map((user: any) => ({
id: user.id,
username: user.username,
globalName: user.globalName ?? null,
avatar: user.avatar ?? null,
bot: !!user.bot
}))
}));
.map(channel => {
const raw = (channel as any).recipients;
const recipientIds: string[] = Array.isArray(raw)
? raw
: Object.values(raw ?? {})
.map((user: any) => user?.id ?? user)
.filter(Boolean);
// 1:1 DMs may store no recipient list; the channel knows the other user.
if (!recipientIds.length && typeof (channel as any).getRecipientId === "function") {
const id = (channel as any).getRecipientId();
if (id) recipientIds.push(id);
}
const lastMessageId = (channel as any).lastMessageId ?? null;
const unread = unreadCounts(channel.id);
return {
id: channel.id,
type: channel.type,
name: (channel as any).name ?? null,
icon: (channel as any).icon ?? null,
lastMessageId,
lastMessageAt: snowflakeToIso(lastMessageId),
unread: unread.unread,
mentions: unread.mentions,
recipients: recipientIds.map(id => {
const user = UserStore.getUser(id);
return {
id,
username: user?.username ?? id,
globalName: user?.globalName ?? null,
avatar: user?.avatar ?? null,
status: presenceOf(id),
bot: !!user?.bot
};
})
};
});
return ok({ channels });
}
src/renderer/puppet/events.ts
+14
View File
@@ -129,6 +129,20 @@ export function initPuppetEvents() {
}
});
dispatcher.subscribe("SPEAKING", (event: any) => {
try {
if (!event?.userId) return;
VesktopNative.puppet.emit({
type: "VOICE_SPEAKING",
userId: event.userId,
speaking: event.speaking !== false,
channelId: event.channelId ?? event.channel_id ?? null
});
} catch (error) {
console.error("[puppet] speaking normalization failed:", error);
}
});
console.log("[veskslop] puppet events active");
});
}
src/shared/puppetProtocol.ts
+15
View File
@@ -18,6 +18,13 @@ export const enum PuppetAction {
StateGet = "state.get",
VoiceGet = "voice.get",
VoiceOverview = "voice.overview",
VoiceJoin = "voice.join",
VoiceLeave = "voice.leave",
VoiceMove = "voice.move",
VoiceUsers = "voice.users",
UserStatus = "user.status",
VoiceDevices = "voice.devices",
VoiceSetDevices = "voice.setDevices",
UserSearch = "user.search",
DmList = "dm.list",
MessageSend = "message.send",
@@ -33,6 +40,13 @@ export const PuppetActions: readonly PuppetAction[] = [
PuppetAction.StateGet,
PuppetAction.VoiceGet,
PuppetAction.VoiceOverview,
PuppetAction.VoiceJoin,
PuppetAction.VoiceLeave,
PuppetAction.VoiceMove,
PuppetAction.VoiceUsers,
PuppetAction.UserStatus,
PuppetAction.VoiceDevices,
PuppetAction.VoiceSetDevices,
PuppetAction.UserSearch,
PuppetAction.DmList,
PuppetAction.MessageSend,
@@ -56,6 +70,7 @@ export const enum PuppetErrorCode {
NotSendable = "NOT_SENDABLE",
SendFailed = "SEND_FAILED",
ReactionFailed = "REACTION_FAILED",
VoiceFailed = "VOICE_FAILED",
DmUnavailable = "DM_UNAVAILABLE",
NotSupported = "NOT_SUPPORTED",
RateLimited = "RATE_LIMITED",