From c3e0011378f86de4c89cae3a3b1109d414f89c6e Mon Sep 17 00:00:00 2001 From: snowspeeder Date: Tue, 4 Aug 2026 07:28:48 -0400 Subject: [PATCH] feat: add Azure Speech voice mode --- package-lock.json | 160 ++++++++++++++++- package.json | 1 + src/client/src/api.ts | 4 +- src/client/src/api/clients.ts | 10 +- src/client/src/api/parsers.ts | 12 +- src/client/src/azureSpeechClient.ts | 164 ++++++++++++++++++ src/client/src/components/PiWebApp.ts | 9 + src/client/src/components/PromptEditor.ts | 101 ++++++++++- src/client/src/components/SettingsDialog.ts | 10 +- .../src/components/promptEditorIcons.ts | 11 ++ .../settings/SettingsAzureSpeechPanel.ts | 123 +++++++++++++ .../sessionController.liveEvents.test.ts | 31 ++++ .../src/controllers/sessionController.ts | 23 +++ src/client/src/settingsRoute.ts | 3 +- src/server/app.ts | 13 +- src/server/azureSpeechRoutes.test.ts | 40 +++++ src/server/azureSpeechRoutes.ts | 135 ++++++++++++++ src/server/index.ts | 16 +- src/shared/apiTypes.ts | 20 +++ 19 files changed, 875 insertions(+), 11 deletions(-) create mode 100644 src/client/src/azureSpeechClient.ts create mode 100644 src/client/src/components/settings/SettingsAzureSpeechPanel.ts create mode 100644 src/server/azureSpeechRoutes.test.ts create mode 100644 src/server/azureSpeechRoutes.ts diff --git a/package-lock.json b/package-lock.json index 863ebc7..6cb274e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,8 @@ "marked": "^18.0.6", "node-pty": "^1.1.0", "typebox": "1.3.6", - "ws": "^8.21.0" + "ws": "^8.21.0", + "microsoft-cognitiveservices-speech-sdk": "^1.51.0" }, "bin": { "pi-web": "dist/cli.js", @@ -10085,6 +10086,163 @@ "peerDependencies": { "zod": "^3.25.28 || ^4" } + }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@types/webrtc": { + "version": "0.0.37", + "resolved": "https://registry.npmjs.org/@types/webrtc/-/webrtc-0.0.37.tgz", + "integrity": "sha512-JGAJC/ZZDhcrrmepU4sPLQLIOIAgs5oIK+Ieq90K8fdaNMhfdfqmYatJdgif1NDQtvrSlTOGJDUYHIDunuufOg==", + "license": "MIT" + }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.8.tgz", + "integrity": "sha512-bLMpVcWZNzq6lYOybwFwOAR1IXKcHnhUNqYeHjl1bET/qE3jFPFH+p8Wrh3rU4xwdnifPxmKNESBYnvnmc75aA==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/bent": { + "version": "7.3.12", + "resolved": "https://registry.npmjs.org/bent/-/bent-7.3.12.tgz", + "integrity": "sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w==", + "license": "Apache-2.0", + "dependencies": { + "bytesish": "^0.4.1", + "caseless": "~0.12.0", + "is-stream": "^2.0.0" + } + }, + "node_modules/bytesish": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/bytesish/-/bytesish-0.4.4.tgz", + "integrity": "sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ==", + "license": "(Apache-2.0 AND MIT)" + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", + "license": "Apache-2.0" + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.51.0.tgz", + "integrity": "sha512-BLLovv5PegOr5Lp52h4CSgL2c/ViFAh9LoU49ILNPyb/Z0giGI7nPV3xNLcmTtHLT+kfYNRGUIA62vORLzP8TA==", + "license": "MIT", + "dependencies": { + "@azure/core-auth": "^1.9.0", + "@types/webrtc": "^0.0.37", + "agent-base": "^6.0.1", + "bent": "^7.3.12", + "https-proxy-agent": "^4.0.0", + "uuid": "^11.1.1", + "ws": "^8.21.0" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "license": "MIT", + "dependencies": { + "agent-base": "5", + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/https-proxy-agent/node_modules/agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/uuid": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } } } } diff --git a/package.json b/package.json index 26e75ac..604a49d 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "fastify": "^5.10.0", "lit": "^3.3.3", "marked": "^18.0.6", + "microsoft-cognitiveservices-speech-sdk": "^1.51.0", "node-pty": "^1.1.0", "typebox": "1.3.6", "ws": "^8.21.0" diff --git a/src/client/src/api.ts b/src/client/src/api.ts index 47c99c1..d574aa4 100644 --- a/src/client/src/api.ts +++ b/src/client/src/api.ts @@ -1,5 +1,5 @@ -export { activityApi, api, configApi, filesApi, gitApi, machinesApi, piPackagesApi, piWebApi, pluginsApi, projectsApi, sessionsApi, terminalsApi, workspacesApi } from "./api/clients"; +export { activityApi, api, azureSpeechApi, configApi, filesApi, gitApi, machinesApi, piPackagesApi, piWebApi, pluginsApi, projectsApi, sessionsApi, terminalsApi, workspacesApi } from "./api/clients"; export { globalSessionEvents, realtimeEvents, sessionEvents, terminalSocket } from "./api/sockets"; export { DEFAULT_WORKSPACE_UPLOADS_FOLDER, effectiveWorkspaceUploadFolder, uploadWorkspaceFile, uploadWorkspaceFiles, workspaceEffectiveUploadFolder, workspaceUploadPath, WorkspaceUploadBatchError, WorkspaceUploadCancelledError } from "./api/workspaceUploads"; export type { UploadWorkspaceFileOptions, UploadWorkspaceFilesOptions, WorkspaceFileUploadProgress, WorkspaceUploadBatchFileProgress, WorkspaceUploadBatchProgress, WorkspaceUploadFileFailure, WorkspaceUploadFileInput, WorkspaceUploadFolderConfig, WorkspaceUploadTask, WorkspaceUploadXhr, WorkspaceUploadXhrFactory } from "./api/workspaceUploads"; -export type { ActiveAgentProfileDescriptor, ArchiveSessionsResponse, AskUserCloseResponse, AskUserQuestion, AskUserSubmission, PendingAskUser, PendingExtensionDialog, AuthProviderOption, AuthProviderStatus, AuthProvidersResponse, AuthStatusSource, AuthType, CommandOption, CommandResult, DeleteWorkspaceFileResponse, ExtensionDialogAnswer, ExtensionDialogCloseReason, ExtensionDialogCloseResponse, ExtensionDialogKind, ExtensionDialogOutcome, FileContentMediaType, FileContentResponse, FileSuggestion, FileTreeEntry, FileTreeResponse, GitDiffResponse, GitFileState, GitStatusFile, GitStatusResponse, Machine, MachineHealth, MachineKind, MachineRuntime, MachineStatus, MessagePage, ModelSelectionResponse, MoveWorkspaceFileOptions, MoveWorkspaceFileResponse, OAuthFlowState, PiPackageInfo, PiPackageInstallRequest, PiPackageMutationAction, PiPackageMutationResponse, PiPackageRemoveRequest, PiPackageScope, PiPackageUpdateRequest, PiPackagesResponse, PiWebAgentDirEnvSource, PiWebCapability, PiWebComponentStatus, PiWebConfigEnvOverrides, PiWebConfigResponse, PiWebConfigValues, PiWebDockerMode, PiWebInstallationInfo, PiWebInstallationKind, PiWebPluginConfig, PiWebPluginConfigMap, PiWebPluginInfo, PiWebPluginsResponse, PiWebPluginScope, PiWebPluginSettings, PiWebReleaseStatus, PiWebRuntimeComponent, PiWebRuntimeResponse, PiWebShortcutConfig, PiWebStatusMessage, PiWebStatusResponse, PiWebUploadsConfig, Project, PromptAttachment, QueuedSessionMessage, RealtimeEvent, RunTerminalCommandInput, SavedPromptAttachment, SessionActivity, SessionBulkArchiveResponse, SessionBulkDeleteArchivedResponse, SessionBulkFailure, SessionBulkMutationRef, SessionBulkMutationRequest, SessionCleanupExecuteResponse, SessionCleanupPreviewResponse, SessionCleanupProjectSummary, SessionCleanupRequest, SessionCleanupThresholds, SessionCleanupTotals, SessionInfo, SessionModel, SessionRef, SessionStatus, SessionStreamSnapshot, SessionUnreadAcknowledgeRequest, SessionUnreadCatalogSnapshot, SessionUnreadEvent, SessionUnreadSummary, SessionTreeNavigateRequest, SessionTreeNavigateResult, SessionTreeNode, SessionTreeNodeKind, SessionTreeSnapshot, SessionTreeSummaryChoice, SessionWarning, SessionWarningSeverity, SlashCommand, SessionUiEvent, TerminalCommandRun, TerminalCommandRunFilter, TerminalCommandRunHandle, TerminalCommandRunStatus, TerminalInfo, TerminalUiEvent, ThinkingLevel, ThinkingLevelsResponse, WriteWorkspaceFileOptions, WriteWorkspaceFileResponse, Workspace, WorkspaceActivity, WorkspaceActivityResponse, WorkspaceActivityUiEvent } from "../../shared/apiTypes"; +export type { ActiveAgentProfileDescriptor, ArchiveSessionsResponse, AzureSpeechSettings, AzureSpeechSettingsUpdate, AzureSpeechToken, AskUserCloseResponse, AskUserQuestion, AskUserSubmission, PendingAskUser, PendingExtensionDialog, AuthProviderOption, AuthProviderStatus, AuthProvidersResponse, AuthStatusSource, AuthType, CommandOption, CommandResult, DeleteWorkspaceFileResponse, ExtensionDialogAnswer, ExtensionDialogCloseReason, ExtensionDialogCloseResponse, ExtensionDialogKind, ExtensionDialogOutcome, FileContentMediaType, FileContentResponse, FileSuggestion, FileTreeEntry, FileTreeResponse, GitDiffResponse, GitFileState, GitStatusFile, GitStatusResponse, Machine, MachineHealth, MachineKind, MachineRuntime, MachineStatus, MessagePage, ModelSelectionResponse, MoveWorkspaceFileOptions, MoveWorkspaceFileResponse, OAuthFlowState, PiPackageInfo, PiPackageInstallRequest, PiPackageMutationAction, PiPackageMutationResponse, PiPackageRemoveRequest, PiPackageScope, PiPackageUpdateRequest, PiPackagesResponse, PiWebAgentDirEnvSource, PiWebCapability, PiWebComponentStatus, PiWebConfigEnvOverrides, PiWebConfigResponse, PiWebConfigValues, PiWebDockerMode, PiWebInstallationInfo, PiWebInstallationKind, PiWebPluginConfig, PiWebPluginConfigMap, PiWebPluginInfo, PiWebPluginsResponse, PiWebPluginScope, PiWebPluginSettings, PiWebReleaseStatus, PiWebRuntimeComponent, PiWebRuntimeResponse, PiWebShortcutConfig, PiWebStatusMessage, PiWebStatusResponse, PiWebUploadsConfig, Project, PromptAttachment, QueuedSessionMessage, RealtimeEvent, RunTerminalCommandInput, SavedPromptAttachment, SessionActivity, SessionBulkArchiveResponse, SessionBulkDeleteArchivedResponse, SessionBulkFailure, SessionBulkMutationRef, SessionBulkMutationRequest, SessionCleanupExecuteResponse, SessionCleanupPreviewResponse, SessionCleanupProjectSummary, SessionCleanupRequest, SessionCleanupThresholds, SessionCleanupTotals, SessionInfo, SessionModel, SessionRef, SessionStatus, SessionStreamSnapshot, SessionUnreadAcknowledgeRequest, SessionUnreadCatalogSnapshot, SessionUnreadEvent, SessionUnreadSummary, SessionTreeNavigateRequest, SessionTreeNavigateResult, SessionTreeNode, SessionTreeNodeKind, SessionTreeSnapshot, SessionTreeSummaryChoice, SessionWarning, SessionWarningSeverity, SlashCommand, SessionUiEvent, TerminalCommandRun, TerminalCommandRunFilter, TerminalCommandRunHandle, TerminalCommandRunStatus, TerminalInfo, TerminalUiEvent, ThinkingLevel, ThinkingLevelsResponse, WriteWorkspaceFileOptions, WriteWorkspaceFileResponse, Workspace, WorkspaceActivity, WorkspaceActivityResponse, WorkspaceActivityUiEvent } from "../../shared/apiTypes"; diff --git a/src/client/src/api/clients.ts b/src/client/src/api/clients.ts index 74efc59..375b047 100644 --- a/src/client/src/api/clients.ts +++ b/src/client/src/api/clients.ts @@ -1,9 +1,11 @@ -import type { AskUserSubmission, DeleteWorkspaceFileResponse, ExtensionDialogAnswer, FileSuggestion, MoveWorkspaceFileOptions, PiPackageInstallRequest, PiPackageRemoveRequest, PiPackageScope, PiPackageUpdateRequest, PiWebConfigValues, PromptAttachment, RunTerminalCommandInput, SessionBulkMutationRef, SessionCleanupRequest, SessionNotificationDismissThrough, SessionRef, SessionTreeNavigateRequest, SessionUnreadAcknowledgeRequest, TerminalCommandRun, TerminalCommandRunFilter, WriteWorkspaceFileOptions } from "../../../shared/apiTypes"; +import type { AskUserSubmission, AzureSpeechSettings, AzureSpeechSettingsUpdate, AzureSpeechToken, DeleteWorkspaceFileResponse, ExtensionDialogAnswer, FileSuggestion, MoveWorkspaceFileOptions, PiPackageInstallRequest, PiPackageRemoveRequest, PiPackageScope, PiPackageUpdateRequest, PiWebConfigValues, PromptAttachment, RunTerminalCommandInput, SessionBulkMutationRef, SessionCleanupRequest, SessionNotificationDismissThrough, SessionRef, SessionTreeNavigateRequest, SessionUnreadAcknowledgeRequest, TerminalCommandRun, TerminalCommandRunFilter, WriteWorkspaceFileOptions } from "../../../shared/apiTypes"; import { resolveAppUrl } from "../appUrl"; import { request } from "./http"; import { arrayOf, parseAborted, + parseAzureSpeechSettings, + parseAzureSpeechToken, parseAskUserCloseResponse, parseAccepted, parseArchived, @@ -137,6 +139,12 @@ export const configApi = { saveConfig: (config: PiWebConfigValues, machineId?: string) => request(configPath(machineId), parsePiWebConfigResponse, { method: "PUT", body: JSON.stringify({ config }) }), }; +export const azureSpeechApi = { + settings: (): Promise => request("api/azure-speech", parseAzureSpeechSettings), + saveSettings: (settings: AzureSpeechSettingsUpdate): Promise => request("api/azure-speech", parseAzureSpeechSettings, { method: "PUT", body: JSON.stringify(settings) }), + token: (): Promise => request("api/azure-speech/token", parseAzureSpeechToken, { method: "POST" }), +}; + export const pluginsApi = { plugins: (machineId?: string) => request(pluginsPath(machineId), parsePiWebPluginsResponse), }; diff --git a/src/client/src/api/parsers.ts b/src/client/src/api/parsers.ts index 59823c7..d6d6e06 100644 --- a/src/client/src/api/parsers.ts +++ b/src/client/src/api/parsers.ts @@ -1,4 +1,4 @@ -import { ASK_USER_ID_MAX_LENGTH, ASK_USER_OPTION_LIMIT, ASK_USER_OTHER_TEXT_MAX_LENGTH, ASK_USER_QUESTION_LIMIT, ASK_USER_TEXT_MAX_LENGTH, EXTENSION_DIALOG_ID_MAX_LENGTH, EXTENSION_DIALOG_INPUT_MAX_LENGTH, EXTENSION_DIALOG_OPTION_LIMIT, EXTENSION_DIALOG_TEXT_MAX_LENGTH, SESSION_NOTIFICATION_LIMIT, SESSION_NOTIFICATION_MESSAGE_BYTES, SESSION_UNREAD_CATALOG_ID_MAX_LENGTH, SESSION_UNREAD_COMPLETED_AT_MAX_LENGTH, SESSION_UNREAD_CWD_MAX_LENGTH, SESSION_UNREAD_LIMIT, SESSION_UNREAD_SESSION_ID_MAX_LENGTH, type ArchiveSessionsResponse, type AskUserCloseReason, type AskUserCloseResponse, type AskUserOutcome, type AskUserQuestion, type AskUserQuestionOption, type AskUserQuestionRecord, type PendingAskUser, type PendingExtensionDialog, type AuthProviderOption, type AuthProviderStatus, type AuthProvidersResponse, type AuthStatusSource, type AuthType, type CommandOption, type CommandResult, type DeleteWorkspaceFileResponse, type ExtensionDialogAnswer, type ExtensionDialogCloseReason, type ExtensionDialogCloseResponse, type ExtensionDialogKind, type ExtensionDialogOutcome, type FileContentResponse, type FileSuggestion, type FileTreeEntry, type FileTreeResponse, type GitDiffResponse, type GitFileState, type GitStatusFile, type GitStatusResponse, type Machine, type MachineHealth, type MachineKind, type MachineRuntime, type MachineStatus, type MessagePage, type ModelSelectionResponse, type MoveWorkspaceFileResponse, type OAuthFlowState, type PiWebAgentDirEnvSource, type PiWebCapability, type PiWebComponentStatus, type PiWebConfigEnvOverrides, type PiWebConfigResponse, type PiWebConfigValues, type PiWebInstallationInfo, type PiWebPluginConfigMap, type PiWebPluginInfo, type PiWebPluginsResponse, type PiWebPluginScope, type PiWebReleaseStatus, type PiWebRuntimeComponent, type PiWebRuntimeResponse, type PiWebServiceComponent, type PiWebShortcutConfig, type PiWebStatusMessage, type PiWebStatusResponse, type PiWebStatusSeverity, type Project, type QueuedSessionMessage, type SavedPromptAttachment, type SessionBulkArchiveResponse, type SessionBulkDeleteArchivedResponse, type SessionBulkFailure, type SessionCleanupExecuteResponse, type SessionCleanupPreviewResponse, type SessionCleanupProjectSummary, type SessionCleanupThresholds, type SessionCleanupTotals, type SessionInfo, type SessionModel, type SessionNotification, type SessionNotificationClearReason, type SessionNotificationDismissThrough, type SessionNotificationInboxDelta, type SessionNotificationInboxEvent, type SessionNotificationInboxSnapshot, type SessionNotificationSeverity, type SessionNotificationSummary, type SessionStatus, type SessionStreamSnapshot, type SessionUnreadCatalogSnapshot, type SessionUnreadEvent, type SessionUnreadSummary, type SessionWarning, type SessionWarningSeverity, type SlashCommand, type TerminalCommandRun, type TerminalCommandRunStatus, type TerminalInfo, type ThinkingLevelsResponse, type WriteWorkspaceFileResponse, type Workspace, type WorkspaceActivity, type WorkspaceActivityResponse } from "../../../shared/apiTypes"; +import { ASK_USER_ID_MAX_LENGTH, ASK_USER_OPTION_LIMIT, ASK_USER_OTHER_TEXT_MAX_LENGTH, ASK_USER_QUESTION_LIMIT, ASK_USER_TEXT_MAX_LENGTH, EXTENSION_DIALOG_ID_MAX_LENGTH, EXTENSION_DIALOG_INPUT_MAX_LENGTH, EXTENSION_DIALOG_OPTION_LIMIT, EXTENSION_DIALOG_TEXT_MAX_LENGTH, SESSION_NOTIFICATION_LIMIT, SESSION_NOTIFICATION_MESSAGE_BYTES, SESSION_UNREAD_CATALOG_ID_MAX_LENGTH, SESSION_UNREAD_COMPLETED_AT_MAX_LENGTH, SESSION_UNREAD_CWD_MAX_LENGTH, SESSION_UNREAD_LIMIT, SESSION_UNREAD_SESSION_ID_MAX_LENGTH, type ArchiveSessionsResponse, type AzureSpeechSettings, type AzureSpeechToken, type AskUserCloseReason, type AskUserCloseResponse, type AskUserOutcome, type AskUserQuestion, type AskUserQuestionOption, type AskUserQuestionRecord, type PendingAskUser, type PendingExtensionDialog, type AuthProviderOption, type AuthProviderStatus, type AuthProvidersResponse, type AuthStatusSource, type AuthType, type CommandOption, type CommandResult, type DeleteWorkspaceFileResponse, type ExtensionDialogAnswer, type ExtensionDialogCloseReason, type ExtensionDialogCloseResponse, type ExtensionDialogKind, type ExtensionDialogOutcome, type FileContentResponse, type FileSuggestion, type FileTreeEntry, type FileTreeResponse, type GitDiffResponse, type GitFileState, type GitStatusFile, type GitStatusResponse, type Machine, type MachineHealth, type MachineKind, type MachineRuntime, type MachineStatus, type MessagePage, type ModelSelectionResponse, type MoveWorkspaceFileResponse, type OAuthFlowState, type PiWebAgentDirEnvSource, type PiWebCapability, type PiWebComponentStatus, type PiWebConfigEnvOverrides, type PiWebConfigResponse, type PiWebConfigValues, type PiWebInstallationInfo, type PiWebPluginConfigMap, type PiWebPluginInfo, type PiWebPluginsResponse, type PiWebPluginScope, type PiWebReleaseStatus, type PiWebRuntimeComponent, type PiWebRuntimeResponse, type PiWebServiceComponent, type PiWebShortcutConfig, type PiWebStatusMessage, type PiWebStatusResponse, type PiWebStatusSeverity, type Project, type QueuedSessionMessage, type SavedPromptAttachment, type SessionBulkArchiveResponse, type SessionBulkDeleteArchivedResponse, type SessionBulkFailure, type SessionCleanupExecuteResponse, type SessionCleanupPreviewResponse, type SessionCleanupProjectSummary, type SessionCleanupThresholds, type SessionCleanupTotals, type SessionInfo, type SessionModel, type SessionNotification, type SessionNotificationClearReason, type SessionNotificationDismissThrough, type SessionNotificationInboxDelta, type SessionNotificationInboxEvent, type SessionNotificationInboxSnapshot, type SessionNotificationSeverity, type SessionNotificationSummary, type SessionStatus, type SessionStreamSnapshot, type SessionUnreadCatalogSnapshot, type SessionUnreadEvent, type SessionUnreadSummary, type SessionWarning, type SessionWarningSeverity, type SlashCommand, type TerminalCommandRun, type TerminalCommandRunStatus, type TerminalInfo, type ThinkingLevelsResponse, type WriteWorkspaceFileResponse, type Workspace, type WorkspaceActivity, type WorkspaceActivityResponse } from "../../../shared/apiTypes"; import type { PiPackageInfo, PiPackageMutationAction, PiPackageMutationResponse, PiPackageScope, PiPackagesResponse, SessionActivity, SessionStartupProgressEvent, SessionTreeNavigateResult, SessionTreeNode, SessionTreeNodeKind, SessionTreeSnapshot } from "../../../shared/apiTypes"; import { parseActiveAgentProfileDescriptor } from "../../../shared/activeAgentProfile"; import { parseKnownPiWebCapabilities } from "../../../shared/capabilities"; @@ -1207,6 +1207,16 @@ export function parseWorkspaceActivityResponse(value: unknown): WorkspaceActivit return { workspaces: arrayOf(parseWorkspaceActivity)(record["workspaces"]), generatedAt: requireString(record, "generatedAt") }; } +export function parseAzureSpeechSettings(value: unknown): AzureSpeechSettings { + const record = requireRecord(value); + return { region: requireString(record, "region"), voice: requireString(record, "voice"), hasApiKey: requireBoolean(record, "hasApiKey") }; +} + +export function parseAzureSpeechToken(value: unknown): AzureSpeechToken { + const record = requireRecord(value); + return { token: requireString(record, "token"), region: requireString(record, "region"), voice: requireString(record, "voice") }; +} + export function parsePiWebConfigResponse(value: unknown): PiWebConfigResponse { const record = requireRecord(value); return { diff --git a/src/client/src/azureSpeechClient.ts b/src/client/src/azureSpeechClient.ts new file mode 100644 index 0000000..dbe0241 --- /dev/null +++ b/src/client/src/azureSpeechClient.ts @@ -0,0 +1,164 @@ +import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk"; +import { azureSpeechApi } from "./api"; + +const VOICE_MODE_STORAGE_KEY = "pi-web.azure-speech.voice-mode"; +export const VOICE_MODE_RESUME_EVENT = "pi-web.azure-speech.resume"; + +export function voiceModeEnabled(): boolean { + return window.localStorage.getItem(VOICE_MODE_STORAGE_KEY) === "true"; +} + +export function setVoiceModeEnabled(enabled: boolean): void { + window.localStorage.setItem(VOICE_MODE_STORAGE_KEY, String(enabled)); +} + +export function resumeVoiceMode(): void { + window.dispatchEvent(new Event(VOICE_MODE_RESUME_EVENT)); +} + +/** Call from the microphone tap handler so iOS permits later response playback. */ +export function primeVoiceAudio(): void { + AzureSpeechClient.primeSpeaker(); +} + +export class AzureSpeechClient { + static speaker: HTMLAudioElement | undefined; + private recognizer: SpeechSDK.SpeechRecognizer | undefined; + private recognitionStopped: (() => void) | undefined; + private synthesizer: SpeechSDK.SpeechSynthesizer | undefined; + + static primeSpeaker(): void { + const speaker = this.speaker ?? new Audio(); + this.speaker = speaker; + // A muted, zero-length WAV played directly from the mic button's user + // gesture unlocks this element for later Azure response playback on iOS. + speaker.muted = true; + speaker.src = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA="; + void speaker.play().then(() => { speaker.pause(); }).catch(() => undefined); + } + + get recording(): boolean { + return this.recognizer !== undefined; + } + + async startRecognition(options: { onReady?: () => void; onInterim: (text: string) => void; onFinal: (text: string) => void; onStopped: () => void; }): Promise { + if (this.recognizer !== undefined) return; + if (!window.isSecureContext || !hasMicrophoneAccess(navigator)) { + throw new Error("Voice input requires PI WEB to be opened over HTTPS. iPhone browsers do not expose the microphone to an http:// LAN address."); + } + const settings = await azureSpeechApi.token(); + const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region); + speechConfig.speechRecognitionLanguage = navigator.language || "en-US"; + // Azure's endpoint detector acts as voice-activity detection: when a + // speaker pauses for this long, it finalizes the utterance and we send it. + speechConfig.setProperty(SpeechSDK.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, "1200"); + const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, SpeechSDK.AudioConfig.fromDefaultMicrophoneInput()); + this.recognizer = recognizer; + this.recognitionStopped = options.onStopped; + recognizer.sessionStarted = () => { options.onReady?.(); }; + recognizer.recognizing = (_sender, event) => { options.onInterim(event.result.text); }; + recognizer.canceled = () => { this.finishRecognition(options.onStopped); }; + recognizer.recognizeOnceAsync( + (result) => { + if (result.reason === SpeechSDK.ResultReason.RecognizedSpeech && result.text.trim() !== "") options.onFinal(result.text); + this.finishRecognition(options.onStopped); + }, + () => { this.finishRecognition(options.onStopped); }, + ); + } + + stopRecognition(): Promise { + if (this.recognizer !== undefined) this.finishRecognition(); + return Promise.resolve(); + } + + async speak(text: string): Promise { + const spoken = text.trim(); + if (spoken === "") return; + this.stopSpeaking(); + const settings = await azureSpeechApi.token(); + const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region); + if (settings.voice !== "") speechConfig.speechSynthesisVoiceName = settings.voice; + // Safari's Media Source support is most reliable with an MP3 stream. + speechConfig.speechSynthesisOutputFormat = SpeechSDK.SpeechSynthesisOutputFormat.Audio24Khz48KBitRateMonoMp3; + // Receive the MP3 bytes from Azure, then play them through the audio + // element primed by the microphone tap. SpeakerAudioDestination relies on + // Media Source Extensions and is unreliable in mobile Safari. + const synthesizer = new SpeechSDK.SpeechSynthesizer(speechConfig, null); + this.synthesizer = synthesizer; + try { + const audioData = await synthesisAudio(synthesizer, spoken); + await playAudioData(audioData); + } finally { + if (this.synthesizer === synthesizer) this.synthesizer = undefined; + synthesizer.close(); + } + } + + stopSpeaking(): void { + const synthesizer = this.synthesizer; + this.synthesizer = undefined; + synthesizer?.close(); + } + + dispose(): void { + void this.stopRecognition(); + this.stopSpeaking(); + } + + private finishRecognition(onStopped?: () => void): void { + const recognizer = this.recognizer; + const stopped = onStopped ?? this.recognitionStopped; + this.recognizer = undefined; + this.recognitionStopped = undefined; + recognizer?.close(); + stopped?.(); + } +} + +function hasMicrophoneAccess(value: Navigator): boolean { + const mediaDevices: unknown = Reflect.get(value, "mediaDevices"); + return typeof mediaDevices === "object" && mediaDevices !== null && typeof Reflect.get(mediaDevices, "getUserMedia") === "function"; +} + +function synthesisAudio(synthesizer: SpeechSDK.SpeechSynthesizer, text: string): Promise { + return new Promise((resolve, reject) => { + synthesizer.speakTextAsync( + text, + (result) => { + if (result.reason !== SpeechSDK.ResultReason.SynthesizingAudioCompleted) { + reject(new Error(`Azure Speech synthesis failed: ${result.errorDetails || String(result.reason)}`)); + return; + } + resolve(result.audioData); + }, + (error) => { reject(new Error(error)); }, + ); + }); +} + +function playAudioData(audioData: ArrayBuffer): Promise { + const speaker = AzureSpeechClient.speaker ?? new Audio(); + AzureSpeechClient.speaker = speaker; + speaker.muted = false; + const url = URL.createObjectURL(new Blob([audioData], { type: "audio/mpeg" })); + speaker.src = url; + return new Promise((resolve, reject) => { + const releaseAudioSession = () => { + speaker.onended = null; + speaker.onerror = null; + speaker.pause(); + speaker.removeAttribute("src"); + speaker.load(); + URL.revokeObjectURL(url); + }; + speaker.onended = () => { + releaseAudioSession(); + // iOS keeps the playback audio session briefly after `ended`; asking for + // the microphone immediately can leave the next recognition silent. + window.setTimeout(resolve, 600); + }; + speaker.onerror = () => { releaseAudioSession(); reject(new Error("Unable to play Azure Speech audio.")); }; + void speaker.play().catch((error: unknown) => { releaseAudioSession(); reject(error instanceof Error ? error : new Error(String(error))); }); + }); +} diff --git a/src/client/src/components/PiWebApp.ts b/src/client/src/components/PiWebApp.ts index 34727ca..ec8add9 100644 --- a/src/client/src/components/PiWebApp.ts +++ b/src/client/src/components/PiWebApp.ts @@ -76,6 +76,7 @@ import { shouldShowMachinesSection, type AppNavigationPanel, type NavigationFocu import "./appShell/AppPanelEdgeControl"; import "./appShell/AppRefreshControl"; import { appStyles } from "./shared"; +import { AzureSpeechClient, resumeVoiceMode, voiceModeEnabled } from "../azureSpeechClient"; const PI_WEB_STATUS_REFRESH_MS = 15 * 60 * 1000; @@ -110,6 +111,7 @@ export class PiWebApp extends LitElement { @query("#navigation-panel") private navigationPanelFrame?: HTMLElement; @query("#workspace-panel") private workspacePanelFrame?: HTMLElement; + private readonly azureSpeech = new AzureSpeechClient(); private readonly sessionUnread = new SessionUnreadController({ onChange: (machineId) => { this.syncUnreadPresence(); @@ -142,6 +144,12 @@ export class PiWebApp extends LitElement { onSelectedSessionReady: ({ machineId, session }) => { void this.commitReadyChatAfterRender(machineId, session); }, + onFinalAssistantResponse: ({ text }) => { + if (!voiceModeEnabled()) return; + void this.azureSpeech.speak(text) + .catch((error: unknown) => { console.warn("Azure Speech synthesis failed", error); }) + .finally(() => { resumeVoiceMode(); }); + }, replacePromptEditorText: async ({ machineId, sessionId, text }) => { await this.updateComplete; if (selectedMachineId(this.state) !== machineId || this.state.selectedSession?.id !== sessionId) return; @@ -397,6 +405,7 @@ export class PiWebApp extends LitElement { this.systemLightThemeMedia?.removeEventListener("change", this.onSystemLightThemeChange); this.keyboard.reset(); this.auth.dispose(); + this.azureSpeech.dispose(); this.sessions.dispose(); this.notifications.dispose(); this.realtime.close(); diff --git a/src/client/src/components/PromptEditor.ts b/src/client/src/components/PromptEditor.ts index 68f7493..c670615 100644 --- a/src/client/src/components/PromptEditor.ts +++ b/src/client/src/components/PromptEditor.ts @@ -6,6 +6,7 @@ import { defaultHighlightStyle, indentOnInput, indentUnit, syntaxHighlighting } import { LitElement, html, type PropertyValues } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { api, type FileSuggestion, type PromptAttachment, type SessionModel, type SessionStatus, type SlashCommand } from "../api"; +import { AzureSpeechClient, primeVoiceAudio, setVoiceModeEnabled, VOICE_MODE_RESUME_EVENT } from "../azureSpeechClient"; import type { PromptAttachmentDelivery } from "../../../shared/apiTypes"; import { capturePromptAttachments, effectivePromptAttachmentDelivery, isInlinePromptAttachment, promptAttachmentsCanUseInlineDelivery, type CapturedAttachment } from "../promptAttachmentCapture"; import { inputModeForDraft, inputModesEqual, type InputMode } from "../inputModes"; @@ -15,7 +16,7 @@ import { clearDraft, loadDraft, saveDraft } from "../promptDraftStorage"; import { loadAttachmentDelivery, saveAttachmentDelivery } from "../attachmentPreferences"; import { createMobilePromptEnterMedia, readPromptEnterPreference, shouldSendPromptOnEnterShortcut, shouldUsePromptEnterShiftShortcut } from "../promptEnterBehavior"; import { promptEditorStyles, type CompletionItem } from "./shared"; -import { renderAttachIcon, renderSendIcon, renderQueueIcon, renderSteerIcon, renderStopIcon, renderThinkingGauge } from "./promptEditorIcons"; +import { renderAttachIcon, renderSendIcon, renderQueueIcon, renderSteerIcon, renderStopIcon, renderThinkingGauge, renderVoiceIcon } from "./promptEditorIcons"; import { thinkingGauge, thinkingLevelLabel } from "../../../shared/thinkingLevels"; import "./AutocompleteMenu"; @@ -55,6 +56,15 @@ export class PromptEditor extends LitElement { @state() private attachments: PendingAttachment[] = []; @state() private attachmentDelivery: PromptAttachmentDelivery = loadAttachmentDelivery(); @state() private attachmentError: string | undefined = undefined; + @state() private voiceRecording = false; + @state() private voiceError: string | undefined = undefined; + private readonly azureSpeech = new AzureSpeechClient(); + private voiceTranscript = ""; + private voicePrefix = ""; + private sendAfterVoiceStop = false; + private readonly resumeVoiceRecognition = () => { + if (this.voiceRecording && !this.azureSpeech.recording) void this.startVoiceTurn(); + }; private attachmentSeq = 0; private requestVersion = 0; private editor: EditorView | undefined; @@ -89,6 +99,7 @@ export class PromptEditor extends LitElement { override firstUpdated(): void { this.createEditor(); + window.addEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition); } protected override updated(changed: PropertyValues) { @@ -97,6 +108,8 @@ export class PromptEditor extends LitElement { } override disconnectedCallback(): void { + window.removeEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition); + this.azureSpeech.dispose(); this.editor?.destroy(); this.editor = undefined; super.disconnectedCallback(); @@ -116,10 +129,12 @@ export class PromptEditor extends LitElement { ${shellMode ? html`
Shell command${shellInputMode.excludeFromContext ? " · excluded from context" : ""}
` : null} ${this.isCompacting && !shellMode ? html`
Compacting history · message will be queued
` : null} ${this.renderAttachments()} + ${this.voiceError === undefined ? null : html``} { this.pick(item); }}>
${this.renderCompactStatus()} + ${this.canSteer && !this.isCompacting ? html`` : null} @@ -467,6 +482,60 @@ export class PromptEditor extends LitElement { this.completions = []; } + private async toggleVoice(): Promise { + this.voiceError = undefined; + if (this.voiceRecording) { + this.voiceRecording = false; + this.sendAfterVoiceStop = false; + setVoiceModeEnabled(false); + await this.azureSpeech.stopRecognition(); + return; + } + primeVoiceAudio(); + this.voiceRecording = true; + setVoiceModeEnabled(true); + await this.startVoiceTurn(); + } + + private async startVoiceTurn(announce = true): Promise { + if (!this.voiceRecording || this.azureSpeech.recording) return; + this.voicePrefix = this.draft.trim(); + this.voiceTranscript = ""; + this.sendAfterVoiceStop = true; + const playReadyChime = announce ? prepareVoiceReadyChime() : undefined; + try { + await this.azureSpeech.startRecognition({ + ...(playReadyChime === undefined ? {} : { onReady: playReadyChime }), + onInterim: (text) => { this.replaceText(this.voiceText(text)); }, + onFinal: (text) => { + this.voiceTranscript = [this.voiceTranscript, text].filter((part) => part !== "").join(" "); + this.replaceText(this.voiceText()); + }, + onStopped: () => { this.stopVoiceTurn(); }, + }); + } catch (error) { + this.voiceError = errorMessage(error); + this.voiceRecording = false; + this.sendAfterVoiceStop = false; + setVoiceModeEnabled(false); + } + } + + private stopVoiceTurn(): void { + const shouldSend = this.sendAfterVoiceStop; + this.sendAfterVoiceStop = false; + if (!this.voiceRecording) return; + if (shouldSend && this.voiceTranscript.trim() !== "") { + this.send("followUp"); + return; + } + void this.startVoiceTurn(false); + } + + private voiceText(interim = ""): string { + return [this.voicePrefix, this.voiceTranscript, interim].filter((part) => part.trim() !== "").join(" "); + } + private send(streamingBehavior?: "steer" | "followUp") { if (this.disabled || this.sending) return; const text = this.draft.trim(); @@ -584,6 +653,36 @@ const codeLikeInputAssistanceAttributes: Record = { dir: "auto", }; +function prepareVoiceReadyChime(): () => void { + // Create and resume the context in the tap handler. That keeps iOS' user- + // gesture audio permission alive until Azure confirms the microphone session. + const context = new AudioContext(); + void context.resume(); + let played = false; + return () => { + if (played) return; + played = true; + const start = context.currentTime; + for (const [offset, frequency] of [[0, 523.25], [0.09, 783.99]] as const) { + const oscillator = context.createOscillator(); + const gain = context.createGain(); + oscillator.type = "sine"; + oscillator.frequency.setValueAtTime(frequency, start + offset); + gain.gain.setValueAtTime(0.0001, start + offset); + gain.gain.exponentialRampToValueAtTime(0.055, start + offset + 0.018); + gain.gain.exponentialRampToValueAtTime(0.0001, start + offset + 0.24); + oscillator.connect(gain).connect(context.destination); + oscillator.start(start + offset); + oscillator.stop(start + offset + 0.25); + } + window.setTimeout(() => { void context.close(); }, 500); + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + function inputAssistanceContentAttributes(draftBeforeCursor: string): Record { // CodeMirror is optimized for code and disables these by default, but the chat prompt is usually prose. return inputModeForDraft(draftBeforeCursor).kind === "normal" ? proseInputAssistanceAttributes : codeLikeInputAssistanceAttributes; diff --git a/src/client/src/components/SettingsDialog.ts b/src/client/src/components/SettingsDialog.ts index fa9b99a..3f2a0bd 100644 --- a/src/client/src/components/SettingsDialog.ts +++ b/src/client/src/components/SettingsDialog.ts @@ -4,6 +4,7 @@ import type { AppAction } from "../actions"; import { configApi, piPackagesApi, pluginsApi, type Machine, type MachineRuntime, type PiPackageMutationResponse, type PiPackageScope, type PiPackagesResponse, type PiWebConfigResponse, type PiWebConfigValues, type PiWebPluginsResponse } from "../api"; import type { SettingsSection } from "../settingsRoute"; import "./settings/SettingsGeneralPanel"; +import "./settings/SettingsAzureSpeechPanel"; import "./settings/SettingsSessiondPanel"; import "./settings/SettingsPackagesPanel"; import "./settings/SettingsPluginsPanel"; @@ -118,6 +119,7 @@ export class SettingsDialog extends LitElement { ${this.renderNavButton("packages", "Pi packages", "Selected machine")} ${this.renderNavButton("plugins", "PI WEB plugins", "Selected machine")} ${this.renderNavButton("shortcuts", "Keyboard", "Gateway shortcuts")} + ${this.renderNavButton("azure-speech", "Azure Speech", "Voice input + replies")}
${this.renderActiveSection()} @@ -148,6 +150,9 @@ export class SettingsDialog extends LitElement { > `; } + if (this.section === "azure-speech") { + return html``; + } if (this.section === "shortcuts") { return html`