diff --git a/src/client/src/azureSpeechClient.ts b/src/client/src/azureSpeechClient.ts index dbe0241..22c345f 100644 --- a/src/client/src/azureSpeechClient.ts +++ b/src/client/src/azureSpeechClient.ts @@ -1,8 +1,8 @@ import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk"; import { azureSpeechApi } from "./api"; +import { transitionVoiceRecognition, type VoiceRecognitionPhase } from "./voiceRecognitionLifecycle"; const VOICE_MODE_STORAGE_KEY = "pi-web.azure-speech.voice-mode"; -export const VOICE_MODE_RESUME_EVENT = "pi-web.azure-speech.resume"; export function voiceModeEnabled(): boolean { return window.localStorage.getItem(VOICE_MODE_STORAGE_KEY) === "true"; @@ -12,10 +12,6 @@ export function setVoiceModeEnabled(enabled: boolean): void { window.localStorage.setItem(VOICE_MODE_STORAGE_KEY, String(enabled)); } -export function resumeVoiceMode(): void { - window.dispatchEvent(new Event(VOICE_MODE_RESUME_EVENT)); -} - /** Call from the microphone tap handler so iOS permits later response playback. */ export function primeVoiceAudio(): void { AzureSpeechClient.primeSpeaker(); @@ -24,7 +20,9 @@ export function primeVoiceAudio(): void { export class AzureSpeechClient { static speaker: HTMLAudioElement | undefined; private recognizer: SpeechSDK.SpeechRecognizer | undefined; - private recognitionStopped: (() => void) | undefined; + private stream: MediaStream | undefined; + private phase: VoiceRecognitionPhase = "idle"; + private generation = 0; private synthesizer: SpeechSDK.SpeechSynthesizer | undefined; static primeSpeaker(): void { @@ -38,38 +36,78 @@ export class AzureSpeechClient { } get recording(): boolean { - return this.recognizer !== undefined; + return this.phase !== "idle"; } - async startRecognition(options: { onReady?: () => void; onInterim: (text: string) => void; onFinal: (text: string) => void; onStopped: () => void; }): Promise { - if (this.recognizer !== undefined) return; + /** + * Begins one persistent microphone session. Its MediaStream is acquired in + * the button's user gesture and is retained between turns, which is required + * for iOS to accept follow-up speech without another tap. + */ + async startRecognition(options: { onReady: () => void; onInterim: (text: string) => void; onFinal: (text: string) => void; onStopped: () => void; }): Promise { + if (this.phase !== "idle") return; if (!window.isSecureContext || !hasMicrophoneAccess(navigator)) { throw new Error("Voice input requires PI WEB to be opened over HTTPS. iPhone browsers do not expose the microphone to an http:// LAN address."); } - const settings = await azureSpeechApi.token(); - const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region); - speechConfig.speechRecognitionLanguage = navigator.language || "en-US"; - // Azure's endpoint detector acts as voice-activity detection: when a - // speaker pauses for this long, it finalizes the utterance and we send it. - speechConfig.setProperty(SpeechSDK.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, "1200"); - const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, SpeechSDK.AudioConfig.fromDefaultMicrophoneInput()); - this.recognizer = recognizer; - this.recognitionStopped = options.onStopped; - recognizer.sessionStarted = () => { options.onReady?.(); }; - recognizer.recognizing = (_sender, event) => { options.onInterim(event.result.text); }; - recognizer.canceled = () => { this.finishRecognition(options.onStopped); }; - recognizer.recognizeOnceAsync( - (result) => { - if (result.reason === SpeechSDK.ResultReason.RecognizedSpeech && result.text.trim() !== "") options.onFinal(result.text); - this.finishRecognition(options.onStopped); - }, - () => { this.finishRecognition(options.onStopped); }, - ); + + const generation = ++this.generation; + // This must be the first asynchronous operation so getUserMedia starts in + // the microphone button's gesture, not after a token/network round trip. + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + this.stream = stream; + try { + const settings = await azureSpeechApi.token(); + if (generation !== this.generation) { stopStream(stream); return; } + + const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region); + speechConfig.speechRecognitionLanguage = navigator.language || "en-US"; + speechConfig.setProperty(SpeechSDK.PropertyId.Speech_SegmentationSilenceTimeoutMs, "1200"); + const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, SpeechSDK.AudioConfig.fromStreamInput(stream)); + this.recognizer = recognizer; + this.phase = transitionVoiceRecognition(this.phase, "start"); + recognizer.recognizing = (_sender, event) => { + if (this.generation === generation && this.phase === "listening") options.onInterim(event.result.text); + }; + recognizer.recognized = (_sender, event) => { + if (this.generation !== generation || this.phase !== "listening") return; + if (event.result.reason !== SpeechSDK.ResultReason.RecognizedSpeech || event.result.text.trim() === "") return; + // Gate the already-authorized stream before notifying the UI. This + // prevents another utterance or TTS from being captured while PI works. + this.setPhase(transitionVoiceRecognition(this.phase, "final")); + options.onFinal(event.result.text); + }; + recognizer.canceled = () => { + if (this.generation !== generation || this.phase === "idle") return; + this.releaseRecognition(); + options.onStopped(); + }; + await callbackPromise((resolve, reject) => { recognizer.startContinuousRecognitionAsync(resolve, reject); }); + if (this.generation !== generation) return; + options.onReady(); + } catch (error) { + if (generation === this.generation) { + if (this.stream !== stream) stopStream(stream); + this.releaseRecognition(); + } + throw error; + } } - stopRecognition(): Promise { - if (this.recognizer !== undefined) this.finishRecognition(); - return Promise.resolve(); + /** Re-enable the existing, user-authorized microphone after response audio. */ + resumeRecognition(): void { + if (this.phase !== "paused") return; + this.setPhase(transitionVoiceRecognition(this.phase, "resume")); + } + + async stopRecognition(): Promise { + if (this.phase === "idle" && this.stream === undefined) return; + ++this.generation; + const recognizer = this.recognizer; + this.releaseRecognition(); + if (recognizer !== undefined) { + await callbackPromise((resolve, reject) => { recognizer.stopContinuousRecognitionAsync(resolve, reject); }).catch(() => undefined); + recognizer.close(); + } } async speak(text: string): Promise { @@ -79,11 +117,7 @@ export class AzureSpeechClient { const settings = await azureSpeechApi.token(); const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region); if (settings.voice !== "") speechConfig.speechSynthesisVoiceName = settings.voice; - // Safari's Media Source support is most reliable with an MP3 stream. speechConfig.speechSynthesisOutputFormat = SpeechSDK.SpeechSynthesisOutputFormat.Audio24Khz48KBitRateMonoMp3; - // Receive the MP3 bytes from Azure, then play them through the audio - // element primed by the microphone tap. SpeakerAudioDestination relies on - // Media Source Extensions and is unreliable in mobile Safari. const synthesizer = new SpeechSDK.SpeechSynthesizer(speechConfig, null); this.synthesizer = synthesizer; try { @@ -106,13 +140,16 @@ export class AzureSpeechClient { this.stopSpeaking(); } - private finishRecognition(onStopped?: () => void): void { - const recognizer = this.recognizer; - const stopped = onStopped ?? this.recognitionStopped; + private setPhase(phase: VoiceRecognitionPhase): void { + this.phase = phase; + for (const track of this.stream?.getAudioTracks() ?? []) track.enabled = phase === "listening"; + } + + private releaseRecognition(): void { + this.phase = transitionVoiceRecognition(this.phase, "stop"); this.recognizer = undefined; - this.recognitionStopped = undefined; - recognizer?.close(); - stopped?.(); + stopStream(this.stream); + this.stream = undefined; } } @@ -121,6 +158,10 @@ function hasMicrophoneAccess(value: Navigator): boolean { return typeof mediaDevices === "object" && mediaDevices !== null && typeof Reflect.get(mediaDevices, "getUserMedia") === "function"; } +function stopStream(stream: MediaStream | undefined): void { + for (const track of stream?.getTracks() ?? []) track.stop(); +} + function synthesisAudio(synthesizer: SpeechSDK.SpeechSynthesizer, text: string): Promise { return new Promise((resolve, reject) => { synthesizer.speakTextAsync( @@ -152,13 +193,12 @@ function playAudioData(audioData: ArrayBuffer): Promise { speaker.load(); URL.revokeObjectURL(url); }; - speaker.onended = () => { - releaseAudioSession(); - // iOS keeps the playback audio session briefly after `ended`; asking for - // the microphone immediately can leave the next recognition silent. - window.setTimeout(resolve, 600); - }; + speaker.onended = () => { releaseAudioSession(); resolve(); }; speaker.onerror = () => { releaseAudioSession(); reject(new Error("Unable to play Azure Speech audio.")); }; void speaker.play().catch((error: unknown) => { releaseAudioSession(); reject(error instanceof Error ? error : new Error(String(error))); }); }); } + +function callbackPromise(start: (resolve: () => void, reject: (error: string) => void) => void): Promise { + return new Promise((resolve, reject) => { start(resolve, (error) => { reject(new Error(error)); }); }); +} diff --git a/src/client/src/components/PiWebApp.ts b/src/client/src/components/PiWebApp.ts index ec8add9..a2d36dc 100644 --- a/src/client/src/components/PiWebApp.ts +++ b/src/client/src/components/PiWebApp.ts @@ -76,7 +76,7 @@ import { shouldShowMachinesSection, type AppNavigationPanel, type NavigationFocu import "./appShell/AppPanelEdgeControl"; import "./appShell/AppRefreshControl"; import { appStyles } from "./shared"; -import { AzureSpeechClient, resumeVoiceMode, voiceModeEnabled } from "../azureSpeechClient"; +import { AzureSpeechClient, voiceModeEnabled } from "../azureSpeechClient"; const PI_WEB_STATUS_REFRESH_MS = 15 * 60 * 1000; @@ -148,7 +148,7 @@ export class PiWebApp extends LitElement { if (!voiceModeEnabled()) return; void this.azureSpeech.speak(text) .catch((error: unknown) => { console.warn("Azure Speech synthesis failed", error); }) - .finally(() => { resumeVoiceMode(); }); + .finally(() => { this.promptEditor?.resumeVoiceInput(); }); }, replacePromptEditorText: async ({ machineId, sessionId, text }) => { await this.updateComplete; diff --git a/src/client/src/components/PromptEditor.ts b/src/client/src/components/PromptEditor.ts index c670615..94bf43d 100644 --- a/src/client/src/components/PromptEditor.ts +++ b/src/client/src/components/PromptEditor.ts @@ -6,7 +6,7 @@ import { defaultHighlightStyle, indentOnInput, indentUnit, syntaxHighlighting } import { LitElement, html, type PropertyValues } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { api, type FileSuggestion, type PromptAttachment, type SessionModel, type SessionStatus, type SlashCommand } from "../api"; -import { AzureSpeechClient, primeVoiceAudio, setVoiceModeEnabled, VOICE_MODE_RESUME_EVENT } from "../azureSpeechClient"; +import { AzureSpeechClient, primeVoiceAudio, setVoiceModeEnabled } from "../azureSpeechClient"; import type { PromptAttachmentDelivery } from "../../../shared/apiTypes"; import { capturePromptAttachments, effectivePromptAttachmentDelivery, isInlinePromptAttachment, promptAttachmentsCanUseInlineDelivery, type CapturedAttachment } from "../promptAttachmentCapture"; import { inputModeForDraft, inputModesEqual, type InputMode } from "../inputModes"; @@ -61,10 +61,6 @@ export class PromptEditor extends LitElement { private readonly azureSpeech = new AzureSpeechClient(); private voiceTranscript = ""; private voicePrefix = ""; - private sendAfterVoiceStop = false; - private readonly resumeVoiceRecognition = () => { - if (this.voiceRecording && !this.azureSpeech.recording) void this.startVoiceTurn(); - }; private attachmentSeq = 0; private requestVersion = 0; private editor: EditorView | undefined; @@ -99,7 +95,6 @@ export class PromptEditor extends LitElement { override firstUpdated(): void { this.createEditor(); - window.addEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition); } protected override updated(changed: PropertyValues) { @@ -108,7 +103,6 @@ export class PromptEditor extends LitElement { } override disconnectedCallback(): void { - window.removeEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition); this.azureSpeech.dispose(); this.editor?.destroy(); this.editor = undefined; @@ -486,50 +480,45 @@ export class PromptEditor extends LitElement { this.voiceError = undefined; if (this.voiceRecording) { this.voiceRecording = false; - this.sendAfterVoiceStop = false; setVoiceModeEnabled(false); await this.azureSpeech.stopRecognition(); return; } + // Must run directly in the button gesture so iOS authorizes this stream + // for every follow-up turn, not just the first one. primeVoiceAudio(); this.voiceRecording = true; setVoiceModeEnabled(true); - await this.startVoiceTurn(); - } - - private async startVoiceTurn(announce = true): Promise { - if (!this.voiceRecording || this.azureSpeech.recording) return; this.voicePrefix = this.draft.trim(); this.voiceTranscript = ""; - this.sendAfterVoiceStop = true; - const playReadyChime = announce ? prepareVoiceReadyChime() : undefined; try { await this.azureSpeech.startRecognition({ - ...(playReadyChime === undefined ? {} : { onReady: playReadyChime }), + onReady: prepareVoiceReadyChime(), onInterim: (text) => { this.replaceText(this.voiceText(text)); }, onFinal: (text) => { - this.voiceTranscript = [this.voiceTranscript, text].filter((part) => part !== "").join(" "); + this.voiceTranscript = text; this.replaceText(this.voiceText()); + this.send("followUp"); + }, + onStopped: () => { + if (!this.voiceRecording) return; + this.voiceRecording = false; + setVoiceModeEnabled(false); }, - onStopped: () => { this.stopVoiceTurn(); }, }); } catch (error) { this.voiceError = errorMessage(error); this.voiceRecording = false; - this.sendAfterVoiceStop = false; setVoiceModeEnabled(false); } } - private stopVoiceTurn(): void { - const shouldSend = this.sendAfterVoiceStop; - this.sendAfterVoiceStop = false; + /** Called by PiWebApp after final TTS playback (or a synthesis failure). */ + resumeVoiceInput(): void { if (!this.voiceRecording) return; - if (shouldSend && this.voiceTranscript.trim() !== "") { - this.send("followUp"); - return; - } - void this.startVoiceTurn(false); + this.voicePrefix = this.draft.trim(); + this.voiceTranscript = ""; + this.azureSpeech.resumeRecognition(); } private voiceText(interim = ""): string { diff --git a/src/client/src/voiceRecognitionLifecycle.test.ts b/src/client/src/voiceRecognitionLifecycle.test.ts new file mode 100644 index 0000000..34fd057 --- /dev/null +++ b/src/client/src/voiceRecognitionLifecycle.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { transitionVoiceRecognition } from "./voiceRecognitionLifecycle"; + +describe("voice recognition lifecycle", () => { + it("keeps the microphone allocated while a final turn waits for TTS", () => { + expect(transitionVoiceRecognition("idle", "start")).toBe("listening"); + expect(transitionVoiceRecognition("listening", "final")).toBe("paused"); + expect(transitionVoiceRecognition("paused", "resume")).toBe("listening"); + }); + + it("ignores duplicate finals until the response resumes listening", () => { + expect(transitionVoiceRecognition("paused", "final")).toBe("paused"); + }); + + it("stops from every active phase", () => { + expect(transitionVoiceRecognition("listening", "stop")).toBe("idle"); + expect(transitionVoiceRecognition("paused", "stop")).toBe("idle"); + }); +}); diff --git a/src/client/src/voiceRecognitionLifecycle.ts b/src/client/src/voiceRecognitionLifecycle.ts new file mode 100644 index 0000000..52b7c27 --- /dev/null +++ b/src/client/src/voiceRecognitionLifecycle.ts @@ -0,0 +1,14 @@ +export type VoiceRecognitionPhase = "idle" | "listening" | "paused"; +export type VoiceRecognitionEvent = "start" | "final" | "resume" | "stop"; + +/** + * The microphone remains allocated across paused turns. A final result is + * accepted once only; subsequent events are ignored until TTS resumes it. + */ +export function transitionVoiceRecognition(phase: VoiceRecognitionPhase, event: VoiceRecognitionEvent): VoiceRecognitionPhase { + if (event === "stop") return "idle"; + if (event === "start" && phase === "idle") return "listening"; + if (event === "final" && phase === "listening") return "paused"; + if (event === "resume" && phase === "paused") return "listening"; + return phase; +}