fix: keep Azure voice mode active between turns

src/client/src/azureSpeechClient.ts

@@ -1,8 +1,8 @@
 import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk";
 import { azureSpeechApi } from "./api";
+import { transitionVoiceRecognition, type VoiceRecognitionPhase } from "./voiceRecognitionLifecycle";
 
 const VOICE_MODE_STORAGE_KEY = "pi-web.azure-speech.voice-mode";
-export const VOICE_MODE_RESUME_EVENT = "pi-web.azure-speech.resume";
 
 export function voiceModeEnabled(): boolean {
   return window.localStorage.getItem(VOICE_MODE_STORAGE_KEY) === "true";
@@ -12,10 +12,6 @@ export function setVoiceModeEnabled(enabled: boolean): void {
   window.localStorage.setItem(VOICE_MODE_STORAGE_KEY, String(enabled));
 }
 
-export function resumeVoiceMode(): void {
-  window.dispatchEvent(new Event(VOICE_MODE_RESUME_EVENT));
-}
-
 /** Call from the microphone tap handler so iOS permits later response playback. */
 export function primeVoiceAudio(): void {
   AzureSpeechClient.primeSpeaker();
@@ -24,7 +20,9 @@ export function primeVoiceAudio(): void {
 export class AzureSpeechClient {
   static speaker: HTMLAudioElement | undefined;
   private recognizer: SpeechSDK.SpeechRecognizer | undefined;
-  private recognitionStopped: (() => void) | undefined;
+  private stream: MediaStream | undefined;
+  private phase: VoiceRecognitionPhase = "idle";
+  private generation = 0;
   private synthesizer: SpeechSDK.SpeechSynthesizer | undefined;
 
   static primeSpeaker(): void {
@@ -38,38 +36,78 @@ export class AzureSpeechClient {
   }
 
   get recording(): boolean {
-    return this.recognizer !== undefined;
+    return this.phase !== "idle";
   }
 
-  async startRecognition(options: { onReady?: () => void; onInterim: (text: string) => void; onFinal: (text: string) => void; onStopped: () => void; }): Promise<void> {
-    if (this.recognizer !== undefined) return;
+  /**
+   * Begins one persistent microphone session. Its MediaStream is acquired in
+   * the button's user gesture and is retained between turns, which is required
+   * for iOS to accept follow-up speech without another tap.
+   */
+  async startRecognition(options: { onReady: () => void; onInterim: (text: string) => void; onFinal: (text: string) => void; onStopped: () => void; }): Promise<void> {
+    if (this.phase !== "idle") return;
     if (!window.isSecureContext || !hasMicrophoneAccess(navigator)) {
       throw new Error("Voice input requires PI WEB to be opened over HTTPS. iPhone browsers do not expose the microphone to an http:// LAN address.");
     }
-    const settings = await azureSpeechApi.token();
-    const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region);
-    speechConfig.speechRecognitionLanguage = navigator.language || "en-US";
-    // Azure's endpoint detector acts as voice-activity detection: when a
-    // speaker pauses for this long, it finalizes the utterance and we send it.
-    speechConfig.setProperty(SpeechSDK.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, "1200");
-    const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, SpeechSDK.AudioConfig.fromDefaultMicrophoneInput());
-    this.recognizer = recognizer;
-    this.recognitionStopped = options.onStopped;
-    recognizer.sessionStarted = () => { options.onReady?.(); };
-    recognizer.recognizing = (_sender, event) => { options.onInterim(event.result.text); };
-    recognizer.canceled = () => { this.finishRecognition(options.onStopped); };
-    recognizer.recognizeOnceAsync(
-      (result) => {
-        if (result.reason === SpeechSDK.ResultReason.RecognizedSpeech && result.text.trim() !== "") options.onFinal(result.text);
-        this.finishRecognition(options.onStopped);
-      },
-      () => { this.finishRecognition(options.onStopped); },
-    );
+
+    const generation = ++this.generation;
+    // This must be the first asynchronous operation so getUserMedia starts in
+    // the microphone button's gesture, not after a token/network round trip.
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    this.stream = stream;
+    try {
+      const settings = await azureSpeechApi.token();
+      if (generation !== this.generation) { stopStream(stream); return; }
+
+      const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region);
+      speechConfig.speechRecognitionLanguage = navigator.language || "en-US";
+      speechConfig.setProperty(SpeechSDK.PropertyId.Speech_SegmentationSilenceTimeoutMs, "1200");
+      const recognizer = new SpeechSDK.SpeechRecognizer(speechConfig, SpeechSDK.AudioConfig.fromStreamInput(stream));
+      this.recognizer = recognizer;
+      this.phase = transitionVoiceRecognition(this.phase, "start");
+      recognizer.recognizing = (_sender, event) => {
+        if (this.generation === generation && this.phase === "listening") options.onInterim(event.result.text);
+      };
+      recognizer.recognized = (_sender, event) => {
+        if (this.generation !== generation || this.phase !== "listening") return;
+        if (event.result.reason !== SpeechSDK.ResultReason.RecognizedSpeech || event.result.text.trim() === "") return;
+        // Gate the already-authorized stream before notifying the UI. This
+        // prevents another utterance or TTS from being captured while PI works.
+        this.setPhase(transitionVoiceRecognition(this.phase, "final"));
+        options.onFinal(event.result.text);
+      };
+      recognizer.canceled = () => {
+        if (this.generation !== generation || this.phase === "idle") return;
+        this.releaseRecognition();
+        options.onStopped();
+      };
+      await callbackPromise((resolve, reject) => { recognizer.startContinuousRecognitionAsync(resolve, reject); });
+      if (this.generation !== generation) return;
+      options.onReady();
+    } catch (error) {
+      if (generation === this.generation) {
+        if (this.stream !== stream) stopStream(stream);
+        this.releaseRecognition();
+      }
+      throw error;
+    }
   }
 
-  stopRecognition(): Promise<void> {
-    if (this.recognizer !== undefined) this.finishRecognition();
-    return Promise.resolve();
+  /** Re-enable the existing, user-authorized microphone after response audio. */
+  resumeRecognition(): void {
+    if (this.phase !== "paused") return;
+    this.setPhase(transitionVoiceRecognition(this.phase, "resume"));
+  }
+
+  async stopRecognition(): Promise<void> {
+    if (this.phase === "idle" && this.stream === undefined) return;
+    ++this.generation;
+    const recognizer = this.recognizer;
+    this.releaseRecognition();
+    if (recognizer !== undefined) {
+      await callbackPromise((resolve, reject) => { recognizer.stopContinuousRecognitionAsync(resolve, reject); }).catch(() => undefined);
+      recognizer.close();
+    }
   }
 
   async speak(text: string): Promise<void> {
@@ -79,11 +117,7 @@ export class AzureSpeechClient {
     const settings = await azureSpeechApi.token();
     const speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(settings.token, settings.region);
     if (settings.voice !== "") speechConfig.speechSynthesisVoiceName = settings.voice;
-    // Safari's Media Source support is most reliable with an MP3 stream.
     speechConfig.speechSynthesisOutputFormat = SpeechSDK.SpeechSynthesisOutputFormat.Audio24Khz48KBitRateMonoMp3;
-    // Receive the MP3 bytes from Azure, then play them through the audio
-    // element primed by the microphone tap. SpeakerAudioDestination relies on
-    // Media Source Extensions and is unreliable in mobile Safari.
     const synthesizer = new SpeechSDK.SpeechSynthesizer(speechConfig, null);
     this.synthesizer = synthesizer;
     try {
@@ -106,13 +140,16 @@ export class AzureSpeechClient {
     this.stopSpeaking();
   }
 
-  private finishRecognition(onStopped?: () => void): void {
-    const recognizer = this.recognizer;
-    const stopped = onStopped ?? this.recognitionStopped;
+  private setPhase(phase: VoiceRecognitionPhase): void {
+    this.phase = phase;
+    for (const track of this.stream?.getAudioTracks() ?? []) track.enabled = phase === "listening";
+  }
+
+  private releaseRecognition(): void {
+    this.phase = transitionVoiceRecognition(this.phase, "stop");
     this.recognizer = undefined;
-    this.recognitionStopped = undefined;
-    recognizer?.close();
-    stopped?.();
+    stopStream(this.stream);
+    this.stream = undefined;
   }
 }
 
@@ -121,6 +158,10 @@ function hasMicrophoneAccess(value: Navigator): boolean {
   return typeof mediaDevices === "object" && mediaDevices !== null && typeof Reflect.get(mediaDevices, "getUserMedia") === "function";
 }
 
+function stopStream(stream: MediaStream | undefined): void {
+  for (const track of stream?.getTracks() ?? []) track.stop();
+}
+
 function synthesisAudio(synthesizer: SpeechSDK.SpeechSynthesizer, text: string): Promise<ArrayBuffer> {
   return new Promise((resolve, reject) => {
     synthesizer.speakTextAsync(
@@ -152,13 +193,12 @@ function playAudioData(audioData: ArrayBuffer): Promise<void> {
       speaker.load();
       URL.revokeObjectURL(url);
     };
-    speaker.onended = () => {
-      releaseAudioSession();
-      // iOS keeps the playback audio session briefly after `ended`; asking for
-      // the microphone immediately can leave the next recognition silent.
-      window.setTimeout(resolve, 600);
-    };
+    speaker.onended = () => { releaseAudioSession(); resolve(); };
     speaker.onerror = () => { releaseAudioSession(); reject(new Error("Unable to play Azure Speech audio.")); };
     void speaker.play().catch((error: unknown) => { releaseAudioSession(); reject(error instanceof Error ? error : new Error(String(error))); });
   });
 }
+
+function callbackPromise(start: (resolve: () => void, reject: (error: string) => void) => void): Promise<void> {
+  return new Promise((resolve, reject) => { start(resolve, (error) => { reject(new Error(error)); }); });
+}

src/client/src/components/PiWebApp.ts

@@ -76,7 +76,7 @@ import { shouldShowMachinesSection, type AppNavigationPanel, type NavigationFocu
 import "./appShell/AppPanelEdgeControl";
 import "./appShell/AppRefreshControl";
 import { appStyles } from "./shared";
-import { AzureSpeechClient, resumeVoiceMode, voiceModeEnabled } from "../azureSpeechClient";
+import { AzureSpeechClient, voiceModeEnabled } from "../azureSpeechClient";
 
 
 const PI_WEB_STATUS_REFRESH_MS = 15 * 60 * 1000;
@@ -148,7 +148,7 @@ export class PiWebApp extends LitElement {
         if (!voiceModeEnabled()) return;
         void this.azureSpeech.speak(text)
           .catch((error: unknown) => { console.warn("Azure Speech synthesis failed", error); })
-          .finally(() => { resumeVoiceMode(); });
+          .finally(() => { this.promptEditor?.resumeVoiceInput(); });
       },
       replacePromptEditorText: async ({ machineId, sessionId, text }) => {
         await this.updateComplete;

src/client/src/components/PromptEditor.ts

@@ -6,7 +6,7 @@ import { defaultHighlightStyle, indentOnInput, indentUnit, syntaxHighlighting }
 import { LitElement, html, type PropertyValues } from "lit";
 import { customElement, property, query, state } from "lit/decorators.js";
 import { api, type FileSuggestion, type PromptAttachment, type SessionModel, type SessionStatus, type SlashCommand } from "../api";
-import { AzureSpeechClient, primeVoiceAudio, setVoiceModeEnabled, VOICE_MODE_RESUME_EVENT } from "../azureSpeechClient";
+import { AzureSpeechClient, primeVoiceAudio, setVoiceModeEnabled } from "../azureSpeechClient";
 import type { PromptAttachmentDelivery } from "../../../shared/apiTypes";
 import { capturePromptAttachments, effectivePromptAttachmentDelivery, isInlinePromptAttachment, promptAttachmentsCanUseInlineDelivery, type CapturedAttachment } from "../promptAttachmentCapture";
 import { inputModeForDraft, inputModesEqual, type InputMode } from "../inputModes";
@@ -61,10 +61,6 @@ export class PromptEditor extends LitElement {
   private readonly azureSpeech = new AzureSpeechClient();
   private voiceTranscript = "";
   private voicePrefix = "";
-  private sendAfterVoiceStop = false;
-  private readonly resumeVoiceRecognition = () => {
-    if (this.voiceRecording && !this.azureSpeech.recording) void this.startVoiceTurn();
-  };
   private attachmentSeq = 0;
   private requestVersion = 0;
   private editor: EditorView | undefined;
@@ -99,7 +95,6 @@ export class PromptEditor extends LitElement {
 
   override firstUpdated(): void {
     this.createEditor();
-    window.addEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition);
   }
 
   protected override updated(changed: PropertyValues) {
@@ -108,7 +103,6 @@ export class PromptEditor extends LitElement {
   }
 
   override disconnectedCallback(): void {
-    window.removeEventListener(VOICE_MODE_RESUME_EVENT, this.resumeVoiceRecognition);
     this.azureSpeech.dispose();
     this.editor?.destroy();
     this.editor = undefined;
@@ -486,50 +480,45 @@ export class PromptEditor extends LitElement {
     this.voiceError = undefined;
     if (this.voiceRecording) {
       this.voiceRecording = false;
-      this.sendAfterVoiceStop = false;
       setVoiceModeEnabled(false);
       await this.azureSpeech.stopRecognition();
       return;
     }
+    // Must run directly in the button gesture so iOS authorizes this stream
+    // for every follow-up turn, not just the first one.
     primeVoiceAudio();
     this.voiceRecording = true;
     setVoiceModeEnabled(true);
-    await this.startVoiceTurn();
-  }
-
-  private async startVoiceTurn(announce = true): Promise<void> {
-    if (!this.voiceRecording || this.azureSpeech.recording) return;
     this.voicePrefix = this.draft.trim();
     this.voiceTranscript = "";
-    this.sendAfterVoiceStop = true;
-    const playReadyChime = announce ? prepareVoiceReadyChime() : undefined;
     try {
       await this.azureSpeech.startRecognition({
-        ...(playReadyChime === undefined ? {} : { onReady: playReadyChime }),
+        onReady: prepareVoiceReadyChime(),
         onInterim: (text) => { this.replaceText(this.voiceText(text)); },
         onFinal: (text) => {
-          this.voiceTranscript = [this.voiceTranscript, text].filter((part) => part !== "").join(" ");
+          this.voiceTranscript = text;
           this.replaceText(this.voiceText());
+          this.send("followUp");
+        },
+        onStopped: () => {
+          if (!this.voiceRecording) return;
+          this.voiceRecording = false;
+          setVoiceModeEnabled(false);
         },
-        onStopped: () => { this.stopVoiceTurn(); },
       });
     } catch (error) {
       this.voiceError = errorMessage(error);
       this.voiceRecording = false;
-      this.sendAfterVoiceStop = false;
       setVoiceModeEnabled(false);
     }
   }
 
-  private stopVoiceTurn(): void {
-    const shouldSend = this.sendAfterVoiceStop;
-    this.sendAfterVoiceStop = false;
+  /** Called by PiWebApp after final TTS playback (or a synthesis failure). */
+  resumeVoiceInput(): void {
     if (!this.voiceRecording) return;
-    if (shouldSend && this.voiceTranscript.trim() !== "") {
-      this.send("followUp");
-      return;
-    }
-    void this.startVoiceTurn(false);
+    this.voicePrefix = this.draft.trim();
+    this.voiceTranscript = "";
+    this.azureSpeech.resumeRecognition();
   }
 
   private voiceText(interim = ""): string {

src/client/src/voiceRecognitionLifecycle.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { transitionVoiceRecognition } from "./voiceRecognitionLifecycle";
+
+describe("voice recognition lifecycle", () => {
+  it("keeps the microphone allocated while a final turn waits for TTS", () => {
+    expect(transitionVoiceRecognition("idle", "start")).toBe("listening");
+    expect(transitionVoiceRecognition("listening", "final")).toBe("paused");
+    expect(transitionVoiceRecognition("paused", "resume")).toBe("listening");
+  });
+
+  it("ignores duplicate finals until the response resumes listening", () => {
+    expect(transitionVoiceRecognition("paused", "final")).toBe("paused");
+  });
+
+  it("stops from every active phase", () => {
+    expect(transitionVoiceRecognition("listening", "stop")).toBe("idle");
+    expect(transitionVoiceRecognition("paused", "stop")).toBe("idle");
+  });
+});

src/client/src/voiceRecognitionLifecycle.ts

@@ -0,0 +1,14 @@
+export type VoiceRecognitionPhase = "idle" | "listening" | "paused";
+export type VoiceRecognitionEvent = "start" | "final" | "resume" | "stop";
+
+/**
+ * The microphone remains allocated across paused turns. A final result is
+ * accepted once only; subsequent events are ignored until TTS resumes it.
+ */
+export function transitionVoiceRecognition(phase: VoiceRecognitionPhase, event: VoiceRecognitionEvent): VoiceRecognitionPhase {
+  if (event === "stop") return "idle";
+  if (event === "start" && phase === "idle") return "listening";
+  if (event === "final" && phase === "listening") return "paused";
+  if (event === "resume" && phase === "paused") return "listening";
+  return phase;
+}