feat: speak a progress update during long voice turns

After two seconds of agent work, send one agent.progress frame with a
synthesized "Hang on while I work on that." clip over the same binary
audio channel (shared output-sequence counter), at most once per turn;
screen-only agent.working heartbeats continue every 15 seconds. This is
the change the 15:11 rebuild already shipped to the running server —
committing it pins the protocol addition to the client that now
tolerates and plays it.

Co-Authored-By: Claude Fable 5 <[email protected]>

docs/voice-api.md

@@ -93,7 +93,7 @@ Azure push-stream recognition emits `transcript.partial` and `transcript.final`
 
 ### Turn output and lifecycle
 
-For either input form, the server emits `agent.working`, then `agent.accepted` after Pi accepted the prompt. While it waits for Pi's native `agent.settled` event (not merely `agent.end`), it repeats the existing `agent.working` frame every 15 seconds; those progress frames stop before `assistant.final` and never overlap synthesized audio. The wait has a two-minute turn timeout. `assistant.delta` is streamed while Pi answers; `assistant.final` contains only assistant text, never STT transcript.
+For either input form, the server emits `agent.working`, then `agent.accepted` after Pi accepted the prompt. If work is still in progress after two seconds, it sends one `agent.progress` event with a short spoken status clip (for example, “Hang on while I work on that.”). It never repeats this spoken update within the same turn; ordinary screen-only `agent.working` heartbeats continue every 15 seconds. While it waits for Pi's native `agent.settled` event (not merely `agent.end`), it repeats the existing `agent.working` frame every 15 seconds; those progress frames stop before `assistant.final` and never overlap synthesized audio. The wait has a two-minute turn timeout. `assistant.delta` is streamed while Pi answers; `assistant.final` contains only assistant text, never STT transcript.
 
 Azure synthesis output is signed little-endian 16-bit, 24 kHz, mono PCM. Every binary server frame has this eight-byte header followed by PCM:
 

src/server/voiceApi.test.ts

@@ -119,6 +119,20 @@ describe("voice API", () => {
     expect(frame.subarray(8)).toEqual(Buffer.from([1, 2, 3, 4]));
   });
 
+  it("does not announce progress for a quick completed turn", () => {
+    vi.useFakeTimers();
+    try {
+      const socket: Pick<WebSocket, "readyState" | "send"> = { readyState: WebSocket.OPEN, send: () => {} };
+      const turn: { closed: boolean; workingTimer: ReturnType<typeof setInterval> | undefined; progressTimer?: ReturnType<typeof setTimeout>; progressAnnounced?: boolean } = { closed: false, workingTimer: undefined };
+      const progress = vi.fn();
+      startWorkingProgress(socket, turn, progress);
+      vi.advanceTimersByTime(1_000);
+      clearWorkingProgress(turn);
+      vi.advanceTimersByTime(5_000);
+      expect(progress).not.toHaveBeenCalled();
+    } finally { vi.useRealTimers(); }
+  });
+
   it("repeats working progress every 15 seconds and stops it before final/audio", () => {
     vi.useFakeTimers();
     try {
@@ -130,16 +144,19 @@ describe("voice API", () => {
           sent.push(JSON.parse(value));
         },
       };
-      const turn: { closed: boolean; workingTimer: ReturnType<typeof setInterval> | undefined } = {
+      const turn: { closed: boolean; workingTimer: ReturnType<typeof setInterval> | undefined; progressTimer?: ReturnType<typeof setTimeout>; progressAnnounced?: boolean } = {
         closed: false,
         workingTimer: undefined,
       };
+      const progress = vi.fn();
 
-      startWorkingProgress(socket, turn);
+      startWorkingProgress(socket, turn, progress);
-      vi.advanceTimersByTime(45_000);
+      vi.advanceTimersByTime(2_000);
+      vi.advanceTimersByTime(43_000);
       clearWorkingProgress(turn);
       vi.advanceTimersByTime(30_000);
 
+      expect(progress).toHaveBeenCalledTimes(1);
       expect(sent).toEqual([
         { type: "agent.working" },
         { type: "agent.working" },

src/server/voiceApi.ts

@@ -30,6 +30,7 @@ const OUTPUT_CHUNK = 16 * 1024;
 const EVENT_OPEN_TIMEOUT_MS = 10_000;
 const TURN_TIMEOUT_MS = 120_000;
 const WORKING_PROGRESS_INTERVAL_MS = 15_000;
+const SPOKEN_PROGRESS_DELAY_MS = 2_000;
 const DEVICE_WORKSPACE_ROOT = "/home/hope/workspaces";
 const VOICE_CONVERSATION_INSTRUCTIONS = "This is a voice conversation. Respond in natural, concise spoken language suitable for Azure DragonHD text-to-speech. Do not use Markdown, headings, tables, bullet lists, code fences, URLs unless explicitly requested, or visual-only references. Speak punctuation naturally and expand ambiguous symbols when useful.";
 interface TokenRecord {
@@ -98,6 +99,8 @@ interface SocketTurn {
   recognizer?: VoiceRecognizer;
   eventSocket: WebSocket | undefined;
   workingTimer: ReturnType<typeof setInterval> | undefined;
+  progressTimer: ReturnType<typeof setTimeout> | undefined;
+  progressAnnounced: boolean | undefined;
   /** VAD/STT result only; never use it as an assistant response. */
   transcript: string;
   /** Assistant output collected from Pi's session event stream only. */
@@ -415,6 +418,8 @@ function wireVoiceSocket(
     closed: false,
     eventSocket: undefined,
     workingTimer: undefined,
+    progressTimer: undefined,
+    progressAnnounced: false,
   };
   sendJson(socket, {
     type: "hello",
@@ -583,7 +588,12 @@ async function submitText(
   conversation.status = "working";
   turn.assistant = "";
   sendJson(socket, { type: "agent.working" });
-  startWorkingProgress(socket, turn);
+  startWorkingProgress(socket, turn, () => {
+    sendJson(socket, { type: "agent.progress", text: "Hang on while I work on that.", audio: true });
+    void speech.synthesize("Hang on while I work on that.", (pcm) => {
+      if (!turn.closed) sendAudio(socket, turn, pcm);
+    });
+  });
   try {
     await withTimeout(
       runTurn(socket, turn, conversation, deps, speech, text),
@@ -721,7 +731,8 @@ function waitForWebSocketOpen(socket: WebSocket): Promise<void> {
 
 export function startWorkingProgress(
   socket: Pick<WebSocket, "readyState" | "send">,
-  turn: Pick<SocketTurn, "closed" | "workingTimer">
+  turn: Pick<SocketTurn, "closed" | "workingTimer"> & Partial<Pick<SocketTurn, "progressTimer" | "progressAnnounced">>,
+  onSpokenProgress?: () => void
 ): void {
   clearWorkingProgress(turn);
   const timer = setInterval(() => {
@@ -730,13 +741,23 @@ export function startWorkingProgress(
   }, WORKING_PROGRESS_INTERVAL_MS);
   timer.unref();
   turn.workingTimer = timer;
+  if (onSpokenProgress !== undefined) {
+    turn.progressTimer = setTimeout(() => {
+      if (turn.closed || turn.progressAnnounced === true) return;
+      turn.progressAnnounced = true;
+      onSpokenProgress();
+    }, SPOKEN_PROGRESS_DELAY_MS);
+    turn.progressTimer.unref();
+  }
 }
 
 export function clearWorkingProgress(
-  turn: Pick<SocketTurn, "workingTimer">
+  turn: Pick<SocketTurn, "workingTimer"> & Partial<Pick<SocketTurn, "progressTimer">>
 ): void {
   if (turn.workingTimer !== undefined) clearInterval(turn.workingTimer);
+  if (turn.progressTimer !== undefined) clearTimeout(turn.progressTimer);
   turn.workingTimer = undefined;
+  turn.progressTimer = undefined;
 }
 
 function closeEventSocket(turn: SocketTurn): void {