diff --git a/docs/voice-api.md b/docs/voice-api.md index 9a24578..471099b 100644 --- a/docs/voice-api.md +++ b/docs/voice-api.md @@ -93,7 +93,7 @@ Azure push-stream recognition emits `transcript.partial` and `transcript.final` ### Turn output and lifecycle -For either input form, the server emits `agent.working`, then `agent.accepted` after Pi accepted the prompt. While it waits for Pi's native `agent.settled` event (not merely `agent.end`), it repeats the existing `agent.working` frame every 15 seconds; those progress frames stop before `assistant.final` and never overlap synthesized audio. The wait has a two-minute turn timeout. `assistant.delta` is streamed while Pi answers; `assistant.final` contains only assistant text, never STT transcript. +For either input form, the server emits `agent.working`, then `agent.accepted` after Pi accepted the prompt. If work is still in progress after two seconds, it sends one `agent.progress` event with a short spoken status clip (for example, “Hang on while I work on that.”). It never repeats this spoken update within the same turn; ordinary screen-only `agent.working` heartbeats continue every 15 seconds. While it waits for Pi's native `agent.settled` event (not merely `agent.end`), it repeats the existing `agent.working` frame every 15 seconds; those progress frames stop before `assistant.final` and never overlap synthesized audio. The wait has a two-minute turn timeout. `assistant.delta` is streamed while Pi answers; `assistant.final` contains only assistant text, never STT transcript. Azure synthesis output is signed little-endian 16-bit, 24 kHz, mono PCM. Every binary server frame has this eight-byte header followed by PCM: diff --git a/src/server/voiceApi.test.ts b/src/server/voiceApi.test.ts index 3d34fb3..9781de5 100644 --- a/src/server/voiceApi.test.ts +++ b/src/server/voiceApi.test.ts @@ -119,6 +119,20 @@ describe("voice API", () => { expect(frame.subarray(8)).toEqual(Buffer.from([1, 2, 3, 4])); }); + it("does not announce progress for a quick completed turn", () => { + vi.useFakeTimers(); + try { + const socket: Pick = { readyState: WebSocket.OPEN, send: () => {} }; + const turn: { closed: boolean; workingTimer: ReturnType | undefined; progressTimer?: ReturnType; progressAnnounced?: boolean } = { closed: false, workingTimer: undefined }; + const progress = vi.fn(); + startWorkingProgress(socket, turn, progress); + vi.advanceTimersByTime(1_000); + clearWorkingProgress(turn); + vi.advanceTimersByTime(5_000); + expect(progress).not.toHaveBeenCalled(); + } finally { vi.useRealTimers(); } + }); + it("repeats working progress every 15 seconds and stops it before final/audio", () => { vi.useFakeTimers(); try { @@ -130,16 +144,19 @@ describe("voice API", () => { sent.push(JSON.parse(value)); }, }; - const turn: { closed: boolean; workingTimer: ReturnType | undefined } = { + const turn: { closed: boolean; workingTimer: ReturnType | undefined; progressTimer?: ReturnType; progressAnnounced?: boolean } = { closed: false, workingTimer: undefined, }; + const progress = vi.fn(); - startWorkingProgress(socket, turn); - vi.advanceTimersByTime(45_000); + startWorkingProgress(socket, turn, progress); + vi.advanceTimersByTime(2_000); + vi.advanceTimersByTime(43_000); clearWorkingProgress(turn); vi.advanceTimersByTime(30_000); + expect(progress).toHaveBeenCalledTimes(1); expect(sent).toEqual([ { type: "agent.working" }, { type: "agent.working" }, diff --git a/src/server/voiceApi.ts b/src/server/voiceApi.ts index d19e4a8..1b5dd56 100644 --- a/src/server/voiceApi.ts +++ b/src/server/voiceApi.ts @@ -30,6 +30,7 @@ const OUTPUT_CHUNK = 16 * 1024; const EVENT_OPEN_TIMEOUT_MS = 10_000; const TURN_TIMEOUT_MS = 120_000; const WORKING_PROGRESS_INTERVAL_MS = 15_000; +const SPOKEN_PROGRESS_DELAY_MS = 2_000; const DEVICE_WORKSPACE_ROOT = "/home/hope/workspaces"; const VOICE_CONVERSATION_INSTRUCTIONS = "This is a voice conversation. Respond in natural, concise spoken language suitable for Azure DragonHD text-to-speech. Do not use Markdown, headings, tables, bullet lists, code fences, URLs unless explicitly requested, or visual-only references. Speak punctuation naturally and expand ambiguous symbols when useful."; interface TokenRecord { @@ -98,6 +99,8 @@ interface SocketTurn { recognizer?: VoiceRecognizer; eventSocket: WebSocket | undefined; workingTimer: ReturnType | undefined; + progressTimer: ReturnType | undefined; + progressAnnounced: boolean | undefined; /** VAD/STT result only; never use it as an assistant response. */ transcript: string; /** Assistant output collected from Pi's session event stream only. */ @@ -415,6 +418,8 @@ function wireVoiceSocket( closed: false, eventSocket: undefined, workingTimer: undefined, + progressTimer: undefined, + progressAnnounced: false, }; sendJson(socket, { type: "hello", @@ -583,7 +588,12 @@ async function submitText( conversation.status = "working"; turn.assistant = ""; sendJson(socket, { type: "agent.working" }); - startWorkingProgress(socket, turn); + startWorkingProgress(socket, turn, () => { + sendJson(socket, { type: "agent.progress", text: "Hang on while I work on that.", audio: true }); + void speech.synthesize("Hang on while I work on that.", (pcm) => { + if (!turn.closed) sendAudio(socket, turn, pcm); + }); + }); try { await withTimeout( runTurn(socket, turn, conversation, deps, speech, text), @@ -721,7 +731,8 @@ function waitForWebSocketOpen(socket: WebSocket): Promise { export function startWorkingProgress( socket: Pick, - turn: Pick + turn: Pick & Partial>, + onSpokenProgress?: () => void ): void { clearWorkingProgress(turn); const timer = setInterval(() => { @@ -730,13 +741,23 @@ export function startWorkingProgress( }, WORKING_PROGRESS_INTERVAL_MS); timer.unref(); turn.workingTimer = timer; + if (onSpokenProgress !== undefined) { + turn.progressTimer = setTimeout(() => { + if (turn.closed || turn.progressAnnounced === true) return; + turn.progressAnnounced = true; + onSpokenProgress(); + }, SPOKEN_PROGRESS_DELAY_MS); + turn.progressTimer.unref(); + } } export function clearWorkingProgress( - turn: Pick + turn: Pick & Partial> ): void { if (turn.workingTimer !== undefined) clearInterval(turn.workingTimer); + if (turn.progressTimer !== undefined) clearTimeout(turn.progressTimer); turn.workingTimer = undefined; + turn.progressTimer = undefined; } function closeEventSocket(turn: SocketTurn): void {