From c5f7bb39e7c71597049aeb41ddd07505d1366f52 Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Thu, 1 Oct 2026 15:59:10 -0400 Subject: [PATCH] fix: delay and vary spoken wait notices --- QUALIFICATION.md | 10 +++-- agent/hermes_llm.py | 24 ++++++++--- tests/unit/test_hermes_llm.py | 81 ++++++++++++++++++++++++++++++++--- 3 files changed, 99 insertions(+), 16 deletions(-) diff --git a/QUALIFICATION.md b/QUALIFICATION.md index 446d94e..2c04c38 100644 --- a/QUALIFICATION.md +++ b/QUALIFICATION.md @@ -90,9 +90,13 @@ also passes `model_options.reasoning.enabled: false` and forwards content only. all runtime secrets and user data. - Hermes approvals have no interactive voice approval transport: fail explicitly rather than hanging or executing/retrying silently. Tool actions are not replayed. -- Long requests emit progress after three seconds/tool start. Spoken wait notices - use independent audio, not `AgentSession.say` which queues behind the response. - First answer cancels notice playback; notice is not added to conversation history. +- Status/tool events provide visual progress immediately. Spoken wait notices + occur only after twelve seconds without answer content, once per request, with + varied wording that never repeats back-to-back within a conversation. Status + and tool-start events alone do not trigger speech. Notices use independent + audio, not `AgentSession.say` which queues behind the response. First answer + cancels notice playback; notice is not added to conversation history. + This revised threshold is covered offline; the live receipts above predate it. - Runtime LiveKit ICE candidates are restricted to Unraid LAN `br0` and loopback with node IP `192.168.86.2`, avoiding dozens of unreachable Docker bridge candidates. Adjust the private runtime YAML for a different host/interface. diff --git a/agent/hermes_llm.py b/agent/hermes_llm.py index 2a29364..0cd63c2 100644 --- a/agent/hermes_llm.py +++ b/agent/hermes_llm.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio import json import math +import random import uuid from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import replace @@ -24,7 +25,13 @@ from livekit.agents.types import ( ProgressCallback = Callable[[dict[str, Any]], Awaitable[None]] WaitNoticeCallback = Callable[[str], Awaitable[None]] -WAIT_NOTICE = "I'm working on that. I'll let you know when it's ready." +WAIT_NOTICE_SECONDS = 12.0 +WAIT_NOTICES = ( + "I'm still working on that. I'll let you know when it's ready.", + "This is taking a little longer. I'm on it.", + "I'm still looking into that. I'll let you know what I find.", + "I need a little more time on this. I'm still on it.", +) class HermesLLMError(APIError): @@ -119,6 +126,7 @@ class HermesLLM(lk_llm.LLM): } self._on_progress = on_progress self._on_wait_notice = on_wait_notice + self._last_wait_notice: str | None = None self._on_answer_start = on_answer_start self._timeout_seconds = timeout_seconds self._owns_client = http_client is None @@ -176,10 +184,17 @@ class HermesLLMStream(lk_llm.LLMStream): if self._noticed or self._has_answer or self._hermes._on_wait_notice is None: return self._noticed = True # latch before awaiting, including competing timer - await self._callback(self._hermes._on_wait_notice, WAIT_NOTICE) + notice = random.choice([ + text for text in WAIT_NOTICES if text != self._hermes._last_wait_notice + ]) + self._hermes._last_wait_notice = notice + await self._callback(self._hermes._on_wait_notice, notice) async def _latency_notice(self) -> None: - await asyncio.sleep(3.0) + # Status/tool-start events are not predictions of long latency: Hermes + # emits them even for ordinary replies. Only elapsed time without an + # answer warrants spoken feedback; UI progress remains independent. + await asyncio.sleep(WAIT_NOTICE_SECONDS) if not self._has_answer: await self._progress("working", "Working on your request.") await self._notice() @@ -244,7 +259,6 @@ class HermesLLMStream(lk_llm.LLMStream): if item.get("status") in ("failed", "error"): raise HermesLLMError("upstream_error") await self._progress("working", "Working on your request.") - await self._notice() continue if event == "hermes.tool.progress": status = item.get("status", "working") @@ -255,8 +269,6 @@ class HermesLLMStream(lk_llm.LLMStream): name = item.get("tool_name", item.get("tool", item.get("name"))) message = f"Working on {name}." if isinstance(name, str) else "Working on your request." await self._progress("working", message) - if status not in ("completed", "complete", "success", "finished"): - await self._notice() continue choices = item.get("choices", []) if not isinstance(choices, list): diff --git a/tests/unit/test_hermes_llm.py b/tests/unit/test_hermes_llm.py index 7f80815..fd4ca03 100644 --- a/tests/unit/test_hermes_llm.py +++ b/tests/unit/test_hermes_llm.py @@ -17,7 +17,7 @@ from livekit.agents import llm as lk_llm from livekit.agents.types import APIConnectOptions sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "agent")) -from hermes_llm import HermesLLM, HermesLLMError, WAIT_NOTICE +from hermes_llm import HermesLLM, HermesLLMError, WAIT_NOTICES, WAIT_NOTICE_SECONDS def data(delta, **kwargs): @@ -113,14 +113,14 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(self.progress[-1]["request_id"], chunks[0].id) self.assertTrue(fake.closed) - async def test_sse_crlf_unicode_multiline_comments_unknown_events_and_tool_notice_once(self): + async def test_sse_crlf_unicode_multiline_comments_unknown_events_and_quick_tools_silent(self): tool = b'event: hermes.tool.progress\r\ndata: {"status":"started",\r\ndata: "tool_name":"search"}\r\n\r\n' body = b":keepalive\r\n\r\nevent: unknown\r\ndata: not json\r\n\r\n" + tool + tool + data({"reasoning": "never speak"}) + data({"content": "caf\u00e9"}) + b"data: [DONE]\r\n\r\n" fake = FakeSSE([body[i:i + 1] for i in range(len(body))]) bridge = self.bridge(fake) result = await bridge.chat(chat_ctx=self.context()).collect() self.assertEqual(result.text, "caf\u00e9") - self.assertEqual(self.notices, [WAIT_NOTICE]) + self.assertEqual(self.notices, []) self.assertEqual([event["status"] for event in self.progress], ["working", "working", "completed"]) self.assertTrue(all(event["type"] == "assistant_progress" for event in self.progress)) self.assertEqual(len({event["request_id"] for event in self.progress}), 1) @@ -133,10 +133,11 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): async def test_latency_notice_without_network_event(self): fake = FakeSSE([], gate=asyncio.Event()) bridge = self.bridge(fake) - # Speed up only the bridge's three-second timer; network remains blocked. + # Speed up only the twelve-second timer; network remains blocked. sleep = asyncio.sleep async def fast_sleep(seconds): - self.assertEqual(seconds, 3.0) + self.assertEqual(seconds, 12.0) + self.assertEqual(seconds, WAIT_NOTICE_SECONDS) await sleep(0) with patch("hermes_llm.asyncio.sleep", side_effect=fast_sleep): stream = bridge.chat(chat_ctx=self.context()) @@ -145,13 +146,79 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): await sleep(0) if self.notices: break - self.assertEqual(self.notices, [WAIT_NOTICE]) + self.assertEqual(len(self.notices), 1) + self.assertIn(self.notices[0], WAIT_NOTICES) self.assertEqual(self.progress[0]["status"], "working") self.assertEqual(self.progress[0]["message"], "Working on your request.") await stream.aclose() self.assertTrue(fake.closed) self.assertEqual(self.progress[-1]["status"], "failed") + async def test_status_and_tools_wait_for_threshold_then_notice_once(self): + threshold = asyncio.Event() + noticed = asyncio.Event() + progress_seen = asyncio.Event() + tool = b'event: hermes.tool.progress\ndata: {"status":"started","tool_name":"search"}\n\n' + fake = FakeSSE([b'event: hermes.status\ndata: {"status":"working"}\n\n', tool, tool], gate=asyncio.Event()) + bridge = self.bridge(fake) + + async def timer_sleep(seconds): + self.assertEqual(seconds, 12.0) + await threshold.wait() + + async def progress(event): + await self.progress_callback(event) + if len(self.progress) == 3: + progress_seen.set() + + async def notice(text): + await self.notice_callback(text) + noticed.set() + + bridge._on_progress = progress + bridge._on_wait_notice = notice + with patch("hermes_llm.asyncio.sleep", side_effect=timer_sleep): + stream = bridge.chat(chat_ctx=self.context()) + try: + await asyncio.wait_for(progress_seen.wait(), timeout=1) + self.assertEqual(self.notices, []) + threshold.set() + await asyncio.wait_for(noticed.wait(), timeout=1) + await stream._notice() # repeated progress cannot duplicate it + self.assertEqual(len(self.notices), 1) + self.assertIn(self.notices[0], WAIT_NOTICES) + finally: + await stream.aclose() + + async def test_delayed_notices_vary_across_turns_and_stay_out_of_history(self): + bridge = self.bridge(FakeSSE([], gate=asyncio.Event())) + sleep = asyncio.sleep + async def fast_sleep(seconds): + self.assertEqual(seconds, 12.0) + await sleep(0) + + with patch("hermes_llm.asyncio.sleep", side_effect=fast_sleep): + for turn in range(8): + noticed = asyncio.Event() + async def notice(text): + await self.notice_callback(text) + noticed.set() + bridge._on_wait_notice = notice + context = self.context() + before = context.to_dict() + stream = bridge.chat(chat_ctx=context) + try: + await asyncio.wait_for(noticed.wait(), timeout=1) + self.assertEqual(len(self.notices), turn + 1) + self.assertIn(self.notices[-1], WAIT_NOTICES) + if turn: + self.assertNotEqual(self.notices[-1], self.notices[-2]) + self.assertEqual(context.to_dict(), before) + messages = json.loads(self.requests[-1].content)["messages"] + self.assertFalse(any(m["content"] in WAIT_NOTICES for m in messages)) + finally: + await stream.aclose() + async def test_timeout_closes_stream_and_never_retries(self): fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event()) stream = self.bridge(fake, timeout_seconds=0.02).chat(chat_ctx=self.context(), conn_options=APIConnectOptions(max_retry=10)) @@ -247,7 +314,7 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase): result = await self.bridge(fake).chat(chat_ctx=self.context()).collect() self.assertEqual(result.text, "Done") self.assertEqual(self.progress[0]["message"], "Working on your request.") - self.assertEqual(self.notices, [WAIT_NOTICE]) + self.assertEqual(self.notices, []) async def test_explicit_cancellation_closes_http_without_retry(self): fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event())