fix: delay and vary spoken wait notices

QUALIFICATION.md

@@ -90,9 +90,13 @@ also passes `model_options.reasoning.enabled: false` and forwards content only.
   all runtime secrets and user data.
 - Hermes approvals have no interactive voice approval transport: fail explicitly
   rather than hanging or executing/retrying silently. Tool actions are not replayed.
-- Long requests emit progress after three seconds/tool start. Spoken wait notices
-  use independent audio, not `AgentSession.say` which queues behind the response.
-  First answer cancels notice playback; notice is not added to conversation history.
+- Status/tool events provide visual progress immediately. Spoken wait notices
+  occur only after twelve seconds without answer content, once per request, with
+  varied wording that never repeats back-to-back within a conversation. Status
+  and tool-start events alone do not trigger speech. Notices use independent
+  audio, not `AgentSession.say` which queues behind the response. First answer
+  cancels notice playback; notice is not added to conversation history.
+  This revised threshold is covered offline; the live receipts above predate it.
 - Runtime LiveKit ICE candidates are restricted to Unraid LAN `br0` and loopback
   with node IP `192.168.86.2`, avoiding dozens of unreachable Docker bridge candidates.
   Adjust the private runtime YAML for a different host/interface.

agent/hermes_llm.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 import asyncio
 import json
 import math
+import random
 import uuid
 from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import replace
@@ -24,7 +25,13 @@ from livekit.agents.types import (
 
 ProgressCallback = Callable[[dict[str, Any]], Awaitable[None]]
 WaitNoticeCallback = Callable[[str], Awaitable[None]]
-WAIT_NOTICE = "I'm working on that. I'll let you know when it's ready."
+WAIT_NOTICE_SECONDS = 12.0
+WAIT_NOTICES = (
+    "I'm still working on that. I'll let you know when it's ready.",
+    "This is taking a little longer. I'm on it.",
+    "I'm still looking into that. I'll let you know what I find.",
+    "I need a little more time on this. I'm still on it.",
+)
 
 
 class HermesLLMError(APIError):
@@ -119,6 +126,7 @@ class HermesLLM(lk_llm.LLM):
         }
         self._on_progress = on_progress
         self._on_wait_notice = on_wait_notice
+        self._last_wait_notice: str | None = None
         self._on_answer_start = on_answer_start
         self._timeout_seconds = timeout_seconds
         self._owns_client = http_client is None
@@ -176,10 +184,17 @@ class HermesLLMStream(lk_llm.LLMStream):
         if self._noticed or self._has_answer or self._hermes._on_wait_notice is None:
             return
         self._noticed = True  # latch before awaiting, including competing timer
-        await self._callback(self._hermes._on_wait_notice, WAIT_NOTICE)
+        notice = random.choice([
+            text for text in WAIT_NOTICES if text != self._hermes._last_wait_notice
+        ])
+        self._hermes._last_wait_notice = notice
+        await self._callback(self._hermes._on_wait_notice, notice)
 
     async def _latency_notice(self) -> None:
-        await asyncio.sleep(3.0)
+        # Status/tool-start events are not predictions of long latency: Hermes
+        # emits them even for ordinary replies. Only elapsed time without an
+        # answer warrants spoken feedback; UI progress remains independent.
+        await asyncio.sleep(WAIT_NOTICE_SECONDS)
         if not self._has_answer:
             await self._progress("working", "Working on your request.")
             await self._notice()
@@ -244,7 +259,6 @@ class HermesLLMStream(lk_llm.LLMStream):
                     if item.get("status") in ("failed", "error"):
                         raise HermesLLMError("upstream_error")
                     await self._progress("working", "Working on your request.")
-                    await self._notice()
                     continue
                 if event == "hermes.tool.progress":
                     status = item.get("status", "working")
@@ -255,8 +269,6 @@ class HermesLLMStream(lk_llm.LLMStream):
                         name = item.get("tool_name", item.get("tool", item.get("name")))
                         message = f"Working on {name}." if isinstance(name, str) else "Working on your request."
                     await self._progress("working", message)
-                    if status not in ("completed", "complete", "success", "finished"):
-                        await self._notice()
                     continue
                 choices = item.get("choices", [])
                 if not isinstance(choices, list):

tests/unit/test_hermes_llm.py

@@ -17,7 +17,7 @@ from livekit.agents import llm as lk_llm
 from livekit.agents.types import APIConnectOptions
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "agent"))
-from hermes_llm import HermesLLM, HermesLLMError, WAIT_NOTICE
+from hermes_llm import HermesLLM, HermesLLMError, WAIT_NOTICES, WAIT_NOTICE_SECONDS
 
 
 def data(delta, **kwargs):
@@ -113,14 +113,14 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase):
         self.assertEqual(self.progress[-1]["request_id"], chunks[0].id)
         self.assertTrue(fake.closed)
 
-    async def test_sse_crlf_unicode_multiline_comments_unknown_events_and_tool_notice_once(self):
+    async def test_sse_crlf_unicode_multiline_comments_unknown_events_and_quick_tools_silent(self):
         tool = b'event: hermes.tool.progress\r\ndata: {"status":"started",\r\ndata: "tool_name":"search"}\r\n\r\n'
         body = b":keepalive\r\n\r\nevent: unknown\r\ndata: not json\r\n\r\n" + tool + tool + data({"reasoning": "never speak"}) + data({"content": "caf\u00e9"}) + b"data: [DONE]\r\n\r\n"
         fake = FakeSSE([body[i:i + 1] for i in range(len(body))])
         bridge = self.bridge(fake)
         result = await bridge.chat(chat_ctx=self.context()).collect()
         self.assertEqual(result.text, "caf\u00e9")
-        self.assertEqual(self.notices, [WAIT_NOTICE])
+        self.assertEqual(self.notices, [])
         self.assertEqual([event["status"] for event in self.progress], ["working", "working", "completed"])
         self.assertTrue(all(event["type"] == "assistant_progress" for event in self.progress))
         self.assertEqual(len({event["request_id"] for event in self.progress}), 1)
@@ -133,10 +133,11 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase):
     async def test_latency_notice_without_network_event(self):
         fake = FakeSSE([], gate=asyncio.Event())
         bridge = self.bridge(fake)
-        # Speed up only the bridge's three-second timer; network remains blocked.
+        # Speed up only the twelve-second timer; network remains blocked.
         sleep = asyncio.sleep
         async def fast_sleep(seconds):
-            self.assertEqual(seconds, 3.0)
+            self.assertEqual(seconds, 12.0)
+            self.assertEqual(seconds, WAIT_NOTICE_SECONDS)
             await sleep(0)
         with patch("hermes_llm.asyncio.sleep", side_effect=fast_sleep):
             stream = bridge.chat(chat_ctx=self.context())
@@ -145,13 +146,79 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase):
                 await sleep(0)
                 if self.notices:
                     break
-            self.assertEqual(self.notices, [WAIT_NOTICE])
+            self.assertEqual(len(self.notices), 1)
+            self.assertIn(self.notices[0], WAIT_NOTICES)
             self.assertEqual(self.progress[0]["status"], "working")
             self.assertEqual(self.progress[0]["message"], "Working on your request.")
             await stream.aclose()
         self.assertTrue(fake.closed)
         self.assertEqual(self.progress[-1]["status"], "failed")
 
+    async def test_status_and_tools_wait_for_threshold_then_notice_once(self):
+        threshold = asyncio.Event()
+        noticed = asyncio.Event()
+        progress_seen = asyncio.Event()
+        tool = b'event: hermes.tool.progress\ndata: {"status":"started","tool_name":"search"}\n\n'
+        fake = FakeSSE([b'event: hermes.status\ndata: {"status":"working"}\n\n', tool, tool], gate=asyncio.Event())
+        bridge = self.bridge(fake)
+
+        async def timer_sleep(seconds):
+            self.assertEqual(seconds, 12.0)
+            await threshold.wait()
+
+        async def progress(event):
+            await self.progress_callback(event)
+            if len(self.progress) == 3:
+                progress_seen.set()
+
+        async def notice(text):
+            await self.notice_callback(text)
+            noticed.set()
+
+        bridge._on_progress = progress
+        bridge._on_wait_notice = notice
+        with patch("hermes_llm.asyncio.sleep", side_effect=timer_sleep):
+            stream = bridge.chat(chat_ctx=self.context())
+            try:
+                await asyncio.wait_for(progress_seen.wait(), timeout=1)
+                self.assertEqual(self.notices, [])
+                threshold.set()
+                await asyncio.wait_for(noticed.wait(), timeout=1)
+                await stream._notice()  # repeated progress cannot duplicate it
+                self.assertEqual(len(self.notices), 1)
+                self.assertIn(self.notices[0], WAIT_NOTICES)
+            finally:
+                await stream.aclose()
+
+    async def test_delayed_notices_vary_across_turns_and_stay_out_of_history(self):
+        bridge = self.bridge(FakeSSE([], gate=asyncio.Event()))
+        sleep = asyncio.sleep
+        async def fast_sleep(seconds):
+            self.assertEqual(seconds, 12.0)
+            await sleep(0)
+
+        with patch("hermes_llm.asyncio.sleep", side_effect=fast_sleep):
+            for turn in range(8):
+                noticed = asyncio.Event()
+                async def notice(text):
+                    await self.notice_callback(text)
+                    noticed.set()
+                bridge._on_wait_notice = notice
+                context = self.context()
+                before = context.to_dict()
+                stream = bridge.chat(chat_ctx=context)
+                try:
+                    await asyncio.wait_for(noticed.wait(), timeout=1)
+                    self.assertEqual(len(self.notices), turn + 1)
+                    self.assertIn(self.notices[-1], WAIT_NOTICES)
+                    if turn:
+                        self.assertNotEqual(self.notices[-1], self.notices[-2])
+                    self.assertEqual(context.to_dict(), before)
+                    messages = json.loads(self.requests[-1].content)["messages"]
+                    self.assertFalse(any(m["content"] in WAIT_NOTICES for m in messages))
+                finally:
+                    await stream.aclose()
+
     async def test_timeout_closes_stream_and_never_retries(self):
         fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event())
         stream = self.bridge(fake, timeout_seconds=0.02).chat(chat_ctx=self.context(), conn_options=APIConnectOptions(max_retry=10))
@@ -247,7 +314,7 @@ class HermesLLMTests(unittest.IsolatedAsyncioTestCase):
         result = await self.bridge(fake).chat(chat_ctx=self.context()).collect()
         self.assertEqual(result.text, "Done")
         self.assertEqual(self.progress[0]["message"], "Working on your request.")
-        self.assertEqual(self.notices, [WAIT_NOTICE])
+        self.assertEqual(self.notices, [])
 
     async def test_explicit_cancellation_closes_http_without_retry(self):
         fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event())