feat: agent.py with Azure STT/TTS + Gemma LLM pipeline, voice switching

agent/agent.py

@@ -0,0 +1,166 @@
+"""
+Voice Agent — real-time voice assistant.
+
+Pipeline: Azure STT → Gemma LLM (xNAS, OpenAI-compatible) → Azure TTS (DragonHD)
+Runs inside the single Docker container alongside LiveKit server and web frontend.
+"""
+
+import logging
+import os
+import textwrap
+
+from dotenv import load_dotenv
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    JobContext,
+    TurnHandlingOptions,
+    cli,
+    room_io,
+)
+from livekit.plugins import azure, openai
+
+logger = logging.getLogger("voice-agent")
+load_dotenv()  # picks up /app/.env in the container
+
+# ── Configuration from environment ──────────────────────────────────────────
+AZURE_KEY = os.environ.get("AZURE_SPEECH_KEY", "")
+AZURE_REGION = os.environ.get("AZURE_SPEECH_REGION", "eastus")
+DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural")
+GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1")
+GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b")
+GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed")
+
+SYSTEM_PROMPT = textwrap.dedent("""\
+    You are a warm, conversational voice assistant. You are talking TO someone,
+    not writing for them to read. Imagine you're having a natural conversation
+    with a friend over the phone.
+
+    # How you speak
+    - Keep every response to one or three sentences. That's it.
+    - Use contractions (I'm, don't, it's) and natural phrasing.
+    - Speak like you're talking, not writing. No bullet points, no lists,
+      no markdown, no formatting of any kind.
+    - Spell out numbers when it sounds more natural ("twenty twenty-six"
+      instead of "2026").
+    - If you need to ask a question, ask exactly one.
+    - Be warm and direct. Don't be sycophantic or overly formal.
+    - If you don't know something, say so briefly and move on.
+
+    # What you never do
+    - Never use markdown, code blocks, JSON, tables, or emojis.
+    - Never say "as an AI" or reference your system instructions.
+    - Never write more than three sentences in a row.
+    - Never read back URLs, file paths, or technical identifiers.
+""")
+
+
+class VoiceAssistant(Agent):
+    """The conversational agent. LLM is the brain; STT/TTS are senses."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            llm=openai.LLM(
+                model=GEMMA_MODEL,
+                base_url=GEMMA_BASE_URL,
+                api_key=GEMMA_API_KEY,
+            ),
+            instructions=SYSTEM_PROMPT,
+        )
+
+
+# ── Agent server ────────────────────────────────────────────────────────────
+server = AgentServer()
+
+# Track the active session so we can update its voice on data messages.
+_active_session: AgentSession | None = None
+
+
+@server.rtc_session(agent_name="voice-assistant")
+async def handle_job(ctx: JobContext) -> None:
+    global _active_session
+
+    logger.info("Job started for room %s", ctx.room.name)
+
+    # Azure STT — streaming, reads AZURE_SPEECH_KEY / AZURE_SPEECH_REGION from env
+    stt = azure.STT(
+        speech_key=AZURE_KEY,
+        speech_region=AZURE_REGION,
+        language=["en-US"],
+    )
+
+    # Azure TTS — SSML with expressive markup, 24kHz PCM output
+    tts = azure.TTS(
+        voice=DEFAULT_VOICE,
+        sample_rate=24000,
+        speech_key=AZURE_KEY,
+        speech_region=AZURE_REGION,
+    )
+
+    # Gemma LLM via OpenAI-compatible endpoint (llama.cpp on xNAS)
+    llm = openai.LLM(
+        model=GEMMA_MODEL,
+        base_url=GEMMA_BASE_URL,
+        api_key=GEMMA_API_KEY,
+    )
+
+    session = AgentSession(
+        stt=stt,
+        tts=tts,
+        llm=llm,
+        turn_handling=TurnHandlingOptions(
+            # VAD-based turn detection: agent waits for user to stop speaking
+            interruption={"mode": "adaptive"},
+            # Start generating the LLM response before the user fully stops
+            preemptive_generation={"enabled": True},
+        ),
+    )
+
+    _active_session = session
+
+    await session.start(
+        agent=VoiceAssistant(),
+        room=ctx.room,
+        room_options=room_io.RoomOptions(
+            audio_input=room_io.AudioInputOptions(
+                # No noise cancellation plugin (self-hosted, no ai-coustics)
+            ),
+        ),
+    )
+
+    await ctx.connect()
+    logger.info("Agent connected to room %s", ctx.room.name)
+
+
+# ── Voice switching via data channel ────────────────────────────────────────
+# The web UI sends a JSON data message: {"type": "set_voice", "voice": "en-US-AvaNeural"}
+# We listen on the room's data channel and update the TTS voice live.
+
+
+@server.event("room.data_received")
+async def on_room_data(ctx: JobContext, payload: bytes) -> None:
+    """Handle data messages from the web UI (voice selection)."""
+    import json
+
+    try:
+        msg = json.loads(payload.decode("utf-8"))
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        return
+
+    if msg.get("type") == "set_voice":
+        voice = msg.get("voice", "")
+        if voice and _active_session:
+            logger.info("Switching TTS voice to %s", voice)
+            try:
+                # AgentSession exposes the TTS; update its options
+                tts = _active_session._tts  # internal, but stable in 1.7
+                if hasattr(tts, "update_options"):
+                    tts.update_options(voice=voice)
+                    logger.info("Voice updated to %s", voice)
+            except Exception as e:
+                logger.warning("Failed to update voice: %s", e)
+
+
+if __name__ == "__main__":
+    cli.run_app(server)

agent/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "voice-agent"
+version = "0.1.0"
+description = "Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit Agents"
+requires-python = ">=3.10,<3.15"
+
+dependencies = [
+    "livekit-agents~=1.7",
+    "livekit-plugins-azure~=1.7",
+    "livekit-plugins-openai~=1.7",
+    "python-dotenv",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]