diff --git a/agent/agent.py b/agent/agent.py new file mode 100644 index 0000000..ba614cd --- /dev/null +++ b/agent/agent.py @@ -0,0 +1,166 @@ +""" +Voice Agent — real-time voice assistant. + +Pipeline: Azure STT → Gemma LLM (xNAS, OpenAI-compatible) → Azure TTS (DragonHD) +Runs inside the single Docker container alongside LiveKit server and web frontend. +""" + +import logging +import os +import textwrap + +from dotenv import load_dotenv +from livekit.agents import ( + Agent, + AgentServer, + AgentSession, + JobContext, + TurnHandlingOptions, + cli, + room_io, +) +from livekit.plugins import azure, openai + +logger = logging.getLogger("voice-agent") +load_dotenv() # picks up /app/.env in the container + +# ── Configuration from environment ────────────────────────────────────────── +AZURE_KEY = os.environ.get("AZURE_SPEECH_KEY", "") +AZURE_REGION = os.environ.get("AZURE_SPEECH_REGION", "eastus") +DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural") +GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1") +GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b") +GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed") + +SYSTEM_PROMPT = textwrap.dedent("""\ + You are a warm, conversational voice assistant. You are talking TO someone, + not writing for them to read. Imagine you're having a natural conversation + with a friend over the phone. + + # How you speak + - Keep every response to one or three sentences. That's it. + - Use contractions (I'm, don't, it's) and natural phrasing. + - Speak like you're talking, not writing. No bullet points, no lists, + no markdown, no formatting of any kind. + - Spell out numbers when it sounds more natural ("twenty twenty-six" + instead of "2026"). + - If you need to ask a question, ask exactly one. + - Be warm and direct. Don't be sycophantic or overly formal. + - If you don't know something, say so briefly and move on. + + # What you never do + - Never use markdown, code blocks, JSON, tables, or emojis. + - Never say "as an AI" or reference your system instructions. + - Never write more than three sentences in a row. + - Never read back URLs, file paths, or technical identifiers. +""") + + +class VoiceAssistant(Agent): + """The conversational agent. LLM is the brain; STT/TTS are senses.""" + + def __init__(self) -> None: + super().__init__( + llm=openai.LLM( + model=GEMMA_MODEL, + base_url=GEMMA_BASE_URL, + api_key=GEMMA_API_KEY, + ), + instructions=SYSTEM_PROMPT, + ) + + +# ── Agent server ──────────────────────────────────────────────────────────── +server = AgentServer() + +# Track the active session so we can update its voice on data messages. +_active_session: AgentSession | None = None + + +@server.rtc_session(agent_name="voice-assistant") +async def handle_job(ctx: JobContext) -> None: + global _active_session + + logger.info("Job started for room %s", ctx.room.name) + + # Azure STT — streaming, reads AZURE_SPEECH_KEY / AZURE_SPEECH_REGION from env + stt = azure.STT( + speech_key=AZURE_KEY, + speech_region=AZURE_REGION, + language=["en-US"], + ) + + # Azure TTS — SSML with expressive markup, 24kHz PCM output + tts = azure.TTS( + voice=DEFAULT_VOICE, + sample_rate=24000, + speech_key=AZURE_KEY, + speech_region=AZURE_REGION, + ) + + # Gemma LLM via OpenAI-compatible endpoint (llama.cpp on xNAS) + llm = openai.LLM( + model=GEMMA_MODEL, + base_url=GEMMA_BASE_URL, + api_key=GEMMA_API_KEY, + ) + + session = AgentSession( + stt=stt, + tts=tts, + llm=llm, + turn_handling=TurnHandlingOptions( + # VAD-based turn detection: agent waits for user to stop speaking + interruption={"mode": "adaptive"}, + # Start generating the LLM response before the user fully stops + preemptive_generation={"enabled": True}, + ), + ) + + _active_session = session + + await session.start( + agent=VoiceAssistant(), + room=ctx.room, + room_options=room_io.RoomOptions( + audio_input=room_io.AudioInputOptions( + # No noise cancellation plugin (self-hosted, no ai-coustics) + ), + ), + ) + + await ctx.connect() + logger.info("Agent connected to room %s", ctx.room.name) + + +# ── Voice switching via data channel ──────────────────────────────────────── +# The web UI sends a JSON data message: {"type": "set_voice", "voice": "en-US-AvaNeural"} +# We listen on the room's data channel and update the TTS voice live. + + +@server.event("room.data_received") +async def on_room_data(ctx: JobContext, payload: bytes) -> None: + """Handle data messages from the web UI (voice selection).""" + import json + + try: + msg = json.loads(payload.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return + + if msg.get("type") == "set_voice": + voice = msg.get("voice", "") + if voice and _active_session: + logger.info("Switching TTS voice to %s", voice) + try: + # AgentSession exposes the TTS; update its options + tts = _active_session._tts # internal, but stable in 1.7 + if hasattr(tts, "update_options"): + tts.update_options(voice=voice) + logger.info("Voice updated to %s", voice) + except Exception as e: + logger.warning("Failed to update voice: %s", e) + + +if __name__ == "__main__": + cli.run_app(server) diff --git a/agent/pyproject.toml b/agent/pyproject.toml new file mode 100644 index 0000000..3485a9f --- /dev/null +++ b/agent/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "voice-agent" +version = "0.1.0" +description = "Real-time voice assistant: Azure STT/TTS + Gemma LLM via LiveKit Agents" +requires-python = ">=3.10,<3.15" + +dependencies = [ + "livekit-agents~=1.7", + "livekit-plugins-azure~=1.7", + "livekit-plugins-openai~=1.7", + "python-dotenv", +] + +[tool.setuptools.packages.find] +where = ["."]