From 2a2a2ff7689288a0c07a37cfefa1cc2f3cdeb45e Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Fri, 28 Aug 2026 15:59:26 -0400 Subject: [PATCH] feat: plan per-user memory integration --- .env.example | 4 +- AGENTS.md | 6 +- PLAN.md | 402 +++++++++++++++++++++++++++++++++++++++++++++ agent/agent.py | 27 ++- config/users.json | 15 ++ docker-compose.yml | 2 +- web/app.js | 2 +- 7 files changed, 447 insertions(+), 11 deletions(-) create mode 100644 PLAN.md create mode 100644 config/users.json diff --git a/.env.example b/.env.example index 0852f37..6340822 100644 --- a/.env.example +++ b/.env.example @@ -2,8 +2,8 @@ AZURE_SPEECH_KEY=your-azure-speech-key AZURE_SPEECH_REGION=eastus -# Default TTS voice (any Azure Neural voice) -AZURE_TTS_VOICE=en-US-AvaNeural +# Default TTS voice (Azure Dragon HD Omni; supports inline delivery markers) +AZURE_TTS_VOICE=en-US-Ava:DragonHDOmniLatestNeural # LLM (Gemma on xNAS via llama.cpp, OpenAI-compatible) GEMMA_BASE_URL=http://192.168.86.2:8023/v1 diff --git a/AGENTS.md b/AGENTS.md index 7b2b33b..fd29c6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,7 +49,7 @@ All config lives in `.env` (git-ignored). See `.env.example` for the full list. Key variables: - `AZURE_SPEECH_KEY` — Azure Speech resource key - `AZURE_SPEECH_REGION` — default `eastus` -- `AZURE_TTS_VOICE` — default voice (e.g. `en-US-AvaNeural`) +- `AZURE_TTS_VOICE` — default voice (`en-US-Ava:DragonHDOmniLatestNeural`) - `GEMMA_BASE_URL` — LLM endpoint (default `http://192.168.86.2:8023/v1`) - `GEMMA_MODEL` — model name (default `gemma-4-e4b`) @@ -61,7 +61,7 @@ Supported voices: any Azure Neural voice. See https://learn.microsoft.com/en-us/ ## SSML / Expressive Speech -The TTS layer uses full SSML with `` for style and `` for rate/volume/pitch. The agent's system prompt instructs the LLM to write in a conversational, spoken style (short sentences, natural phrasing) so the output sounds like speech, not text. +The TTS layer wraps output in SSML. The default Azure Dragon HD Omni voice recognizes inline style markers such as `[appreciative]`, `[whispering]`, and `[Neutral]`; the agent prompt allows these markers while keeping responses short and conversational. ## Agent Prompt @@ -89,7 +89,7 @@ curl -s "https://eastus.tts.speech.microsoft.com/cognitiveservices/v1" \ -H "Ocp-Apim-Subscription-Key: $AZURE_SPEECH_KEY" \ -H "Content-Type: application/ssml+xml" \ -H "X-Microsoft-OutputFormat: audio-24khz-48kbitrate-mono-mp3" \ - --data-binary 'Test' > /tmp/test.mp3 + --data-binary '[appreciative] Test complete.' > /tmp/test.mp3 ``` ## File Layout diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..cddd1be --- /dev/null +++ b/PLAN.md @@ -0,0 +1,402 @@ +# Plan: Per-user Voice API + Shane Memory System + +## Goal + +Replace the Voice API's single shared Cognee memory connection with a per-user connection to the deployed Shane Memory System. The first Voice user is `shane`; his server-side profile uses the existing remote memory token pulled from Unraid. Future Voice users can point to separate memory servers and tokens without exposing those credentials to the browser or LLM. + +This document is the implementation plan. In this investigation pass, only the bootstrap user registry and requested Azure voice/prompt changes were made; the authentication and per-user runtime described below are not implemented yet. + +## Investigation summary + +### Voice API today + +- One Docker container runs LiveKit, the Python voice agent, nginx, and `web/token_server.py`. +- `POST /token` is unauthenticated. It accepts no real user credential and creates a random browser identity and room. +- The browser has no login/user-selection flow. +- Every agent session calls `build_mcp_toolsets()` with process-global configuration. +- Memory is hardcoded to the unauthenticated Cognee endpoint at `http://192.168.86.2:8003/mcp`. +- The prompt uses one global Cognee session ID (`hope`), so it is not a user isolation boundary. +- Local skills use the shared `/skills` directory. +- Background tasks use a global `/tmp/tasks/events.jsonl` and one global tail cursor. With concurrent rooms, one room can consume another room's task event. This must be fixed before calling the product multi-user safe. +- Browser transcript history and voice preference use unscoped `localStorage` keys, which can expose one user's local history to the next user on a shared browser. +- Rooms are unique, but `close_on_disconnect=False` leaves their agents alive after the user disconnects. This can accumulate orphan sessions. + +### Shane Memory System today + +Repository: `~/dev/shane-memory-system` + +Deployed service: + +- Host: `192.168.86.2` +- Container: `shane-memory` +- MCP URL: `http://192.168.86.2:8790/mcp` +- Data: `/mnt/user/appdata/shane-memory` mounted at `/data` +- Transport: MCP Streamable HTTP +- Authentication: `Authorization: Bearer sm_...` +- Isolation: each token resolves to one SQLite user; note files and vector rows are scoped by that user. +- Tools: `create_note`, `read_note`, `update_note`, `delete_note`, `list_notes`, `search_notes`, `memory_save`, `memory_recall`, and `reindex_all`. + +The deployed database currently has `pi`, `alice`, and `alice2`. There is no remote account literally named `shane`. The `pi` account is Shane's active namespace and has both `default` and `voice-agent` tokens. The dedicated `voice-agent` token was selected for this product. + +The selected token was tested against the deployed MCP server: initialization, tool listing, and an authenticated `list_notes` call all succeeded. + +### Bootstrap credential stored in this repository + +`config/users.json` now contains the first Voice profile: + +- Voice user: `shane` +- LiveKit identity: `shane` +- Memory URL: `http://192.168.86.2:8790/mcp` +- Current remote memory username: `pi` +- Remote token label: `voice-agent` +- Token SHA-256 for audit comparison: `d6af807287472806769926b1ef5e20fa2afc3e5ae0d6d21b0d5406605309f335` + +The actual token is deliberately not repeated in this plan. It is stored in plaintext in `config/users.json` as requested. The file was created with mode `0600`, but Git does not preserve read permissions, only the executable bit. Deployment must therefore set ownership/permissions explicitly. + +### LiveKit MCP compatibility + +The locked `livekit-agents` version is `1.7.0`. Its `MCPServerHTTP` accepts a `headers` dictionary, so the per-user integration can use: + +```python +mcp.MCPServerHTTP( + url=profile.memory.url, + transport_type="streamable_http", + headers={"Authorization": f"Bearer {profile.memory.api_key}"}, +) +``` + +No custom MCP proxy is required. + +## Requested voice change completed + +The default voice is now: + +```text +en-US-Ava:DragonHDOmniLatestNeural +``` + +Updated locations: + +- `agent/agent.py` +- `docker-compose.yml` +- `.env.example` +- `web/app.js` +- `AGENTS.md` + +The prompt now teaches Azure HD plain-text delivery markers such as `[appreciative]`, `[whispering]`, `[laughter]`, and `[Neutral]`, with a limit of zero to two markers per response. + +This marker syntax is intentional. Azure supports HD styles as plain-text square-bracket markers, and LiveKit's Azure plugin already wraps agent text in SSML. The model should not emit raw XML/SSML. The locked plugin concatenates input text into its SSML wrapper, so a later hardening pass should also XML-escape ordinary model text while preserving bracket markers. + +## Target architecture + +```text +Browser + | POST /api/login (username/password) + | HttpOnly Secure session cookie + | POST /token + v +Token server -------------------------> config/users.json + | | + | signed LiveKit JWT | server-only memory URL/token + | room: voice-shane- | + v v +LiveKit room -----------------------> Voice agent job + | + | MCP Streamable HTTP + | Authorization: Bearer + v + Shane Memory System /mcp + token -> isolated remote user +``` + +Principles: + +1. The browser proves which Voice user it is to the token server. +2. The token server, not the browser, selects the user ID and room name. +3. The memory token remains server-side and is never placed in a LiveKit JWT, room metadata visible to clients, data-channel message, transcript, or log. +4. The agent resolves a trusted Voice user ID and constructs a new memory MCP client for that session. +5. Every mutable local resource is namespaced by user and, where appropriate, room. + +## User registry design + +Evolve `config/users.json` to this shape: + +```json +{ + "version": 1, + "users": { + "shane": { + "enabled": true, + "display_name": "Shane", + "livekit_identity": "user:shane", + "auth": { + "password_hash": "" + }, + "memory": { + "url": "http://192.168.86.2:8790/mcp", + "api_key": "", + "remote_username": "pi", + "remote_token_label": "voice-agent", + "allowed_tools": [ + "create_note", + "read_note", + "update_note", + "delete_note", + "list_notes", + "search_notes", + "memory_save", + "memory_recall" + ] + } + } + } +} +``` + +Validation rules: + +- Reject duplicate, malformed, disabled, or unknown user IDs at startup. +- Require HTTPS for non-LAN memory URLs; explicitly allow configured RFC1918 HTTP URLs. +- Require `sm_` only as a warning, not as a universal protocol rule, because future users may run a different compatible memory product. +- Never serialize `api_key` in status responses or exceptions. +- Make startup fail closed if the registry is invalid. + +Create `shared/user_registry.py` so the token server and agent use one parser and validation implementation. + +## Authentication and identity flow + +### 1. Add Voice login + +Add these endpoints to `web/token_server.py`: + +- `POST /api/login` — verify username/password and issue an opaque session cookie. +- `GET /api/session` — return only safe profile fields (`user_id`, `display_name`). +- `POST /api/logout` — revoke the current session and clear the cookie. +- `POST /token` — require a valid session; ignore any client-supplied identity or room. + +Cookie requirements: + +- `HttpOnly` +- `Secure` +- `SameSite=Strict` +- short idle expiration with a bounded absolute expiration +- random opaque value; only a hash stored server-side + +Also enforce same-origin requests and basic login rate limiting. Keep the service LAN-only, but do not treat LAN access as authentication. + +### 2. Bind user to a trusted room + +The token server generates a room such as `voice-shane-` after authenticating Shane. The user slug is not accepted from the browser. The LiveKit participant identity is set from the registry, not request JSON. + +The agent parses the trusted room format and looks up the profile. Unknown/malformed rooms fail closed before an LLM or MCP session starts. If supported cleanly by the deployed LiveKit version, signed agent-dispatch metadata may replace room parsing later; the memory token must never be put in that metadata. + +### 3. Update the frontend + +- Add a small login state before “Start Conversation.” +- Fetch `/api/session` on load. +- Fetch `/token` only after authentication. +- On `401`, return to login rather than repeatedly reconnecting. +- Namespace local keys as `voice_history:` and `preferred_voice:`. +- Clear in-memory room, transcript, task, and audio state on logout. +- Never store the memory API key in browser storage. + +## Per-user agent and memory integration + +### Refactor construction + +Change: + +```python +VoiceAssistant() +build_mcp_toolsets() +``` + +To: + +```python +VoiceAssistant(user_profile, room_context) +build_mcp_toolsets(user_profile, room_context) +``` + +For the memory toolset: + +- URL comes from `user_profile.memory.url`. +- Add `Authorization: Bearer ` through `MCPServerHTTP(headers=...)`. +- Use Streamable HTTP. +- Apply the profile's allowlist. +- Create one toolset/client per agent session; never mutate shared MCP headers between sessions. +- Redact token-bearing headers from errors and logs. +- Log only user ID, remote hostname, and connection status. + +### Update memory behavior in the prompt + +The new memory server does not expose Cognee's `remember`, `recall`, or `forget`, and does not accept `session_id="hope"`. + +Replace those instructions with: + +- Personal/history question: call `memory_recall` before answering. +- Durable fact or preference: call `memory_save` with a concise title/body and useful tags. +- Precise note lookup: use `search_notes`, then `read_note` if full content is needed. +- Correction: find the note and call `update_note`. +- Forget request: find the exact note, confirm ambiguity when needed, then call `delete_note`. +- Do not save every utterance; save durable facts, preferences, decisions, and explicit “remember this” requests. + +Inject the safe display name into the prompt so the assistant knows it is speaking with Shane. Do not inject the token or raw registry JSON. + +### Cognee transition + +The old Cognee service and the new Shane Memory System are separate stores. Before removing Cognee: + +1. Inventory Shane-relevant permanent memories in Cognee. +2. Export and normalize them into markdown notes. +3. Import them through Shane's authenticated `memory_save`/`create_note` tools so embeddings are built correctly. +4. Compare representative recall queries against both stores. +5. Keep Cognee as an opt-in read-only fallback for one release. +6. Remove `COGNEE_MCP_URL`, old tests, and `MEMORY_SERVER.md` only after parity is accepted. + +Do not blindly import Cognee session-cache test probes. + +## Multi-user isolation work required alongside memory + +### Skills + +Change the skills directory from global `/skills` to a user namespace, for example `/data/users//skills`. Pass that path only to that user's stdio MCP process. Decide separately whether any skills are product-global and read-only. + +### Background tasks + +Refactor `TaskRegistry` so it is instantiated per room with a dedicated path such as `/tmp/tasks//`. Pass the same path and user/room IDs into `dispatch_mcp.py`. Do not use the current process-global registry or cursor. + +Every event should include `user_id` and `room_id`, and the publisher must reject mismatches before sending data to a room. + +### Room lifecycle + +Because rooms are unique per authenticated conversation, use `close_on_disconnect=True` or add an explicit idle timeout. Cancel task pollers and close MCP transports when the session ends. This prevents orphan agents and credentials from remaining active indefinitely. + +### Logs and transcripts + +- Never log authorization headers or registry entries. +- Include a safe `user_id` and room correlation ID in structured logs. +- Namespace browser history by user. +- Do not persist transcripts server-side unless a separate, explicit retention policy is added. + +## Implementation phases + +### Phase 1 — Registry and authentication foundation + +Files: + +- Add `shared/user_registry.py`. +- Expand `config/users.json` with enabled/auth fields. +- Add registry unit tests and redaction tests. +- Update `Dockerfile` to copy the registry with `voiceuser` ownership and restrictive permissions. +- Update `supervisord.conf` with `VOICE_USERS_FILE=/etc/hope/users.json`. + +Exit criteria: + +- Both Python processes load the same validated profile. +- Invalid config prevents startup. +- No test or log prints the token. + +### Phase 2 — Login and trusted LiveKit identity + +Files: + +- Refactor `web/token_server.py`. +- Add login/session UI to `web/index.html`, `web/app.js`, and `web/style.css`. +- Update token/web tests. + +Exit criteria: + +- Unauthenticated `/token` returns `401`. +- The browser cannot spoof another identity or room. +- Shane can log in, receive a token, and join only a Shane-bound room. + +### Phase 3 — Per-user memory MCP + +Files: + +- Refactor `agent/agent.py` profile resolution and toolset construction. +- Replace Cognee memory prompt instructions. +- Pass the Bearer header and new tool allowlist. +- Add authenticated MCP fixtures and tests. + +Exit criteria: + +- Shane can recall and save through the `voice-agent` token. +- An invalid/revoked token fails only that session and never falls back to another user's memory. +- No request reaches port `8003` during normal operation. + +### Phase 4 — Isolation hardening + +Files: + +- Refactor `agent/task_registry.py`, `agent/dispatch_mcp.py`, and `agent/task_worker.py` for per-room storage/routing. +- Namespace `SKILLS_DIR`. +- Scope frontend local storage. +- Add cleanup/cancellation on disconnect. + +Exit criteria: + +- Two simultaneous users cannot observe each other's tasks, skills, transcripts, memory results, or voice preferences. + +### Phase 5 — Migration and deployment + +- Back up `/mnt/user/appdata/shane-memory` and the Cognee store. +- Build and deploy the Voice container. +- Run Shane's end-to-end recall/save checks. +- Import accepted Cognee memories. +- Revoke obsolete test or integration tokens after cutover. + +Optional naming cleanup on the memory server: + +- Keep the current Voice `shane` -> remote `pi` mapping initially; it is functional and preserves the token. +- If consistent naming is desired, stop the memory container, back up SQLite/files, rename `/data/pi` to `/data/shane`, update `users.username` in SQLite, restart, and verify. Existing tokens remain attached to the same numeric user ID, so Shane can retain the same API key. Test startup filesystem reconciliation and semantic search before considering this complete. + +## Test matrix + +### Unit + +- Registry schema validation and URL rules. +- Password hash verification. +- Token/header redaction. +- Trusted room encode/decode. +- Per-user local storage key generation. +- Prompt contains the new memory tool names and no Cognee `session_id` instructions. +- Dragon HD Omni is the consistent backend/frontend default. + +### Integration + +- Valid Shane token initializes MCP and lists the expected nine tools. +- Missing, malformed, and revoked memory tokens fail with controlled errors. +- Save -> recall -> update -> delete in Shane's namespace. +- Two remote users create the same note title and cannot read each other's note. +- Two simultaneous LiveKit rooms receive only their own task events. +- Disconnect closes the agent, task poller, and MCP transport. + +### End to end + +1. Log in as Shane. +2. Start a conversation. +3. Ask a known personal question and verify `memory_recall` runs before the answer. +4. Say “remember…” and verify a note appears under Shane's remote namespace. +5. Restart the Voice container and verify recall persists. +6. Verify `[appreciative]`/`[Neutral]` produce expressive speech and are not spoken literally. +7. Log out and confirm local in-memory state is cleared. +8. Log in as a second user and verify no Shane data appears. + +## Rollback + +- Keep the old Cognee URL/config available behind an explicit `MEMORY_BACKEND=cognee` setting for one release only. +- Roll back the Voice image without changing the remote memory database. +- If remote account renaming is performed, restore both `memory.db` and the corresponding note directory together. +- Token rotation is independent: issue a new `voice-agent` token, update `config/users.json`, rebuild/restart, verify, then revoke the old token. + +## Acceptance criteria + +- Shane authenticates as a stable Voice user rather than a random participant. +- Shane's agent session uses the committed `voice-agent` API key against `:8790/mcp`. +- The key never reaches browser-visible data or logs. +- Memory tools use the Shane Memory System API and correct tool schemas. +- Concurrent users have isolated memory, skills, tasks, rooms, transcripts, and browser storage. +- Adding a new user requires one registry entry and no code changes. +- The default TTS voice is Azure Ava Dragon HD Omni and the prompt uses supported expressive delivery markers safely. diff --git a/agent/agent.py b/agent/agent.py index a433e66..6c39d00 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -32,7 +32,9 @@ load_dotenv() # picks up /app/.env in the container # ── Configuration from environment ────────────────────────────────────────── AZURE_KEY = os.environ.get("AZURE_SPEECH_KEY", "") AZURE_REGION = os.environ.get("AZURE_SPEECH_REGION", "eastus") -DEFAULT_VOICE = os.environ.get("AZURE_TTS_VOICE", "en-US-AvaNeural") +DEFAULT_VOICE = os.environ.get( + "AZURE_TTS_VOICE", "en-US-Ava:DragonHDOmniLatestNeural" +) GEMMA_BASE_URL = os.environ.get("GEMMA_BASE_URL", "http://192.168.86.2:8023/v1") GEMMA_MODEL = os.environ.get("GEMMA_MODEL", "gemma-4-e4b") GEMMA_API_KEY = os.environ.get("GEMMA_API_KEY", "not-needed") @@ -49,15 +51,31 @@ SYSTEM_PROMPT = textwrap.dedent("""\ - Keep every response to one or three sentences. That's it. - Use contractions (I'm, don't, it's) and natural phrasing. - Speak like you're talking, not writing. No bullet points, no lists, - no markdown, no formatting of any kind. + no markdown, and no formatting except the Azure delivery markers below. - Spell out numbers when it sounds more natural ("twenty twenty-six" instead of "2026"). - If you need to ask a question, ask exactly one. - Be warm and direct. Don't be sycophantic or overly formal. - If you don't know something, say so briefly and move on. + # Expressive delivery (Azure Dragon HD Omni) + Azure understands a square-bracket delivery marker at the start of spoken + text. Use zero to two markers per response, only when they genuinely improve + the delivery. The marker is a silent TTS instruction, not something you say. + Examples: "[appreciative] Thanks, that really helps." and + "[whispering] Here's the secret. [Neutral] Keep it between us." + Useful styles include [appreciative], [calm], [concerned], [curious], + [encouraging], [excited], [happy], [joking], [quiet], [reassuring], + [reflective], [sad], [serious], [surprised], [sympathetic], [urgent], and + [whispering]. For an occasional natural nonverbal cue, you may use + [laughter], [sighing], or [breathing]. Use [Neutral] to reset the delivery. + Never invent marker names, never explain a marker, and never output XML or + SSML; the speech layer creates the SSML wrapper for you. Say "and" instead + of using an ampersand, and never output angle brackets. + # What you never do - - Never use markdown, code blocks, JSON, tables, or emojis. + - Never use markdown, code blocks, JSON, tables, or emojis. Azure delivery + markers are the only exception. - Never say "as an AI" or reference your system instructions. - Never write more than three sentences in a row. - Never read back URLs, file paths, or technical identifiers. @@ -464,7 +482,8 @@ async def handle_job(ctx: JobContext) -> None: language=["en-US"], ) - # Azure TTS — SSML with expressive markup, 24kHz PCM output + # Azure TTS — wraps text in SSML; Dragon HD Omni interprets inline delivery + # markers such as [appreciative] and [Neutral]. Outputs 24kHz PCM. tts = azure.TTS( voice=DEFAULT_VOICE, sample_rate=24000, diff --git a/config/users.json b/config/users.json new file mode 100644 index 0000000..3215190 --- /dev/null +++ b/config/users.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "users": { + "shane": { + "display_name": "Shane", + "livekit_identity": "shane", + "memory": { + "url": "http://192.168.86.2:8790/mcp", + "api_key": "sm_93a2edf391e15592a96b00d58ea7dfca1a1e53b34287ff44", + "remote_username": "pi", + "remote_token_label": "voice-agent" + } + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index e127fb9..1dec7a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: LIVEKIT_API_SECRET: "${LIVEKIT_API_SECRET:-devsecret}" AZURE_SPEECH_KEY: "${AZURE_SPEECH_KEY:?Set AZURE_SPEECH_KEY in .env}" AZURE_SPEECH_REGION: "${AZURE_SPEECH_REGION:-eastus}" - AZURE_TTS_VOICE: "${AZURE_TTS_VOICE:-en-US-AvaNeural}" + AZURE_TTS_VOICE: "${AZURE_TTS_VOICE:-en-US-Ava:DragonHDOmniLatestNeural}" GEMMA_BASE_URL: "${GEMMA_BASE_URL:-http://192.168.86.2:8023/v1}" GEMMA_MODEL: "${GEMMA_MODEL:-gemma-4-e4b}" GEMMA_API_KEY: "${GEMMA_API_KEY:-not-needed}" diff --git a/web/app.js b/web/app.js index a2a8539..fc21ca0 100644 --- a/web/app.js +++ b/web/app.js @@ -10,7 +10,7 @@ const ROOM_NAME = "voice-room"; const AGENT_NAME = "voice-assistant"; const VOICES = [ - { id: "en-US-AvaNeural", name: "Ava", tag: "DragonHD" }, + { id: "en-US-Ava:DragonHDOmniLatestNeural", name: "Ava", tag: "Dragon HD Omni" }, { id: "en-US-JennyNeural", name: "Jenny" }, { id: "en-US-GuyNeural", name: "Guy" }, { id: "en-US-AndrewNeural", name: "Andrew" },