feat: provision private Hermes assistants for authenticated Hope accounts

This commit is contained in:
snowspeeder committed 2026-10-01 13:30:18 -04:00
1 parent da65da68b3
commit 2aedea84b5
33 files changed
+3063 -1743

No files matched your search

+285
View File
@@ -0,0 +1,285 @@
"""Offline control-plane tests: no Docker, provider, LAN, or deployment access."""
import asyncio
import sqlite3
import time
import pytest
from fastapi.testclient import TestClient
from control.accounts import Accounts, password_matches, public_user
from control.app import Settings, create_app
PASSWORD = 'correct-horse-password'
SECRET = 'internal-secret-' + 'x' * 32
class FakeProvisioner:
def __init__(self):
self.ensured = []
self.removed = []
self.fail_ensure = False
self.fail_remove = False
self.healthy = True
async def ensure(self, user):
self.ensured.append(user['id'])
if self.fail_ensure:
raise RuntimeError('sensitive-provider-key-must-not-leak')
return {'container_name': user['container_name'], 'hermes_url': 'http://127.0.0.1:1234', 'hermes_api_key': 'private-hermes-key'}
async def ready(self, user):
return self.healthy
async def remove(self, user):
self.removed.append(user['id'])
if self.fail_remove:
raise RuntimeError('container ownership verification failed')
@pytest.fixture
def environment(tmp_path):
fake = FakeProvisioner()
settings = Settings(db_path=str(tmp_path / 'accounts.db'), admin_password=PASSWORD, service_secret=SECRET, allowed_origins=('https://testserver',), reconcile_interval=3600)
app = create_app(settings, fake)
with TestClient(app, base_url='https://testserver') as client:
yield client, app, fake, settings
def login(client, username='admin', password=PASSWORD):
result = client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': username, 'password': password})
assert result.status_code == 200, result.text
return result.json()
def auth(result):
return {'Authorization': 'Bearer ' + result['token']}
def wait_state(client, headers, user_id, status):
# Requests yield to the ASGI lifecycle loop; bounded wait, no network.
deadline = time.monotonic() + 3
while time.monotonic() < deadline:
users = client.get('/api/admin/users', headers=headers).json()['users']
found = next((u for u in users if u['id'] == user_id), None)
if (status is None and found is None) or (found and found['status'] == status):
return found
raise AssertionError(f'Account did not reach {status}: {found}')
def create(client, headers, username='john-doe'):
response = client.post('/api/admin/users', headers=headers, json={'username': username, 'display_name': 'John Doe', 'password': PASSWORD})
assert response.status_code == 201, response.text
return response.json()['user']
def test_bootstrap_required(tmp_path):
app = create_app(Settings(db_path=str(tmp_path / 'missing.db'), admin_password=None), FakeProvisioner())
with pytest.raises(RuntimeError, match='CONTROL_ADMIN_PASSWORD'):
with TestClient(app):
pass
def test_password_sessions_cookie_schema(environment):
client, app, fake, settings = environment
assert client.get('/api/auth/me').status_code == 401
session = login(client)
assert set(session['user']) == {'id', 'username', 'display_name', 'role', 'status', 'container_name', 'error'}
assert len(session['user']['id']) == 32
assert session['expires_at'] > time.time()
cookie = client.cookies.get('hope_session')
assert cookie == session['token']
raw = app.state.accounts.get(session['user']['id'])
assert raw['password_hash'] != PASSWORD
assert password_matches(PASSWORD, raw['password_hash'])
row = app.state.accounts.db.execute('SELECT hash FROM sessions').fetchone()
assert row['hash'] != session['token']
assert client.get('/api/auth/me').status_code == 200
assert client.post('/api/auth/logout').status_code == 403
assert client.post('/api/auth/logout', headers={'Origin': 'https://evil.example'}).status_code == 403
assert client.post('/api/auth/logout', headers=auth(session)).status_code == 200
assert client.get('/api/auth/me', headers=auth(session)).status_code == 401
again = client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': 'admin', 'password': PASSWORD})
cookie_header = again.headers['set-cookie'].lower()
assert all(x in cookie_header for x in ('httponly', 'secure', 'samesite=strict'))
assert client.post('/api/auth/logout', headers={'Origin': 'https://testserver'}).status_code == 200
def test_create_auth_reset_and_validation(environment):
client, app, fake, settings = environment
admin = auth(login(client))
user = create(client, admin)
ready = wait_state(client, admin, user['id'], 'ready')
assert ready['container_name'] == f"hope-hermes-john-doe-{user['id'][:8]}"
assert client.post('/api/admin/users', headers=admin, json={'username': 'Bad/Name', 'display_name': 'Bad', 'password': PASSWORD}).status_code == 400
assert client.post('/api/admin/users', headers=admin, json={'username': 'john-doe', 'display_name': 'John', 'password': PASSWORD}).status_code == 409
ordinary = auth(login(client, 'john-doe'))
assert client.get('/api/admin/users', headers=ordinary).status_code == 403
assert client.get('/internal/users/' + user['id'], headers=ordinary).status_code == 401
assert client.post('/api/admin/users/' + user['id'] + '/password', headers=admin, json={'password': 'new-safe-password'}).status_code == 200
assert client.get('/api/auth/me', headers=ordinary).status_code == 401
assert client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': 'john-doe', 'password': PASSWORD}).status_code == 401
login(client, 'john-doe', 'new-safe-password')
def test_last_admin_and_delete_revokes_sessions_rooms(environment):
client, app, fake, settings = environment
session = login(client)
admin = auth(session)
assert client.delete('/api/admin/users/' + session['user']['id'], headers=admin).status_code == 409
user = create(client, admin)
wait_state(client, admin, user['id'], 'ready')
ordinary = login(client, 'john-doe')
internal = {'X-Control-Secret': SECRET}
resolve = client.post('/internal/resolve-session', headers=internal, json={'token': ordinary['token']})
assert resolve.status_code == 200
assert resolve.json()['hermes_api_key'] == 'private-hermes-key'
assert 'password_hash' not in resolve.text
assert client.post('/internal/rooms', headers=internal, json={'user_id': user['id'], 'room_name': 'voice-test', 'session_token': ordinary['token']}).status_code == 201
assert client.get('/internal/rooms/voice-test', headers=internal).status_code == 200
fake.fail_remove = True
assert client.delete('/api/admin/users/' + user['id'], headers=admin).status_code == 202
assert client.get('/api/auth/me', headers=auth(ordinary)).status_code == 401
assert client.get('/internal/rooms/voice-test', headers=internal).status_code == 404
assert client.post('/internal/resolve-session', headers=internal, json={'token': ordinary['token']}).status_code == 401
wait_state(client, admin, user['id'], 'deleting')
fake.fail_remove = False
# Wait for failed first operation to finish before retry scheduling.
async def settled():
await asyncio.gather(*app.state.lifecycle.tasks.values())
client.portal.call(settled)
client.post('/api/admin/users/' + user['id'] + '/retry', headers=admin)
wait_state(client, admin, user['id'], None)
assert user['id'] in fake.removed
def test_failure_visible_and_restart_reconcile(tmp_path):
fake = FakeProvisioner()
fake.fail_ensure = True
settings = Settings(db_path=str(tmp_path / 'restart.db'), admin_password=PASSWORD, service_secret=SECRET, reconcile_interval=3600)
with TestClient(create_app(settings, fake), base_url='https://testserver') as client:
session = login(client)
header = auth(session)
user = wait_state(client, header, session['user']['id'], 'error')
assert 'sensitive' not in user['error']
fake.fail_ensure = False
settings.admin_password = None
with TestClient(create_app(settings, fake), base_url='https://testserver') as client:
wait_state(client, header, session['user']['id'], 'ready')
assert client.get('/api/auth/me', headers=header).status_code == 200
def test_expiry_throttle_and_bounded_input(environment):
client, app, fake, settings = environment
session = login(client)
with app.state.accounts.lock, app.state.accounts.db:
app.state.accounts.db.execute('UPDATE sessions SET expires_at=0')
assert client.get('/api/auth/me', headers=auth(session)).status_code == 401
client.cookies.clear()
assert client.post('/api/auth/login', json={'username': 'admin', 'password': 'p' * 257}).status_code == 400
assert client.post('/api/auth/login', content='x' * 9000, headers={'Content-Type': 'application/json'}).status_code == 413
assert client.post('/api/auth/login', content='{}', headers={'Content-Type': 'text/plain'}).status_code == 415
client.cookies.clear()
for _ in range(settings.login_limit):
result = client.post('/api/auth/login', json={'username': 'missing', 'password': PASSWORD})
assert result.status_code == 401
assert client.post('/api/auth/login', json={'username': 'missing', 'password': PASSWORD}).status_code == 429
def test_internal_fail_closed(environment):
client, app, fake, settings = environment
assert client.post('/internal/resolve-session', json={'token': 'x' * 32}).status_code == 401
assert client.get('/internal/users/' + 'a' * 32, headers={'X-Control-Secret': 'wrong'}).status_code == 401
settings.service_secret = None
assert client.get('/internal/rooms/voice-test', headers={'X-Control-Secret': SECRET}).status_code == 401
def test_room_session_binding_and_logout(environment):
client, app, fake, settings = environment
admin_session = login(client)
header = auth(admin_session)
wait_state(client, header, admin_session['user']['id'], 'ready')
user = create(client, header, 'alice')
wait_state(client, header, user['id'], 'ready')
session = login(client, 'alice')
internal = {'X-Control-Secret': SECRET}
mismatch = {'user_id': user['id'], 'room_name': 'wrong', 'session_token': admin_session['token']}
assert client.post('/internal/rooms', headers=internal, json=mismatch).status_code == 409
grant = {'user_id': user['id'], 'room_name': 'voice-alice', 'session_token': session['token']}
assert client.post('/internal/rooms', headers=internal, json=grant).status_code == 201
assert client.get('/internal/rooms/voice-alice', headers=internal).status_code == 200
assert client.post('/api/auth/logout', headers=auth(session)).status_code == 200
assert client.get('/internal/rooms/voice-alice', headers=internal).status_code == 404
def test_failed_removal_restart_retries(tmp_path):
path = str(tmp_path / 'removal.db')
store = Accounts(path)
store.create('admin', 'Administrator', 'admin', PASSWORD)
user = store.create('alice', 'Alice', 'user', PASSWORD)
store.begin_delete(user['id'])
store.close()
settings = Settings(db_path=path, admin_password=None, service_secret=SECRET, reconcile_interval=3600)
fake = FakeProvisioner()
fake.fail_remove = True
with TestClient(create_app(settings, fake), base_url='https://testserver') as client:
header = auth(login(client))
wait_state(client, header, user['id'], 'deleting')
assert client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': 'alice', 'password': PASSWORD}).status_code == 401
fake.fail_remove = False
with TestClient(create_app(settings, fake), base_url='https://testserver') as client:
header = auth(login(client))
wait_state(client, header, user['id'], None)
assert fake.removed.count(user['id']) >= 2
def test_reset_between_password_verification_and_session_issue(environment, monkeypatch):
client, app, fake, settings = environment
original = app.state.accounts.issue_session
def reset_then_issue(user_id, ttl, expected_hash):
app.state.accounts.reset_password(user_id, 'changed-safe-password')
return original(user_id, ttl, expected_hash)
monkeypatch.setattr(app.state.accounts, 'issue_session', reset_then_issue)
response = client.post('/api/auth/login', json={'username': 'admin', 'password': PASSWORD})
assert response.status_code == 401
assert app.state.accounts.db.execute('SELECT count(*) FROM sessions').fetchone()[0] == 0
monkeypatch.setattr(app.state.accounts, 'issue_session', original)
login(client, password='changed-safe-password')
def test_successes_and_other_accounts_do_not_share_failure_budget(environment):
client, app, fake, settings = environment
for _ in range(settings.login_limit + 2):
login(client)
for _ in range(settings.login_limit):
assert client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': 'missing', 'password': PASSWORD}).status_code == 401
assert client.post('/api/auth/login', headers={'Origin': 'https://testserver'}, json={'username': 'missing', 'password': PASSWORD}).status_code == 429
login(client) # Same IP, different account remains usable.
@pytest.mark.parametrize('peer,trusted', [('127.0.0.1', True), ('::1', True), ('203.0.113.10', False)])
def test_real_ip_only_trusted_from_loopback(tmp_path, peer, trusted):
settings = Settings(db_path=str(tmp_path / 'proxy.db'), admin_password=PASSWORD, login_limit=2)
with TestClient(create_app(settings, FakeProvisioner()), base_url='https://testserver', client=(peer, 1234)) as client:
body = {'username': 'missing', 'password': PASSWORD}
for _ in range(settings.login_limit):
assert client.post('/api/auth/login', headers={'X-Real-IP': '198.51.100.1'}, json=body).status_code == 401
assert client.post('/api/auth/login', headers={'X-Real-IP': '198.51.100.1'}, json=body).status_code == 429
other = client.post('/api/auth/login', headers={'X-Real-IP': '198.51.100.2', 'X-Forwarded-For': '198.51.100.3'}, json=body)
assert other.status_code == (401 if trusted else 429)
def test_database_delete_intent_persists(tmp_path):
db = str(tmp_path / 'durable.db')
store = Accounts(db)
admin = store.create('admin', 'Administrator', 'admin', PASSWORD)
user = store.create('alice', 'Alice', 'user', PASSWORD)
token, expiry = store.issue_session(user['id'], 60, user['password_hash'])
store.begin_delete(user['id'])
store.close()
store = Accounts(db)
assert store.get(user['id'])['desired'] == 'deleted'
assert store.resolve_session(token) is None
assert store.get(admin['id'])['role'] == 'admin'
store.close()
+298
View File
@@ -0,0 +1,298 @@
"""Offline bridge contracts against the real installed LiveKit SDK.
Run without deployment fixtures:
agent/.venv/bin/python -m unittest discover -s tests/unit -p test_hermes_llm.py -v
"""
from __future__ import annotations
import asyncio
import json
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
import httpx
from livekit.agents import llm as lk_llm
from livekit.agents.types import APIConnectOptions
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "agent"))
from hermes_llm import HermesLLM, HermesLLMError, WAIT_NOTICE
def data(delta, **kwargs):
return ("data: " + json.dumps({"choices": [{"index": 0, "delta": delta, **kwargs}]}, ensure_ascii=False) + "\n\n").encode()
class FakeSSE(httpx.AsyncByteStream):
def __init__(self, chunks, *, gate=None):
self.chunks = chunks
self.gate = gate
self.closed = False
self.started = asyncio.Event()
async def __aiter__(self):
self.started.set()
for chunk in self.chunks:
yield chunk
if self.gate is not None:
await self.gate.wait()
async def aclose(self):
self.closed = True
class HermesLLMTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
self.progress = []
self.notices = []
self.requests = []
self.clients = []
async def asyncTearDown(self):
for client in self.clients:
await client.aclose()
async def progress_callback(self, event):
self.progress.append(event)
async def notice_callback(self, message):
self.notices.append(message)
def bridge(self, stream, *, status=200, headers=None, **kwargs):
def handler(request):
self.requests.append(request)
return httpx.Response(status, headers=headers or {"content-type": "text/event-stream"}, stream=stream)
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
self.clients.append(client)
return HermesLLM(
base_url="https://trusted.hermes/v1/", api_key="secret-key",
user_id="alice", room_id="stable-room", on_progress=self.progress_callback,
on_wait_notice=self.notice_callback, http_client=client, **kwargs,
)
def context(self):
return lk_llm.ChatContext([
lk_llm.ChatMessage(role="system", content=["EXACT system prompt\nDo not alter."]),
lk_llm.ChatMessage(role="user", content=["hello", "world"]),
lk_llm.ChatMessage(role="assistant", content=["Earlier answer"]),
lk_llm.ChatMessage(role="developer", content=["unsupported"]),
lk_llm.FunctionCall(call_id="call", name="local_tool", arguments='{"secret":true}'),
lk_llm.FunctionCallOutput(call_id="call", name="local_tool", output="local secret", is_error=False),
])
async def test_sdk_chunks_and_exact_request_contract(self):
body = data({"reasoning_content": "PRIVATE"}) + data({"content": "Hi "}) + data({"content": "there"}) + b"data: [DONE]\n\n"
fake = FakeSSE([body[:9], body[9:25], body[25:]])
bridge = self.bridge(fake)
context = self.context()
before = context.to_dict()
stream = bridge.chat(chat_ctx=context, tools=[object()], tool_choice="required", extra_kwargs={"model": "wrong", "tools": ["wrong"]})
async with stream:
chunks = [chunk async for chunk in stream]
self.assertTrue(isinstance(stream, lk_llm.LLMStream))
self.assertEqual([chunk.delta.content for chunk in chunks], ["Hi ", "there"])
self.assertTrue(all(chunk.delta.role == "assistant" and not chunk.delta.tool_calls for chunk in chunks))
request = self.requests[0]
self.assertEqual(str(request.url), "https://trusted.hermes/v1/chat/completions")
self.assertEqual(request.headers["authorization"], "Bearer secret-key")
self.assertEqual(request.headers["x-hermes-session-id"], "stable-room")
self.assertEqual(request.headers["x-hermes-session-key"], "hope:alice")
self.assertEqual(json.loads(request.content), {
"model": "hermes-agent", "stream": True,
"messages": [
{"role": "system", "content": "EXACT system prompt\nDo not alter."},
{"role": "user", "content": "hello\nworld"},
{"role": "assistant", "content": "Earlier answer"},
], "model_options": {"reasoning": {"enabled": False}},
})
self.assertEqual(context.to_dict(), before)
self.assertEqual(stream.tools, [])
self.assertEqual(self.notices, [])
self.assertEqual(self.progress[-1]["status"], "completed")
self.assertEqual(self.progress[-1]["request_id"], chunks[0].id)
self.assertTrue(fake.closed)
async def test_sse_crlf_unicode_multiline_comments_unknown_events_and_tool_notice_once(self):
tool = b'event: hermes.tool.progress\r\ndata: {"status":"started",\r\ndata: "tool_name":"search"}\r\n\r\n'
body = b":keepalive\r\n\r\nevent: unknown\r\ndata: not json\r\n\r\n" + tool + tool + data({"reasoning": "never speak"}) + data({"content": "caf\u00e9"}) + b"data: [DONE]\r\n\r\n"
fake = FakeSSE([body[i:i + 1] for i in range(len(body))])
bridge = self.bridge(fake)
result = await bridge.chat(chat_ctx=self.context()).collect()
self.assertEqual(result.text, "caf\u00e9")
self.assertEqual(self.notices, [WAIT_NOTICE])
self.assertEqual([event["status"] for event in self.progress], ["working", "working", "completed"])
self.assertTrue(all(event["type"] == "assistant_progress" for event in self.progress))
self.assertEqual(len({event["request_id"] for event in self.progress}), 1)
async def test_tool_after_answer_does_not_speak_notice(self):
fake = FakeSSE([data({"content": "Answer"}), b'event: hermes.tool.progress\ndata: {"status":"working","message":"Searching"}\n\n', b"data: [DONE]\n\n"])
await self.bridge(fake).chat(chat_ctx=self.context()).collect()
self.assertEqual(self.notices, [])
async def test_latency_notice_without_network_event(self):
fake = FakeSSE([], gate=asyncio.Event())
bridge = self.bridge(fake)
# Speed up only the bridge's three-second timer; network remains blocked.
sleep = asyncio.sleep
async def fast_sleep(seconds):
self.assertEqual(seconds, 3.0)
await sleep(0)
with patch("hermes_llm.asyncio.sleep", side_effect=fast_sleep):
stream = bridge.chat(chat_ctx=self.context())
await fake.started.wait()
for _ in range(10):
await sleep(0)
if self.notices:
break
self.assertEqual(self.notices, [WAIT_NOTICE])
self.assertEqual(self.progress[0]["status"], "working")
self.assertEqual(self.progress[0]["message"], "Working on your request.")
await stream.aclose()
self.assertTrue(fake.closed)
self.assertEqual(self.progress[-1]["status"], "failed")
async def test_timeout_closes_stream_and_never_retries(self):
fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event())
stream = self.bridge(fake, timeout_seconds=0.02).chat(chat_ctx=self.context(), conn_options=APIConnectOptions(max_retry=10))
with self.assertRaises(HermesLLMError) as caught:
await stream.collect()
self.assertEqual(caught.exception.code, "timeout")
self.assertFalse(caught.exception.retryable)
self.assertEqual(len(self.requests), 1)
self.assertTrue(fake.closed)
self.assertEqual(self.progress[-1]["status"], "failed")
async def test_failures_are_explicit_redacted_and_not_retried(self):
cases = [
(401, b"secret-key sensitive payload", "http_401"),
(503, b"sensitive upstream failure", "http_503"),
(200, b"data: malformed-secret-key\n\n", "invalid_sse"),
(200, b'data: {"error":"secret-key"}\n\n', "upstream_error"),
(200, b'event: hermes.tool.progress\ndata: {"status":"failed","message":"secret-key"}\n\n', "tool_failed"),
(200, data({"content": "partial"}), "incomplete_stream"),
(200, data({"reasoning_content": "secret-key"}) + b"data: [DONE]\n\n", "empty_response"),
]
for status, body, code in cases:
with self.subTest(code=code):
before = len(self.requests)
fake = FakeSSE([body])
stream = self.bridge(fake, status=status).chat(chat_ctx=self.context())
with self.assertRaises(HermesLLMError) as caught:
await stream.collect()
self.assertEqual(caught.exception.code, code)
self.assertNotIn("secret-key", str(caught.exception))
self.assertNotIn("sensitive", repr(caught.exception))
self.assertEqual(len(self.requests), before + 1)
self.assertEqual(self.progress[-1]["status"], "failed")
self.assertTrue(fake.closed)
async def test_invalid_content_type_and_redirect_not_followed(self):
for status, expected in [(200, "invalid_content_type"), (302, "http_302")]:
bridge = self.bridge(FakeSSE([b"html"]), status=status, headers={"content-type": "text/html", "location": "https://attacker.invalid"})
with self.assertRaises(HermesLLMError) as caught:
await bridge.chat(chat_ctx=self.context()).collect()
self.assertEqual(caught.exception.code, expected)
self.assertEqual(len(self.requests), 2)
async def test_callbacks_fail_without_replaying_actions(self):
async def broken(_):
raise RuntimeError("secret-key")
fake = FakeSSE([b'event: hermes.tool.progress\ndata: {"status":"working"}\n\n', data({"content": "Done"}), b"data: [DONE]\n\n"])
bridge = self.bridge(fake)
bridge._on_progress = bridge._on_wait_notice = broken
result = await bridge.chat(chat_ctx=self.context()).collect()
self.assertEqual(result.text, "Done")
self.assertEqual(len(self.requests), 1)
async def test_sessions_stable_across_requests_and_caller_owned_client(self):
bridge = self.bridge(FakeSSE([data({"content": "Done"}), b"data: [DONE]\n\n"]))
ids = []
for _ in range(2):
stream = bridge.chat(chat_ctx=self.context())
ids.append(stream.request_id)
await stream.collect()
self.assertNotEqual(*ids)
self.assertTrue(all(req.headers["x-hermes-session-id"] == "stable-room" for req in self.requests))
await bridge.aclose()
self.assertFalse(bridge._client.is_closed)
async def test_origin_base_url_and_eof_finish_frame(self):
fake = FakeSSE([data({"content": "Done"}), b'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}'])
bridge = self.bridge(fake)
origin = HermesLLM(base_url="https://trusted.hermes", api_key="secret-key", user_id="alice", room_id="stable-room", on_progress=self.progress_callback, http_client=bridge._client)
self.assertEqual((await origin.chat(chat_ctx=self.context()).collect()).text, "Done")
self.assertEqual(str(self.requests[0].url), "https://trusted.hermes/v1/chat/completions")
async def test_approval_and_error_finishes_close_immediately(self):
cases = [
(b'event: approval.request\ndata: {"secret":"hidden"}\n\n', "approval_unsupported"),
(b'event: hermes.approval.request\ndata: {}\n\n', "approval_unsupported"),
(data({}, finish_reason="error"), "upstream_error"),
(data({}, finish_reason="approval_required"), "approval_unsupported"),
(b'data: {"hermes":{"error":"secret"},"choices":[]}\n\n', "upstream_error"),
(b'event: hermes.error\ndata: {"message":"secret"}\n\n', "upstream_error"),
]
for body, code in cases:
with self.subTest(code=code):
fake = FakeSSE([body], gate=asyncio.Event())
with self.assertRaises(HermesLLMError) as caught:
await asyncio.wait_for(self.bridge(fake).chat(chat_ctx=self.context()).collect(), timeout=1)
self.assertEqual(caught.exception.code, code)
self.assertTrue(fake.closed)
self.assertEqual(self.progress[-1]["status"], "failed")
async def test_status_event_is_generic_and_never_answer_text(self):
fake = FakeSSE([b'event: hermes.status\ndata: {"message":"sensitive internals"}\n\n', data({"content": "Done"}), b"data: [DONE]\n\n"])
result = await self.bridge(fake).chat(chat_ctx=self.context()).collect()
self.assertEqual(result.text, "Done")
self.assertEqual(self.progress[0]["message"], "Working on your request.")
self.assertEqual(self.notices, [WAIT_NOTICE])
async def test_explicit_cancellation_closes_http_without_retry(self):
fake = FakeSSE([b":keepalive\n\n"], gate=asyncio.Event())
stream = self.bridge(fake).chat(chat_ctx=self.context())
await fake.started.wait()
await stream.aclose()
self.assertTrue(fake.closed)
self.assertEqual(len(self.requests), 1)
self.assertEqual(self.progress[-1]["status"], "failed")
self.assertEqual(self.progress[-1]["message"], "Request cancelled.")
self.assertTrue(stream._task.cancelled())
async def test_answer_start_hook_runs_once_before_any_content(self):
calls = []
async def answer_start():
calls.append("answer_start")
fake = FakeSSE([data({"reasoning_content": "ignored"}), data({"content": "First"}), data({"content": " second"}), b"data: [DONE]\n\n"])
bridge = self.bridge(fake, on_answer_start=answer_start)
async with bridge.chat(chat_ctx=self.context()) as stream:
async for chunk in stream:
self.assertEqual(calls[0], "answer_start")
calls.append(chunk.delta.content)
self.assertEqual(calls, ["answer_start", "First", " second"])
async def test_answer_start_failure_does_not_retry_or_drop_answer(self):
async def answer_start():
raise RuntimeError("secret-key")
fake = FakeSSE([data({"content": "Done"}), b"data: [DONE]\n\n"])
result = await self.bridge(fake, on_answer_start=answer_start).chat(chat_ctx=self.context()).collect()
self.assertEqual(result.text, "Done")
self.assertEqual(len(self.requests), 1)
async def test_transport_exception_redacted(self):
def broken(request):
self.requests.append(request)
raise httpx.ConnectError("secret-key sensitive payload", request=request)
client = httpx.AsyncClient(transport=httpx.MockTransport(broken))
self.clients.append(client)
bridge = HermesLLM(base_url="https://trusted.hermes", api_key="secret-key", user_id="alice", room_id="room", on_progress=self.progress_callback, http_client=client)
with self.assertRaises(HermesLLMError) as caught:
await bridge.chat(chat_ctx=self.context()).collect()
self.assertEqual(caught.exception.code, "transport_error")
self.assertTrue(caught.exception.__suppress_context__)
self.assertEqual(len(self.requests), 1)
if __name__ == "__main__":
unittest.main()
+214
View File
@@ -0,0 +1,214 @@
"""Offline Docker HTTP boundary tests; no Docker daemon or model calls."""
import asyncio
import copy
import json
import os
import httpx
import pytest
from control.provisioner import OwnershipError, Provisioner, ProvisioningError, container_name
ALICE = {'id': 'a' * 32, 'username': 'alice'}
BOB = {'id': 'b' * 32, 'username': 'bob'}
class DockerBoundary:
def __init__(self):
self.containers, self.networks, self.created, self.deleted = {}, {}, [], []
self.fail_start = False
def __call__(self, request):
path, method = request.url.path, request.method
if path == '/networks/create':
spec = json.loads(request.content)
name = spec['Name']
self.networks[name] = dict(spec, Id='net-' + name, Internal=False)
return httpx.Response(201, json={'Id': 'net-' + name})
if path.startswith('/networks/'):
key = path.split('/')[2]
net = self.networks.get(key) or next((n for n in self.networks.values() if n['Id'] == key), None)
if method == 'DELETE' and net:
del self.networks[net['Name']]
return httpx.Response(204)
return httpx.Response(200, json=net) if net else httpx.Response(404)
if path == '/containers/create':
name = request.url.params['name']
if name in self.containers:
return httpx.Response(409)
spec = json.loads(request.content)
self.created.append(spec)
self.containers[name] = {'Id': 'id-' + name, 'Config': copy.deepcopy(spec),
'State': {'Running': False}, 'NetworkSettings': {'Ports': {}}}
return httpx.Response(201, json={'Id': 'id-' + name})
key = path.split('/')[2]
info = self.containers.get(key) or next((c for c in self.containers.values() if c['Id'] == key), None)
if not info:
return httpx.Response(404)
if method == 'GET':
return httpx.Response(200, json=info)
if method == 'POST':
if self.fail_start:
self.fail_start = False
return httpx.Response(500)
info['State']['Running'] = True
info['NetworkSettings']['Ports'] = {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': str(31000 + len(self.containers))}]}
return httpx.Response(204)
self.deleted.append((key, dict(request.url.params)))
name = next(n for n, c in self.containers.items() if c is info)
del self.containers[name]
return httpx.Response(204)
@pytest.fixture
def boundary(tmp_path, monkeypatch):
monkeypatch.delenv('HERMES_UPSTREAM_URL', raising=False)
monkeypatch.delenv('HERMES_UPSTREAM_MODEL', raising=False)
docker, health_requests = DockerBoundary(), []
def health(request):
health_requests.append(request)
assert request.url.host == '127.0.0.1'
assert request.url.path == '/v1/models'
return httpx.Response(200, json={'data': []})
dc = httpx.AsyncClient(base_url='http://docker', transport=httpx.MockTransport(docker))
hc = httpx.AsyncClient(transport=httpx.MockTransport(health))
p = Provisioner(tmp_path / 'private', docker_client=dc, health_client=hc,
upstream_api_key='runtime-provider-secret', runtime_uid=os.geteuid() or 10000,
runtime_gid=os.getegid() or 10000, ready_timeout=.1, poll_interval=.001)
yield p, docker, health_requests
asyncio.run(dc.aclose())
asyncio.run(hc.aclose())
def test_account_isolation_and_runtime_configuration(boundary):
p, docker, health = boundary
async def exercise():
alice, bob = await p.ensure(ALICE), await p.ensure(BOB)
assert await p.ready(ALICE)
return alice, bob
alice, bob = asyncio.run(exercise())
assert alice['container_name'] == container_name(ALICE)
assert alice['hermes_url'].startswith('http://127.0.0.1:')
assert alice['hermes_api_key'] != bob['hermes_api_key']
assert len(alice['hermes_api_key']) >= 32
for user, spec, connection in zip((ALICE, BOB), docker.created, (alice, bob)):
assert spec['Labels'] == {'hope.managed': 'true', 'hope.user_id': user['id']}
assert spec['User'].split(':')[0] != '0'
assert spec['Entrypoint'] == ['/opt/hermes/.venv/bin/python']
env = dict(entry.split('=', 1) for entry in spec['Env'])
assert env['HERMES_UPSTREAM_API_KEY'] == 'runtime-provider-secret'
assert env['API_SERVER_KEY'] == connection['hermes_api_key']
assert env['API_SERVER_ENABLED'] == 'true'
assert env['API_SERVER_HOST'] == '0.0.0.0'
assert env['API_SERVER_PORT'] == '8642'
host = spec['HostConfig']
assert host['NetworkMode'] == 'hope-hermes-net-' + user['id']
assert host['PortBindings']['8642/tcp'] == [{'HostIp': '127.0.0.1', 'HostPort': ''}]
assert host['Memory'] > 0 and host['NanoCpus'] > 0 and host['PidsLimit'] > 0
assert host['Privileged'] is False and host['Init'] is True
assert host['CapDrop'] == ['ALL']
assert host['SecurityOpt'] == ['no-new-privileges:true']
assert len(host['Binds']) == 2
assert all(user['id'] in mount and 'docker.sock' not in mount for mount in host['Binds'])
private, home, workspace = p._paths(user)
text = (home / 'config.yaml').read_text()
config = json.loads(text)
assert config['model'] == {'default': 'auto', 'provider': 'custom:hope-upstream'}
assert config['providers']['hope-upstream'] == {'api': 'http://192.168.86.5:2099/v1',
'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': 'auto',
'models': ['auto'], 'discover_models': False}
assert config['agent']['reasoning_effort'] == 'none'
assert config['terminal'] == {'backend': 'local', 'cwd': '/workspace'}
assert {'memory', 'browser', 'web', 'terminal', 'file', 'skills'} <= set(config['platform_toolsets']['api_server'])
assert config['gateway']['api_server']['max_concurrent_runs'] == 1
assert 'runtime-provider-secret' not in text and connection['hermes_api_key'] not in text
assert (private / 'api-key').stat().st_mode & 0o777 == 0o600
assert private.stat().st_mode & 0o777 == 0o700
assert workspace != home
assert all(req.headers['Authorization'].startswith('Bearer ') for req in health)
def test_retries_restart_reuse_key_and_preserve_home(boundary):
p, docker, _ = boundary
docker.fail_start = True
with pytest.raises(ProvisioningError, match='HTTP 500'):
asyncio.run(p.ensure(ALICE))
private, home, workspace = p._paths(ALICE)
key = (private / 'api-key').read_text().strip()
(workspace / 'notes.txt').write_text('private work')
(home / 'config.yaml').write_text(json.dumps({'retained': True}))
result = asyncio.run(p.ensure(ALICE))
assert result['hermes_api_key'] == key
assert len(docker.created) == 1
restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
assert asyncio.run(restarted.ensure(ALICE)) == result
assert json.loads((home / 'config.yaml').read_text()) == {'retained': True}
asyncio.run(restarted.remove(ALICE))
asyncio.run(restarted.remove(ALICE))
assert docker.deleted == [('id-' + container_name(ALICE), {'force': 'true', 'v': 'false'})]
assert (workspace / 'notes.txt').read_text() == 'private work'
assert (private / 'api-key').read_text().strip() == key
assert not asyncio.run(restarted.ready(ALICE))
@pytest.mark.parametrize('labels', [{}, {'hope.managed': 'false', 'hope.user_id': ALICE['id']},
{'hope.managed': 'true', 'hope.user_id': BOB['id']}])
def test_no_adoption_or_deletion_of_mismatched_labels(boundary, labels):
p, docker, _ = boundary
docker.containers[container_name(ALICE)] = {'Id': 'uncontrolled', 'Config': {'Labels': labels}, 'State': {'Running': True}}
with pytest.raises(OwnershipError):
asyncio.run(p.ensure(ALICE))
with pytest.raises(OwnershipError):
asyncio.run(p.remove(ALICE))
assert not asyncio.run(p.ready(ALICE))
assert docker.deleted == [] and docker.created == []
assert not p.state_dir.exists()
def test_bounded_readiness_failure_retains_container_for_retry(boundary):
p, docker, _ = boundary
bad_health = httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(503)))
p.health = bad_health
with pytest.raises(ProvisioningError, match='readiness timed out'):
asyncio.run(p.ensure(ALICE))
assert len(docker.created) == 1 and not docker.deleted
assert container_name(ALICE) in docker.containers
assert p._read_key(ALICE)
asyncio.run(bad_health.aclose())
def test_untrusted_paths_and_public_bind_fail_closed(boundary):
p, docker, _ = boundary
for user in ({'id': '../bob', 'username': 'alice'}, {'id': ALICE['id'], 'username': '../bob'}):
with pytest.raises(ValueError):
asyncio.run(p.ensure(user))
asyncio.run(p.ensure(ALICE))
info = docker.containers[container_name(ALICE)]
info['NetworkSettings']['Ports']['8642/tcp'][0]['HostIp'] = '0.0.0.0'
assert not asyncio.run(p.ready(ALICE))
with pytest.raises(ProvisioningError, match='loopback'):
asyncio.run(p.ensure(ALICE))
def test_owned_container_cannot_rotate_missing_private_secret(boundary):
p, docker, _ = boundary
asyncio.run(p.ensure(ALICE))
private, _, _ = p._paths(ALICE)
key_path = private / 'api-key'
assert key_path.resolve() == p.state_dir / ALICE['id'] / 'api-key'
key_path.unlink()
with pytest.raises(ProvisioningError, match='no persisted API credential'):
asyncio.run(p.ensure(ALICE))
assert len(docker.created) == 1
def test_upstream_runtime_overrides(monkeypatch):
monkeypatch.setenv('HERMES_UPSTREAM_URL', 'http://127.0.0.1:9999/v1')
monkeypatch.setenv('HERMES_UPSTREAM_MODEL', 'served-model')
config = Provisioner.config()
assert config['model']['default'] == 'served-model'
provider = config['providers']['hope-upstream']
assert provider['api'] == 'http://127.0.0.1:9999/v1'
assert provider['models'] == ['served-model']
assert provider['default_model'] == 'served-model'
+49
View File
@@ -0,0 +1,49 @@
"""Offline bootstrap authorization: no anonymous or client-selected routing."""
import importlib.util
import json
from pathlib import Path
from unittest.mock import patch
import httpx
from fastapi.testclient import TestClient
spec = importlib.util.spec_from_file_location('hope_token', Path(__file__).parents[2] / 'web/token_server.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
def test_anonymous_and_cookie_origin_denied():
with patch.multiple(module, CONTROL_SECRET='x'*32, API_KEY='key', API_SECRET='secret'):
client = TestClient(module.app)
assert client.post('/token').status_code == 401
assert client.post('/token', headers={'Cookie':'hope_session='+'y'*40,'Origin':'https://evil.invalid'}).status_code == 403
def test_authenticated_route_is_server_derived():
seen=[]
uid='a'*32
def handler(request):
body=json.loads(request.content)
seen.append((request.url.path,body))
assert request.headers['X-Control-Secret']=='x'*32
if request.url.path.endswith('resolve-session'):
return httpx.Response(200,json={'user':{'id':uid,'display_name':'John Doe'},'hermes_url':'http://127.0.0.1:1234','hermes_api_key':'private'})
assert body['session_token']=='y'*40
assert body['user_id']==uid
assert body['room_name'].startswith('voice-'+uid+'-')
return httpx.Response(201,json={})
original=httpx.AsyncClient
def factory(**kwargs):
return original(transport=httpx.MockTransport(handler),**kwargs)
with patch.multiple(module,CONTROL_SECRET='x'*32,API_KEY='key',API_SECRET='secret'),patch.object(module.httpx,'AsyncClient',factory):
response=TestClient(module.app).post('/token',headers={'Authorization':'Bearer '+'y'*40},json={'user_id':'victim','room':'victim'})
assert response.status_code==200
body=response.json()
assert body['user']['id']==uid
assert 'hermes_api_key' not in body
import base64
claims=json.loads(base64.urlsafe_b64decode(body['token'].split('.')[1]+'=='))
assert claims['sub']=='user:'+uid
assert claims['video']['canPublishSources']==['microphone']
assert claims['exp']-claims['nbf']==305
assert response.headers['Cache-Control']=='no-store'