feat: provision private Hermes assistants for authenticated Hope accounts

tests/unit/test_provisioner.py

@@ -0,0 +1,214 @@
+"""Offline Docker HTTP boundary tests; no Docker daemon or model calls."""
+import asyncio
+import copy
+import json
+import os
+
+import httpx
+import pytest
+
+from control.provisioner import OwnershipError, Provisioner, ProvisioningError, container_name
+
+ALICE = {'id': 'a' * 32, 'username': 'alice'}
+BOB = {'id': 'b' * 32, 'username': 'bob'}
+
+
+class DockerBoundary:
+    def __init__(self):
+        self.containers, self.networks, self.created, self.deleted = {}, {}, [], []
+        self.fail_start = False
+
+    def __call__(self, request):
+        path, method = request.url.path, request.method
+        if path == '/networks/create':
+            spec = json.loads(request.content)
+            name = spec['Name']
+            self.networks[name] = dict(spec, Id='net-' + name, Internal=False)
+            return httpx.Response(201, json={'Id': 'net-' + name})
+        if path.startswith('/networks/'):
+            key = path.split('/')[2]
+            net = self.networks.get(key) or next((n for n in self.networks.values() if n['Id'] == key), None)
+            if method == 'DELETE' and net:
+                del self.networks[net['Name']]
+                return httpx.Response(204)
+            return httpx.Response(200, json=net) if net else httpx.Response(404)
+        if path == '/containers/create':
+            name = request.url.params['name']
+            if name in self.containers:
+                return httpx.Response(409)
+            spec = json.loads(request.content)
+            self.created.append(spec)
+            self.containers[name] = {'Id': 'id-' + name, 'Config': copy.deepcopy(spec),
+                'State': {'Running': False}, 'NetworkSettings': {'Ports': {}}}
+            return httpx.Response(201, json={'Id': 'id-' + name})
+        key = path.split('/')[2]
+        info = self.containers.get(key) or next((c for c in self.containers.values() if c['Id'] == key), None)
+        if not info:
+            return httpx.Response(404)
+        if method == 'GET':
+            return httpx.Response(200, json=info)
+        if method == 'POST':
+            if self.fail_start:
+                self.fail_start = False
+                return httpx.Response(500)
+            info['State']['Running'] = True
+            info['NetworkSettings']['Ports'] = {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': str(31000 + len(self.containers))}]}
+            return httpx.Response(204)
+        self.deleted.append((key, dict(request.url.params)))
+        name = next(n for n, c in self.containers.items() if c is info)
+        del self.containers[name]
+        return httpx.Response(204)
+
+
+@pytest.fixture
+def boundary(tmp_path, monkeypatch):
+    monkeypatch.delenv('HERMES_UPSTREAM_URL', raising=False)
+    monkeypatch.delenv('HERMES_UPSTREAM_MODEL', raising=False)
+    docker, health_requests = DockerBoundary(), []
+    def health(request):
+        health_requests.append(request)
+        assert request.url.host == '127.0.0.1'
+        assert request.url.path == '/v1/models'
+        return httpx.Response(200, json={'data': []})
+    dc = httpx.AsyncClient(base_url='http://docker', transport=httpx.MockTransport(docker))
+    hc = httpx.AsyncClient(transport=httpx.MockTransport(health))
+    p = Provisioner(tmp_path / 'private', docker_client=dc, health_client=hc,
+        upstream_api_key='runtime-provider-secret', runtime_uid=os.geteuid() or 10000,
+        runtime_gid=os.getegid() or 10000, ready_timeout=.1, poll_interval=.001)
+    yield p, docker, health_requests
+    asyncio.run(dc.aclose())
+    asyncio.run(hc.aclose())
+
+
+def test_account_isolation_and_runtime_configuration(boundary):
+    p, docker, health = boundary
+    async def exercise():
+        alice, bob = await p.ensure(ALICE), await p.ensure(BOB)
+        assert await p.ready(ALICE)
+        return alice, bob
+    alice, bob = asyncio.run(exercise())
+    assert alice['container_name'] == container_name(ALICE)
+    assert alice['hermes_url'].startswith('http://127.0.0.1:')
+    assert alice['hermes_api_key'] != bob['hermes_api_key']
+    assert len(alice['hermes_api_key']) >= 32
+    for user, spec, connection in zip((ALICE, BOB), docker.created, (alice, bob)):
+        assert spec['Labels'] == {'hope.managed': 'true', 'hope.user_id': user['id']}
+        assert spec['User'].split(':')[0] != '0'
+        assert spec['Entrypoint'] == ['/opt/hermes/.venv/bin/python']
+        env = dict(entry.split('=', 1) for entry in spec['Env'])
+        assert env['HERMES_UPSTREAM_API_KEY'] == 'runtime-provider-secret'
+        assert env['API_SERVER_KEY'] == connection['hermes_api_key']
+        assert env['API_SERVER_ENABLED'] == 'true'
+        assert env['API_SERVER_HOST'] == '0.0.0.0'
+        assert env['API_SERVER_PORT'] == '8642'
+        host = spec['HostConfig']
+        assert host['NetworkMode'] == 'hope-hermes-net-' + user['id']
+        assert host['PortBindings']['8642/tcp'] == [{'HostIp': '127.0.0.1', 'HostPort': ''}]
+        assert host['Memory'] > 0 and host['NanoCpus'] > 0 and host['PidsLimit'] > 0
+        assert host['Privileged'] is False and host['Init'] is True
+        assert host['CapDrop'] == ['ALL']
+        assert host['SecurityOpt'] == ['no-new-privileges:true']
+        assert len(host['Binds']) == 2
+        assert all(user['id'] in mount and 'docker.sock' not in mount for mount in host['Binds'])
+        private, home, workspace = p._paths(user)
+        text = (home / 'config.yaml').read_text()
+        config = json.loads(text)
+        assert config['model'] == {'default': 'auto', 'provider': 'custom:hope-upstream'}
+        assert config['providers']['hope-upstream'] == {'api': 'http://192.168.86.5:2099/v1',
+            'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': 'auto',
+            'models': ['auto'], 'discover_models': False}
+        assert config['agent']['reasoning_effort'] == 'none'
+        assert config['terminal'] == {'backend': 'local', 'cwd': '/workspace'}
+        assert {'memory', 'browser', 'web', 'terminal', 'file', 'skills'} <= set(config['platform_toolsets']['api_server'])
+        assert config['gateway']['api_server']['max_concurrent_runs'] == 1
+        assert 'runtime-provider-secret' not in text and connection['hermes_api_key'] not in text
+        assert (private / 'api-key').stat().st_mode & 0o777 == 0o600
+        assert private.stat().st_mode & 0o777 == 0o700
+        assert workspace != home
+    assert all(req.headers['Authorization'].startswith('Bearer ') for req in health)
+
+
+def test_retries_restart_reuse_key_and_preserve_home(boundary):
+    p, docker, _ = boundary
+    docker.fail_start = True
+    with pytest.raises(ProvisioningError, match='HTTP 500'):
+        asyncio.run(p.ensure(ALICE))
+    private, home, workspace = p._paths(ALICE)
+    key = (private / 'api-key').read_text().strip()
+    (workspace / 'notes.txt').write_text('private work')
+    (home / 'config.yaml').write_text(json.dumps({'retained': True}))
+    result = asyncio.run(p.ensure(ALICE))
+    assert result['hermes_api_key'] == key
+    assert len(docker.created) == 1
+    restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
+        upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
+    assert asyncio.run(restarted.ensure(ALICE)) == result
+    assert json.loads((home / 'config.yaml').read_text()) == {'retained': True}
+    asyncio.run(restarted.remove(ALICE))
+    asyncio.run(restarted.remove(ALICE))
+    assert docker.deleted == [('id-' + container_name(ALICE), {'force': 'true', 'v': 'false'})]
+    assert (workspace / 'notes.txt').read_text() == 'private work'
+    assert (private / 'api-key').read_text().strip() == key
+    assert not asyncio.run(restarted.ready(ALICE))
+
+
+@pytest.mark.parametrize('labels', [{}, {'hope.managed': 'false', 'hope.user_id': ALICE['id']},
+    {'hope.managed': 'true', 'hope.user_id': BOB['id']}])
+def test_no_adoption_or_deletion_of_mismatched_labels(boundary, labels):
+    p, docker, _ = boundary
+    docker.containers[container_name(ALICE)] = {'Id': 'uncontrolled', 'Config': {'Labels': labels}, 'State': {'Running': True}}
+    with pytest.raises(OwnershipError):
+        asyncio.run(p.ensure(ALICE))
+    with pytest.raises(OwnershipError):
+        asyncio.run(p.remove(ALICE))
+    assert not asyncio.run(p.ready(ALICE))
+    assert docker.deleted == [] and docker.created == []
+    assert not p.state_dir.exists()
+
+
+def test_bounded_readiness_failure_retains_container_for_retry(boundary):
+    p, docker, _ = boundary
+    bad_health = httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(503)))
+    p.health = bad_health
+    with pytest.raises(ProvisioningError, match='readiness timed out'):
+        asyncio.run(p.ensure(ALICE))
+    assert len(docker.created) == 1 and not docker.deleted
+    assert container_name(ALICE) in docker.containers
+    assert p._read_key(ALICE)
+    asyncio.run(bad_health.aclose())
+
+
+def test_untrusted_paths_and_public_bind_fail_closed(boundary):
+    p, docker, _ = boundary
+    for user in ({'id': '../bob', 'username': 'alice'}, {'id': ALICE['id'], 'username': '../bob'}):
+        with pytest.raises(ValueError):
+            asyncio.run(p.ensure(user))
+    asyncio.run(p.ensure(ALICE))
+    info = docker.containers[container_name(ALICE)]
+    info['NetworkSettings']['Ports']['8642/tcp'][0]['HostIp'] = '0.0.0.0'
+    assert not asyncio.run(p.ready(ALICE))
+    with pytest.raises(ProvisioningError, match='loopback'):
+        asyncio.run(p.ensure(ALICE))
+
+
+def test_owned_container_cannot_rotate_missing_private_secret(boundary):
+    p, docker, _ = boundary
+    asyncio.run(p.ensure(ALICE))
+    private, _, _ = p._paths(ALICE)
+    key_path = private / 'api-key'
+    assert key_path.resolve() == p.state_dir / ALICE['id'] / 'api-key'
+    key_path.unlink()
+    with pytest.raises(ProvisioningError, match='no persisted API credential'):
+        asyncio.run(p.ensure(ALICE))
+    assert len(docker.created) == 1
+
+
+def test_upstream_runtime_overrides(monkeypatch):
+    monkeypatch.setenv('HERMES_UPSTREAM_URL', 'http://127.0.0.1:9999/v1')
+    monkeypatch.setenv('HERMES_UPSTREAM_MODEL', 'served-model')
+    config = Provisioner.config()
+    assert config['model']['default'] == 'served-model'
+    provider = config['providers']['hope-upstream']
+    assert provider['api'] == 'http://127.0.0.1:9999/v1'
+    assert provider['models'] == ['served-model']
+    assert provider['default_model'] == 'served-model'