215 lines
10 KiB
Python
215 lines
10 KiB
Python
"""Offline Docker HTTP boundary tests; no Docker daemon or model calls."""
|
|
import asyncio
|
|
import copy
|
|
import json
|
|
import os
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from control.provisioner import OwnershipError, Provisioner, ProvisioningError, container_name
|
|
|
|
ALICE = {'id': 'a' * 32, 'username': 'alice'}
|
|
BOB = {'id': 'b' * 32, 'username': 'bob'}
|
|
|
|
|
|
class DockerBoundary:
|
|
def __init__(self):
|
|
self.containers, self.networks, self.created, self.deleted = {}, {}, [], []
|
|
self.fail_start = False
|
|
|
|
def __call__(self, request):
|
|
path, method = request.url.path, request.method
|
|
if path == '/networks/create':
|
|
spec = json.loads(request.content)
|
|
name = spec['Name']
|
|
self.networks[name] = dict(spec, Id='net-' + name, Internal=False)
|
|
return httpx.Response(201, json={'Id': 'net-' + name})
|
|
if path.startswith('/networks/'):
|
|
key = path.split('/')[2]
|
|
net = self.networks.get(key) or next((n for n in self.networks.values() if n['Id'] == key), None)
|
|
if method == 'DELETE' and net:
|
|
del self.networks[net['Name']]
|
|
return httpx.Response(204)
|
|
return httpx.Response(200, json=net) if net else httpx.Response(404)
|
|
if path == '/containers/create':
|
|
name = request.url.params['name']
|
|
if name in self.containers:
|
|
return httpx.Response(409)
|
|
spec = json.loads(request.content)
|
|
self.created.append(spec)
|
|
self.containers[name] = {'Id': 'id-' + name, 'Config': copy.deepcopy(spec),
|
|
'State': {'Running': False}, 'NetworkSettings': {'Ports': {}}}
|
|
return httpx.Response(201, json={'Id': 'id-' + name})
|
|
key = path.split('/')[2]
|
|
info = self.containers.get(key) or next((c for c in self.containers.values() if c['Id'] == key), None)
|
|
if not info:
|
|
return httpx.Response(404)
|
|
if method == 'GET':
|
|
return httpx.Response(200, json=info)
|
|
if method == 'POST':
|
|
if self.fail_start:
|
|
self.fail_start = False
|
|
return httpx.Response(500)
|
|
info['State']['Running'] = True
|
|
info['NetworkSettings']['Ports'] = {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': str(31000 + len(self.containers))}]}
|
|
return httpx.Response(204)
|
|
self.deleted.append((key, dict(request.url.params)))
|
|
name = next(n for n, c in self.containers.items() if c is info)
|
|
del self.containers[name]
|
|
return httpx.Response(204)
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path, monkeypatch):
|
|
monkeypatch.delenv('HERMES_UPSTREAM_URL', raising=False)
|
|
monkeypatch.delenv('HERMES_UPSTREAM_MODEL', raising=False)
|
|
docker, health_requests = DockerBoundary(), []
|
|
def health(request):
|
|
health_requests.append(request)
|
|
assert request.url.host == '127.0.0.1'
|
|
assert request.url.path == '/v1/models'
|
|
return httpx.Response(200, json={'data': []})
|
|
dc = httpx.AsyncClient(base_url='http://docker', transport=httpx.MockTransport(docker))
|
|
hc = httpx.AsyncClient(transport=httpx.MockTransport(health))
|
|
p = Provisioner(tmp_path / 'private', docker_client=dc, health_client=hc,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=os.geteuid() or 10000,
|
|
runtime_gid=os.getegid() or 10000, ready_timeout=.1, poll_interval=.001)
|
|
yield p, docker, health_requests
|
|
asyncio.run(dc.aclose())
|
|
asyncio.run(hc.aclose())
|
|
|
|
|
|
def test_account_isolation_and_runtime_configuration(boundary):
|
|
p, docker, health = boundary
|
|
async def exercise():
|
|
alice, bob = await p.ensure(ALICE), await p.ensure(BOB)
|
|
assert await p.ready(ALICE)
|
|
return alice, bob
|
|
alice, bob = asyncio.run(exercise())
|
|
assert alice['container_name'] == container_name(ALICE)
|
|
assert alice['hermes_url'].startswith('http://127.0.0.1:')
|
|
assert alice['hermes_api_key'] != bob['hermes_api_key']
|
|
assert len(alice['hermes_api_key']) >= 32
|
|
for user, spec, connection in zip((ALICE, BOB), docker.created, (alice, bob)):
|
|
assert spec['Labels'] == {'hope.managed': 'true', 'hope.user_id': user['id']}
|
|
assert spec['User'].split(':')[0] != '0'
|
|
assert spec['Entrypoint'] == ['/opt/hermes/.venv/bin/python']
|
|
env = dict(entry.split('=', 1) for entry in spec['Env'])
|
|
assert env['HERMES_UPSTREAM_API_KEY'] == 'runtime-provider-secret'
|
|
assert env['API_SERVER_KEY'] == connection['hermes_api_key']
|
|
assert env['API_SERVER_ENABLED'] == 'true'
|
|
assert env['API_SERVER_HOST'] == '0.0.0.0'
|
|
assert env['API_SERVER_PORT'] == '8642'
|
|
host = spec['HostConfig']
|
|
assert host['NetworkMode'] == 'hope-hermes-net-' + user['id']
|
|
assert host['PortBindings']['8642/tcp'] == [{'HostIp': '127.0.0.1', 'HostPort': ''}]
|
|
assert host['Memory'] > 0 and host['NanoCpus'] > 0 and host['PidsLimit'] > 0
|
|
assert host['Privileged'] is False and host['Init'] is True
|
|
assert host['CapDrop'] == ['ALL']
|
|
assert host['SecurityOpt'] == ['no-new-privileges:true']
|
|
assert len(host['Binds']) == 2
|
|
assert all(user['id'] in mount and 'docker.sock' not in mount for mount in host['Binds'])
|
|
private, home, workspace = p._paths(user)
|
|
text = (home / 'config.yaml').read_text()
|
|
config = json.loads(text)
|
|
assert config['model'] == {'default': 'auto', 'provider': 'custom:hope-upstream'}
|
|
assert config['providers']['hope-upstream'] == {'api': 'http://192.168.86.5:2099/v1',
|
|
'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': 'auto',
|
|
'models': ['auto'], 'discover_models': False}
|
|
assert config['agent']['reasoning_effort'] == 'none'
|
|
assert config['terminal'] == {'backend': 'local', 'cwd': '/workspace'}
|
|
assert {'memory', 'browser', 'web', 'terminal', 'file', 'skills'} <= set(config['platform_toolsets']['api_server'])
|
|
assert config['gateway']['api_server']['max_concurrent_runs'] == 1
|
|
assert 'runtime-provider-secret' not in text and connection['hermes_api_key'] not in text
|
|
assert (private / 'api-key').stat().st_mode & 0o777 == 0o600
|
|
assert private.stat().st_mode & 0o777 == 0o700
|
|
assert workspace != home
|
|
assert all(req.headers['Authorization'].startswith('Bearer ') for req in health)
|
|
|
|
|
|
def test_retries_restart_reuse_key_and_preserve_home(boundary):
|
|
p, docker, _ = boundary
|
|
docker.fail_start = True
|
|
with pytest.raises(ProvisioningError, match='HTTP 500'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, home, workspace = p._paths(ALICE)
|
|
key = (private / 'api-key').read_text().strip()
|
|
(workspace / 'notes.txt').write_text('private work')
|
|
(home / 'config.yaml').write_text(json.dumps({'retained': True}))
|
|
result = asyncio.run(p.ensure(ALICE))
|
|
assert result['hermes_api_key'] == key
|
|
assert len(docker.created) == 1
|
|
restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
|
|
assert asyncio.run(restarted.ensure(ALICE)) == result
|
|
assert json.loads((home / 'config.yaml').read_text()) == {'retained': True}
|
|
asyncio.run(restarted.remove(ALICE))
|
|
asyncio.run(restarted.remove(ALICE))
|
|
assert docker.deleted == [('id-' + container_name(ALICE), {'force': 'true', 'v': 'false'})]
|
|
assert (workspace / 'notes.txt').read_text() == 'private work'
|
|
assert (private / 'api-key').read_text().strip() == key
|
|
assert not asyncio.run(restarted.ready(ALICE))
|
|
|
|
|
|
@pytest.mark.parametrize('labels', [{}, {'hope.managed': 'false', 'hope.user_id': ALICE['id']},
|
|
{'hope.managed': 'true', 'hope.user_id': BOB['id']}])
|
|
def test_no_adoption_or_deletion_of_mismatched_labels(boundary, labels):
|
|
p, docker, _ = boundary
|
|
docker.containers[container_name(ALICE)] = {'Id': 'uncontrolled', 'Config': {'Labels': labels}, 'State': {'Running': True}}
|
|
with pytest.raises(OwnershipError):
|
|
asyncio.run(p.ensure(ALICE))
|
|
with pytest.raises(OwnershipError):
|
|
asyncio.run(p.remove(ALICE))
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
assert docker.deleted == [] and docker.created == []
|
|
assert not p.state_dir.exists()
|
|
|
|
|
|
def test_bounded_readiness_failure_retains_container_for_retry(boundary):
|
|
p, docker, _ = boundary
|
|
bad_health = httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(503)))
|
|
p.health = bad_health
|
|
with pytest.raises(ProvisioningError, match='readiness timed out'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 1 and not docker.deleted
|
|
assert container_name(ALICE) in docker.containers
|
|
assert p._read_key(ALICE)
|
|
asyncio.run(bad_health.aclose())
|
|
|
|
|
|
def test_untrusted_paths_and_public_bind_fail_closed(boundary):
|
|
p, docker, _ = boundary
|
|
for user in ({'id': '../bob', 'username': 'alice'}, {'id': ALICE['id'], 'username': '../bob'}):
|
|
with pytest.raises(ValueError):
|
|
asyncio.run(p.ensure(user))
|
|
asyncio.run(p.ensure(ALICE))
|
|
info = docker.containers[container_name(ALICE)]
|
|
info['NetworkSettings']['Ports']['8642/tcp'][0]['HostIp'] = '0.0.0.0'
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
with pytest.raises(ProvisioningError, match='loopback'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
|
|
|
|
def test_owned_container_cannot_rotate_missing_private_secret(boundary):
|
|
p, docker, _ = boundary
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, _, _ = p._paths(ALICE)
|
|
key_path = private / 'api-key'
|
|
assert key_path.resolve() == p.state_dir / ALICE['id'] / 'api-key'
|
|
key_path.unlink()
|
|
with pytest.raises(ProvisioningError, match='no persisted API credential'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 1
|
|
|
|
|
|
def test_upstream_runtime_overrides(monkeypatch):
|
|
monkeypatch.setenv('HERMES_UPSTREAM_URL', 'http://127.0.0.1:9999/v1')
|
|
monkeypatch.setenv('HERMES_UPSTREAM_MODEL', 'served-model')
|
|
config = Provisioner.config()
|
|
assert config['model']['default'] == 'served-model'
|
|
provider = config['providers']['hope-upstream']
|
|
assert provider['api'] == 'http://127.0.0.1:9999/v1'
|
|
assert provider['models'] == ['served-model']
|
|
assert provider['default_model'] == 'served-model'
|