427 lines
20 KiB
Python
427 lines
20 KiB
Python
"""Offline Docker HTTP boundary tests; no Docker daemon or model calls."""
|
|
import asyncio
|
|
import copy
|
|
import json
|
|
import os
|
|
|
|
import httpx
|
|
import pytest
|
|
import yaml
|
|
import socket
|
|
|
|
from control.provisioner import OwnershipError, Provisioner, ProvisioningError, container_name
|
|
|
|
ALICE = {'id': 'a' * 32, 'username': 'alice'}
|
|
BOB = {'id': 'b' * 32, 'username': 'bob'}
|
|
|
|
|
|
class DockerBoundary:
|
|
def __init__(self):
|
|
self.containers, self.networks, self.created, self.deleted = {}, {}, [], []
|
|
self.fail_start = False
|
|
self.restarted = []
|
|
self.fail_restart = False
|
|
|
|
def __call__(self, request):
|
|
path, method = request.url.path, request.method
|
|
if path == '/networks/create':
|
|
spec = json.loads(request.content)
|
|
name = spec['Name']
|
|
self.networks[name] = dict(spec, Id='net-' + name, Internal=False)
|
|
return httpx.Response(201, json={'Id': 'net-' + name})
|
|
if path.startswith('/networks/'):
|
|
key = path.split('/')[2]
|
|
net = self.networks.get(key) or next((n for n in self.networks.values() if n['Id'] == key), None)
|
|
if method == 'DELETE' and net:
|
|
del self.networks[net['Name']]
|
|
return httpx.Response(204)
|
|
return httpx.Response(200, json=net) if net else httpx.Response(404)
|
|
if path == '/containers/create':
|
|
name = request.url.params['name']
|
|
if name in self.containers:
|
|
return httpx.Response(409)
|
|
spec = json.loads(request.content)
|
|
self.created.append(spec)
|
|
self.containers[name] = {'Id': 'id-' + name, 'Config': copy.deepcopy(spec),
|
|
'HostConfig': copy.deepcopy(spec['HostConfig']),
|
|
'State': {'Running': False}, 'NetworkSettings': {'Ports': {}}}
|
|
return httpx.Response(201, json={'Id': 'id-' + name})
|
|
key = path.split('/')[2]
|
|
info = self.containers.get(key) or next((c for c in self.containers.values() if c['Id'] == key), None)
|
|
if not info:
|
|
return httpx.Response(404)
|
|
if method == 'GET':
|
|
return httpx.Response(200, json=info)
|
|
if method == 'POST':
|
|
if path.endswith('/restart'):
|
|
self.restarted.append(key)
|
|
if self.fail_restart:
|
|
self.fail_restart = False
|
|
return httpx.Response(500)
|
|
elif not path.endswith('/start'):
|
|
raise AssertionError(path)
|
|
if self.fail_start:
|
|
self.fail_start = False
|
|
return httpx.Response(500)
|
|
info['State']['Running'] = True
|
|
info['NetworkSettings']['Ports'] = {'8642/tcp': [{'HostIp': '127.0.0.1', 'HostPort': str(31000 + len(self.containers))}]}
|
|
return httpx.Response(204)
|
|
self.deleted.append((key, dict(request.url.params)))
|
|
name = next(n for n, c in self.containers.items() if c is info)
|
|
del self.containers[name]
|
|
return httpx.Response(204)
|
|
|
|
|
|
@pytest.fixture
|
|
def boundary(tmp_path, monkeypatch):
|
|
monkeypatch.delenv('HERMES_UPSTREAM_URL', raising=False)
|
|
monkeypatch.delenv('HERMES_UPSTREAM_MODEL', raising=False)
|
|
docker, health_requests = DockerBoundary(), []
|
|
def health(request):
|
|
health_requests.append(request)
|
|
assert request.url.host == '127.0.0.1'
|
|
assert request.url.path == '/v1/models'
|
|
return httpx.Response(200, json={'data': []})
|
|
dc = httpx.AsyncClient(base_url='http://docker', transport=httpx.MockTransport(docker))
|
|
hc = httpx.AsyncClient(transport=httpx.MockTransport(health))
|
|
p = Provisioner(tmp_path / 'private', docker_client=dc, health_client=hc,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=os.geteuid() or 10000,
|
|
runtime_gid=os.getegid() or 10000, ready_timeout=.1, poll_interval=.001)
|
|
yield p, docker, health_requests
|
|
asyncio.run(dc.aclose())
|
|
asyncio.run(hc.aclose())
|
|
|
|
|
|
def test_account_isolation_and_runtime_configuration(boundary):
|
|
p, docker, health = boundary
|
|
async def exercise():
|
|
alice, bob = await p.ensure(ALICE), await p.ensure(BOB)
|
|
assert await p.ready(ALICE)
|
|
return alice, bob
|
|
alice, bob = asyncio.run(exercise())
|
|
assert alice['container_name'] == container_name(ALICE)
|
|
assert alice['hermes_url'].startswith('http://127.0.0.1:')
|
|
assert alice['hermes_api_key'] != bob['hermes_api_key']
|
|
assert len(alice['hermes_api_key']) >= 32
|
|
for user, spec, connection in zip((ALICE, BOB), docker.created, (alice, bob)):
|
|
assert spec['Labels'] == {'hope.managed': 'true', 'hope.user_id': user['id']}
|
|
assert spec['User'].split(':')[0] != '0'
|
|
assert spec['Entrypoint'] == ['/opt/hermes/.venv/bin/python']
|
|
env = dict(entry.split('=', 1) for entry in spec['Env'])
|
|
assert env['HERMES_UPSTREAM_API_KEY'] == 'runtime-provider-secret'
|
|
assert env['API_SERVER_KEY'] == connection['hermes_api_key']
|
|
assert env['API_SERVER_ENABLED'] == 'true'
|
|
assert env['API_SERVER_HOST'] == '127.0.0.1'
|
|
assert int(env['API_SERVER_PORT']) == int(connection['hermes_url'].rsplit(':', 1)[1])
|
|
host = spec['HostConfig']
|
|
assert host['NetworkMode'] == 'host'
|
|
assert 'PortBindings' not in host
|
|
assert 'ExposedPorts' not in spec
|
|
assert host['Memory'] > 0 and host['NanoCpus'] > 0 and host['PidsLimit'] > 0
|
|
assert host['Privileged'] is False and host['Init'] is True
|
|
assert host['CapDrop'] == ['ALL']
|
|
assert host['SecurityOpt'] == ['no-new-privileges:true']
|
|
assert len(host['Binds']) == 2
|
|
assert all(user['id'] in mount and 'docker.sock' not in mount for mount in host['Binds'])
|
|
private, home, workspace = p._paths(user)
|
|
text = (home / 'config.yaml').read_text()
|
|
config = json.loads(text)
|
|
assert config['model'] == {'default': 'auto', 'provider': 'custom:hope-upstream'}
|
|
assert config['providers']['hope-upstream'] == {'api': 'http://192.168.86.5:2099/v1',
|
|
'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': 'auto',
|
|
'models': {'auto': {'supports_vision': True}}, 'discover_models': False}
|
|
assert config['agent']['reasoning_effort'] == 'none'
|
|
assert config['terminal'] == {'backend': 'local', 'cwd': '/workspace'}
|
|
assert {'memory', 'browser', 'web', 'terminal', 'file', 'skills'} <= set(config['platform_toolsets']['api_server'])
|
|
assert config['gateway']['api_server']['max_concurrent_runs'] == 1
|
|
assert 'runtime-provider-secret' not in text and connection['hermes_api_key'] not in text
|
|
assert (private / 'api-key').stat().st_mode & 0o777 == 0o600
|
|
assert private.stat().st_mode & 0o777 == 0o700
|
|
assert workspace != home
|
|
assert all(req.headers['Authorization'].startswith('Bearer ') for req in health)
|
|
|
|
|
|
def test_retries_restart_reuse_key_and_preserve_home(boundary):
|
|
p, docker, _ = boundary
|
|
docker.fail_start = True
|
|
with pytest.raises(ProvisioningError, match='HTTP 500'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, home, workspace = p._paths(ALICE)
|
|
key = (private / 'api-key').read_text().strip()
|
|
(workspace / 'notes.txt').write_text('private work')
|
|
(home / 'config.yaml').write_text(json.dumps({'retained': True}))
|
|
result = asyncio.run(p.ensure(ALICE))
|
|
assert result['hermes_api_key'] == key
|
|
assert len(docker.created) == 1
|
|
restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
|
|
assert asyncio.run(restarted.ensure(ALICE)) == result
|
|
assert yaml.safe_load((home / 'config.yaml').read_text()) == {
|
|
'retained': True, 'gateway': {'api_server': {'host': '127.0.0.1', 'port': p._ports()[ALICE['id']]}}}
|
|
asyncio.run(restarted.remove(ALICE))
|
|
asyncio.run(restarted.remove(ALICE))
|
|
assert docker.deleted == [('id-' + container_name(ALICE), {'force': 'true', 'v': 'false'})]
|
|
assert (workspace / 'notes.txt').read_text() == 'private work'
|
|
assert (private / 'api-key').read_text().strip() == key
|
|
assert not asyncio.run(restarted.ready(ALICE))
|
|
|
|
|
|
@pytest.mark.parametrize('labels', [{}, {'hope.managed': 'false', 'hope.user_id': ALICE['id']},
|
|
{'hope.managed': 'true', 'hope.user_id': BOB['id']}])
|
|
def test_no_adoption_or_deletion_of_mismatched_labels(boundary, labels):
|
|
p, docker, _ = boundary
|
|
docker.containers[container_name(ALICE)] = {'Id': 'uncontrolled', 'Config': {'Labels': labels}, 'State': {'Running': True}}
|
|
with pytest.raises(OwnershipError):
|
|
asyncio.run(p.ensure(ALICE))
|
|
with pytest.raises(OwnershipError):
|
|
asyncio.run(p.remove(ALICE))
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
assert docker.deleted == [] and docker.created == []
|
|
assert not p.state_dir.exists()
|
|
|
|
|
|
def test_bounded_readiness_failure_retains_container_for_retry(boundary):
|
|
p, docker, _ = boundary
|
|
bad_health = httpx.AsyncClient(transport=httpx.MockTransport(lambda req: httpx.Response(503)))
|
|
p.health = bad_health
|
|
with pytest.raises(ProvisioningError, match='readiness timed out'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 1 and not docker.deleted
|
|
assert container_name(ALICE) in docker.containers
|
|
assert p._read_key(ALICE)
|
|
asyncio.run(bad_health.aclose())
|
|
|
|
|
|
def test_untrusted_paths_and_public_bind_fail_closed(boundary):
|
|
p, docker, _ = boundary
|
|
for user in ({'id': '../bob', 'username': 'alice'}, {'id': ALICE['id'], 'username': '../bob'}):
|
|
with pytest.raises(ValueError):
|
|
asyncio.run(p.ensure(user))
|
|
asyncio.run(p.ensure(ALICE))
|
|
info = docker.containers[container_name(ALICE)]
|
|
info['Config']['Env'] = [e.replace('API_SERVER_HOST=127.0.0.1', 'API_SERVER_HOST=0.0.0.0')
|
|
for e in info['Config']['Env']]
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
with pytest.raises(ProvisioningError, match='loopback'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
|
|
|
|
def test_owned_container_cannot_rotate_missing_private_secret(boundary):
|
|
p, docker, _ = boundary
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, _, _ = p._paths(ALICE)
|
|
key_path = private / 'api-key'
|
|
assert key_path.resolve() == p.state_dir / ALICE['id'] / 'api-key'
|
|
key_path.unlink()
|
|
with pytest.raises(ProvisioningError, match='no persisted API credential'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 1
|
|
|
|
|
|
@pytest.mark.parametrize('models', [
|
|
['auto', 'text-only'],
|
|
{'auto': {'supports_vision': False, 'context_window': 32768}, 'text-only': {'supports_vision': False}},
|
|
])
|
|
def test_auto_vision_migration_preserves_settings_and_reloads_once(boundary, models):
|
|
p, docker, _ = boundary
|
|
original = asyncio.run(p.ensure(ALICE))
|
|
private, home, workspace = p._paths(ALICE)
|
|
config = yaml.safe_load((home / 'config.yaml').read_text())
|
|
config['providers']['hope-upstream']['models'] = models
|
|
config['model']['default'] = 'text-only'
|
|
config['agent']['reasoning_effort'] = 'high'
|
|
config['retained'] = {'private': True}
|
|
(home / 'config.yaml').write_text(yaml.safe_dump(config))
|
|
(workspace / 'work.txt').write_text('retained')
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
assert asyncio.run(p.ensure(ALICE)) == original
|
|
result = yaml.safe_load((home / 'config.yaml').read_text())
|
|
expected = copy.deepcopy(config)
|
|
if isinstance(models, list):
|
|
expected['providers']['hope-upstream']['models'] = {m: {} for m in models}
|
|
expected['providers']['hope-upstream']['models']['auto']['supports_vision'] = True
|
|
assert result == expected
|
|
assert 'supports_vision' not in result['model']
|
|
assert docker.restarted == ['id-' + container_name(ALICE)]
|
|
assert not docker.deleted and len(docker.created) == 1
|
|
assert (workspace / 'work.txt').read_text() == 'retained'
|
|
assert not p._reload_marker(ALICE).exists()
|
|
assert asyncio.run(p.ready(ALICE))
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.restarted) == 1
|
|
|
|
|
|
def test_failed_reload_is_retried_after_control_restart(boundary):
|
|
p, docker, _ = boundary
|
|
original = asyncio.run(p.ensure(ALICE))
|
|
_, home, _ = p._paths(ALICE)
|
|
config = yaml.safe_load((home / 'config.yaml').read_text())
|
|
config['providers']['hope-upstream']['models'] = ['auto']
|
|
(home / 'config.yaml').write_text(yaml.safe_dump(config))
|
|
docker.fail_restart = True
|
|
with pytest.raises(ProvisioningError, match='HTTP 500'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert p._reload_marker(ALICE).exists()
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
|
|
assert asyncio.run(restarted.ensure(ALICE)) == original
|
|
assert len(docker.restarted) == 2 and not docker.deleted
|
|
assert not p._reload_marker(ALICE).exists()
|
|
|
|
|
|
def test_vision_patch_skips_unverified_models_and_preserves_metadata():
|
|
for config in ({'retained': True}, {'providers': {'hope-upstream': {'models': ['text-only']}}}):
|
|
original = copy.deepcopy(config)
|
|
assert Provisioner._declare_auto_vision(config) is False
|
|
assert config == original
|
|
for models in ('auto', [None], {'auto': None}):
|
|
with pytest.raises(ValueError):
|
|
Provisioner._declare_auto_vision({'providers': {'hope-upstream': {'models': models}}})
|
|
|
|
|
|
def test_upstream_runtime_overrides(monkeypatch):
|
|
monkeypatch.setenv('HERMES_UPSTREAM_URL', 'http://127.0.0.1:9999/v1')
|
|
monkeypatch.setenv('HERMES_UPSTREAM_MODEL', 'served-model')
|
|
config = Provisioner.config()
|
|
assert config['model']['default'] == 'served-model'
|
|
provider = config['providers']['hope-upstream']
|
|
assert provider['api'] == 'http://127.0.0.1:9999/v1'
|
|
assert provider['models'] == ['served-model']
|
|
assert provider['default_model'] == 'served-model'
|
|
|
|
|
|
def test_migrate_legacy_container_preserves_state_and_key(boundary):
|
|
p, docker, _ = boundary
|
|
old = asyncio.run(p.ensure(ALICE))
|
|
_, home, workspace = p._paths(ALICE)
|
|
(workspace / 'retained.txt').write_text('account work')
|
|
(home / 'config.yaml').write_text('custom: retained\ngateway:\n api_server:\n host: 0.0.0.0\n port: 8642\n')
|
|
info = docker.containers[container_name(ALICE)]
|
|
info['HostConfig']['NetworkMode'] = 'hope-hermes-net-' + ALICE['id']
|
|
net = info['HostConfig']['NetworkMode']
|
|
docker.networks[net] = {'Id': net, 'Name': net, 'Labels': info['Config']['Labels']}
|
|
assert not asyncio.run(p.ready(ALICE))
|
|
result = asyncio.run(p.ensure(ALICE))
|
|
assert result == old
|
|
assert len(docker.created) == 2 and len(docker.deleted) == 1
|
|
assert not docker.networks
|
|
assert (workspace / 'retained.txt').read_text() == 'account work'
|
|
config = yaml.safe_load((home / 'config.yaml').read_text())
|
|
assert config['custom'] == 'retained'
|
|
assert config['gateway']['api_server'] == {'host': '127.0.0.1', 'port': p._ports()[ALICE['id']]}
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 2
|
|
|
|
|
|
def test_ports_unique_concurrent_persisted_and_skip_busy_port(boundary):
|
|
p, docker, _ = boundary
|
|
with socket.socket() as occupied:
|
|
occupied.bind(('127.0.0.1', 0))
|
|
start = occupied.getsockname()[1]
|
|
p.api_port_range = (start, min(start + 10, 65535))
|
|
async def create():
|
|
return await asyncio.gather(p.ensure(ALICE), p.ensure(BOB))
|
|
alice, bob = asyncio.run(create())
|
|
ports = p._ports()
|
|
assert ports[ALICE['id']] != start
|
|
assert ports[ALICE['id']] != ports[BOB['id']]
|
|
assert (p.state_dir / 'api-ports.json').stat().st_mode & 0o777 == 0o600
|
|
restarted = Provisioner(p.state_dir, docker_client=p.docker, health_client=p.health,
|
|
upstream_api_key='runtime-provider-secret', runtime_uid=p.runtime_uid, runtime_gid=p.runtime_gid)
|
|
assert asyncio.run(restarted.ensure(ALICE)) == alice
|
|
|
|
|
|
def test_port_exhaustion_and_corrupt_registry_fail_closed(boundary):
|
|
p, docker, _ = boundary
|
|
with socket.socket() as occupied:
|
|
occupied.bind(('127.0.0.1', 0))
|
|
p.api_port_range = (occupied.getsockname()[1], occupied.getsockname()[1])
|
|
with pytest.raises(ProvisioningError, match='No available'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
p._directory(p.state_dir)
|
|
p._write_private(p.state_dir / 'api-ports.json', json.dumps({ALICE['id']: 20000, BOB['id']: 20000}))
|
|
with pytest.raises(ProvisioningError, match='Duplicate'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert not docker.created
|
|
|
|
|
|
def test_migration_start_failure_retries_without_losing_data(boundary):
|
|
p, docker, _ = boundary
|
|
result = asyncio.run(p.ensure(ALICE))
|
|
docker.containers[container_name(ALICE)]['HostConfig']['NetworkMode'] = 'legacy'
|
|
docker.fail_start = True
|
|
with pytest.raises(ProvisioningError, match='HTTP 500'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert asyncio.run(p.ensure(ALICE)) == result
|
|
assert len(docker.created) == 2
|
|
|
|
|
|
def test_webui_migration_unique_ports_passwords_and_cookies(boundary, monkeypatch):
|
|
p, docker, _ = boundary
|
|
original = asyncio.run(p.ensure(ALICE))
|
|
p.webui_enabled = True
|
|
p.webui_host = '127.0.0.1'
|
|
p.image = 'hope-hermes-webui:62e4d7b'
|
|
async def healthy(connection):
|
|
return True
|
|
monkeypatch.setattr(p, '_healthy', healthy)
|
|
alice, bob = asyncio.run(p.ensure(ALICE)), asyncio.run(p.ensure(BOB))
|
|
assert alice == original
|
|
assert len(docker.created) == 3
|
|
a = dict(e.split('=', 1) for e in docker.created[1]['Env'])
|
|
b = dict(e.split('=', 1) for e in docker.created[2]['Env'])
|
|
assert a['HERMES_WEBUI_PORT'] != b['HERMES_WEBUI_PORT']
|
|
assert a['HERMES_WEBUI_PASSWORD'] != b['HERMES_WEBUI_PASSWORD']
|
|
assert a['HERMES_WEBUI_COOKIE_NAME'] != b['HERMES_WEBUI_COOKIE_NAME']
|
|
assert a['HERMES_WEBUI_PASSWORD'] != alice['hermes_api_key']
|
|
assert a['HERMES_WEBUI_GATEWAY_API_KEY'] == alice['hermes_api_key']
|
|
assert a['HERMES_WEBUI_GATEWAY_BASE_URL'] == alice['hermes_url']
|
|
assert a['HERMES_WEBUI_SECURE'] == '1'
|
|
assert a['HERMES_WEBUI_CHAT_BACKEND'] == 'gateway'
|
|
password = (p.state_dir / ALICE['id'] / 'webui-password')
|
|
assert password.stat().st_mode & 0o777 == 0o600
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 3
|
|
|
|
|
|
def test_webui_password_symlink_fails_closed(boundary, monkeypatch):
|
|
p, docker, _ = boundary
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, _, _ = p._paths(ALICE)
|
|
(private / 'webui-password').symlink_to(private / 'api-key')
|
|
p.webui_enabled = True
|
|
p.webui_host = '127.0.0.1'
|
|
with pytest.raises(ProvisioningError, match='symlink'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert len(docker.created) == 1
|
|
|
|
|
|
@pytest.mark.parametrize('password', ['x', 'short passphrase'])
|
|
def test_webui_accepts_user_chosen_short_password(boundary, monkeypatch, password):
|
|
p, docker, _ = boundary
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, _, _ = p._paths(ALICE)
|
|
(private / 'webui-password').write_text(password + '\n')
|
|
p.webui_enabled = True
|
|
p.webui_host = '127.0.0.1'
|
|
async def healthy(connection):
|
|
return True
|
|
monkeypatch.setattr(p, '_healthy', healthy)
|
|
asyncio.run(p.ensure(ALICE))
|
|
env = dict(entry.split('=', 1) for entry in docker.created[-1]['Env'])
|
|
assert env['HERMES_WEBUI_PASSWORD'] == password
|
|
|
|
|
|
@pytest.mark.parametrize('password', ['', ' ', 'first\nsecond'])
|
|
def test_webui_rejects_blank_or_multiline_password_before_recreation(boundary, password):
|
|
p, docker, _ = boundary
|
|
asyncio.run(p.ensure(ALICE))
|
|
private, _, _ = p._paths(ALICE)
|
|
(private / 'webui-password').write_text(password)
|
|
p.webui_enabled = True
|
|
p.webui_host = '127.0.0.1'
|
|
with pytest.raises(ProvisioningError, match='nonempty and single-line'):
|
|
asyncio.run(p.ensure(ALICE))
|
|
assert not docker.deleted
|