526 lines
27 KiB
Python
526 lines
27 KiB
Python
"""Trusted Docker-UDS control plane for independent Hermes accounts.
|
|
|
|
Hermes uses host networking for LAN egress with per-account loopback listeners.
|
|
Only this service may receive the host Docker socket. Bind-source paths must be
|
|
identical in the control service and Docker host (mount the runtime root there).
|
|
The official image's root/s6 boot is deliberately bypassed: a nonroot Python
|
|
bootstrap seeds bundled skills, then execs the verified .venv Hermes CLI. Image
|
|
compatibility still needs deployment qualification before an image pin is promoted.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import socket
|
|
import tempfile
|
|
|
|
import yaml
|
|
|
|
import httpx
|
|
|
|
|
|
class ProvisioningError(RuntimeError):
|
|
"""Safe-to-display provisioning failure (never includes Docker env/secrets)."""
|
|
|
|
|
|
class OwnershipError(ProvisioningError):
|
|
"""An existing resource is not owned by this account."""
|
|
|
|
|
|
# Run as the image's hermes UID without executing root-only s6 startup. No source
|
|
# checkout/profile is copied: only the image's public bundled skills are seeded.
|
|
_BOOTSTRAP = """
|
|
import os, pathlib, shutil, subprocess, signal, time, ssl
|
|
home = pathlib.Path('/opt/data')
|
|
for name in ('home', 'sessions', 'memories', 'skills', 'cron', 'hooks', 'logs', 'cache'):
|
|
(home / name).mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
source = pathlib.Path('/opt/hermes/skills')
|
|
if source.is_dir() and not (home / '.hope-skills-seeded').exists():
|
|
shutil.copytree(source, home / 'skills', dirs_exist_ok=True)
|
|
(home / '.hope-skills-seeded').touch(mode=0o600)
|
|
os.makedirs('/tmp/hermes-runtime', mode=0o700, exist_ok=True)
|
|
if os.environ.get('HOPE_WEBUI_ENABLED') != 'true':
|
|
os.execv('/opt/hermes/.venv/bin/hermes', ['hermes', 'gateway', 'run'])
|
|
# Generate a private per-account certificate once; never start without usable TLS.
|
|
from cryptography import x509
|
|
from cryptography.x509.oid import NameOID
|
|
from cryptography.hazmat.primitives import hashes, serialization
|
|
from cryptography.hazmat.primitives.asymmetric import rsa
|
|
import datetime, ipaddress
|
|
tls = home / 'webui-tls'
|
|
tls.mkdir(mode=0o700, exist_ok=True)
|
|
cert, key = tls / 'cert.pem', tls / 'key.pem'
|
|
if not cert.exists() and not key.exists():
|
|
private = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
|
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'Hope Hermes WebUI')])
|
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
names = [x509.DNSName('localhost'), x509.IPAddress(ipaddress.ip_address('127.0.0.1')),
|
|
x509.IPAddress(ipaddress.ip_address(os.environ['HERMES_WEBUI_HOST']))]
|
|
certificate = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject)
|
|
.public_key(private.public_key()).serial_number(x509.random_serial_number())
|
|
.not_valid_before(now - datetime.timedelta(minutes=5))
|
|
.not_valid_after(now + datetime.timedelta(days=3650))
|
|
.add_extension(x509.SubjectAlternativeName(names), critical=False).sign(private, hashes.SHA256()))
|
|
key.write_bytes(private.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
|
|
serialization.NoEncryption()))
|
|
os.chmod(key, 0o600)
|
|
cert.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
|
|
ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER).load_cert_chain(str(cert), str(key))
|
|
children = []
|
|
def stop(signum=None, frame=None):
|
|
for child in children:
|
|
if child.poll() is None:
|
|
child.terminate()
|
|
signal.signal(signal.SIGTERM, stop)
|
|
signal.signal(signal.SIGINT, stop)
|
|
try:
|
|
children.append(subprocess.Popen(['/opt/hermes/.venv/bin/hermes', 'gateway', 'run']))
|
|
children.append(subprocess.Popen(['/opt/hermes/.venv/bin/python', '/opt/hermes-webui/bootstrap.py',
|
|
'--foreground', '--no-browser', '--skip-agent-install']))
|
|
while all(child.poll() is None for child in children):
|
|
time.sleep(.5)
|
|
finally:
|
|
stop()
|
|
for child in children:
|
|
try:
|
|
child.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
child.kill()
|
|
child.wait()
|
|
raise SystemExit(1)
|
|
"""
|
|
|
|
|
|
def container_name(user: dict) -> str:
|
|
uid, username = user.get('id'), user.get('username')
|
|
if not isinstance(uid, str) or not re.fullmatch(r'[0-9a-f]{32}', uid):
|
|
raise ValueError('Account id must be a lowercase UUID hex string')
|
|
if not isinstance(username, str) or not re.fullmatch(r'[a-z][a-z0-9]*(?:-[a-z0-9]+)*', username) or len(username) > 48:
|
|
raise ValueError('Account username must be a lowercase slug')
|
|
return f'hope-hermes-{username}-{uid[:8]}'
|
|
|
|
|
|
class Provisioner:
|
|
def __init__(self, state_dir=None, *, docker_socket=None, image=None,
|
|
upstream_api_key=None, docker_client=None, health_client=None,
|
|
ready_timeout=60, poll_interval=.5, runtime_uid=10000,
|
|
runtime_gid=10000, api_port_range=None, memory_bytes=2 * 1024 ** 3, nano_cpus=2_000_000_000):
|
|
bounds = api_port_range or os.environ.get('HERMES_API_PORT_RANGE', '20000-29999')
|
|
try:
|
|
if isinstance(bounds, str):
|
|
bounds = tuple(map(int, bounds.split('-')))
|
|
if len(bounds) != 2 or any(type(port) is not int for port in bounds) or not 1024 <= bounds[0] <= bounds[1] <= 65535:
|
|
raise ValueError()
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError('Invalid Hermes API port range') from exc
|
|
self.api_port_range = bounds
|
|
self.state_dir = Path(state_dir or os.environ.get('HOPE_HERMES_DATA_DIR', '.runtime/hermes')).absolute()
|
|
self.webui_enabled = os.environ.get('HOPE_WEBUI_ENABLED', 'false').lower() == 'true'
|
|
self.webui_host = os.environ.get('HERMES_WEBUI_HOST', '192.168.86.2')
|
|
self.image = image or os.environ.get('HERMES_IMAGE', 'nousresearch/hermes-agent@sha256:d4da4a40cd7a28aba983775d9fd31d94cbf153eeb0cb9e844d6d0f612b7c24db')
|
|
self.upstream_api_key = upstream_api_key if upstream_api_key is not None else os.environ.get('HERMES_UPSTREAM_API_KEY', '')
|
|
self.ready_timeout, self.poll_interval = ready_timeout, poll_interval
|
|
self.runtime_uid, self.runtime_gid = runtime_uid, runtime_gid
|
|
if runtime_uid <= 0 or runtime_gid <= 0:
|
|
raise ValueError('Hermes must run nonroot')
|
|
self.memory_bytes, self.nano_cpus = memory_bytes, nano_cpus
|
|
self._own_docker, self._own_health = docker_client is None, health_client is None
|
|
# No TCP Docker endpoint, DOCKER_HOST, subprocess, or proxy inheritance.
|
|
self.docker = docker_client or httpx.AsyncClient(
|
|
transport=httpx.AsyncHTTPTransport(uds=docker_socket or os.environ.get('DOCKER_SOCKET', '/var/run/docker.sock')),
|
|
base_url='http://docker', timeout=20, trust_env=False)
|
|
self.health = health_client or httpx.AsyncClient(timeout=3, trust_env=False, follow_redirects=False)
|
|
self._locks = {}
|
|
self._ports_lock = asyncio.Lock()
|
|
|
|
def _ports(self, webui=False):
|
|
path = self.state_dir / ('webui-ports.json' if webui else 'api-ports.json')
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Port registry cannot be a symlink')
|
|
try:
|
|
ports = json.loads(path.read_text())
|
|
except FileNotFoundError:
|
|
return {}
|
|
if not isinstance(ports, dict) or any(
|
|
not re.fullmatch(r'[0-9a-f]{32}', uid) or type(port) is not int
|
|
or not 1024 <= port <= 65535 for uid, port in ports.items()):
|
|
raise ProvisioningError('Invalid Hermes port registry')
|
|
if len(set(ports.values())) != len(ports):
|
|
raise ProvisioningError('Duplicate Hermes API ports')
|
|
return ports
|
|
|
|
async def _port(self, user, webui=False):
|
|
# One control process owns this registry, including concurrent accounts.
|
|
async with self._ports_lock:
|
|
ports = self._ports(webui)
|
|
if user['id'] in ports:
|
|
return ports[user['id']]
|
|
used = set(ports.values()) | set(self._ports(not webui).values())
|
|
bounds = tuple(map(int, os.environ.get('HERMES_WEBUI_PORT_RANGE', '30000-30999').split('-'))) if webui else self.api_port_range
|
|
if len(bounds) != 2 or not 1024 <= bounds[0] <= bounds[1] <= 65535:
|
|
raise ProvisioningError('Invalid WebUI port range')
|
|
for port in range(bounds[0], bounds[1] + 1):
|
|
if port in used:
|
|
continue
|
|
# Control must run in the Docker host network namespace.
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
|
try:
|
|
probe.bind((self.webui_host if webui else '127.0.0.1', port))
|
|
except OSError:
|
|
continue
|
|
ports[user['id']] = port
|
|
self._directory(self.state_dir)
|
|
self._write_private(self.state_dir / ('webui-ports.json' if webui else 'api-ports.json'), json.dumps(ports) + '\n')
|
|
return port
|
|
raise ProvisioningError('No available Hermes API port')
|
|
|
|
async def aclose(self):
|
|
if self._own_docker:
|
|
await self.docker.aclose()
|
|
if self._own_health:
|
|
await self.health.aclose()
|
|
|
|
def _lock(self, user):
|
|
container_name(user)
|
|
return self._locks.setdefault(user['id'], asyncio.Lock())
|
|
|
|
async def _request(self, method, path, *, allowed=(200,), **kwargs):
|
|
try:
|
|
response = await self.docker.request(method, path, **kwargs)
|
|
except httpx.HTTPError as exc:
|
|
raise ProvisioningError('Docker control socket unavailable; retry provisioning') from exc
|
|
if response.status_code not in allowed:
|
|
raise ProvisioningError(f'Docker operation failed (HTTP {response.status_code}); retry provisioning')
|
|
return response
|
|
|
|
async def _inspect(self, name):
|
|
response = await self._request('GET', f'/containers/{name}/json', allowed=(200, 404))
|
|
return response.json() if response.status_code == 200 else None
|
|
|
|
@staticmethod
|
|
def _owned(resource, user):
|
|
labels = resource.get('Config', {}).get('Labels') or resource.get('Labels') or {}
|
|
if labels.get('hope.managed') != 'true' or labels.get('hope.user_id') != user['id']:
|
|
raise OwnershipError('Refusing resource whose Hope ownership labels do not match')
|
|
|
|
@staticmethod
|
|
def _directory(path):
|
|
# Do not follow a user-created symlink into another account's state.
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime directory cannot be a symlink')
|
|
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
os.chmod(path, 0o700)
|
|
|
|
@staticmethod
|
|
def _write_private(path, value):
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime file cannot be a symlink')
|
|
fd, temp = tempfile.mkstemp(prefix='.hope-', dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'w') as stream:
|
|
stream.write(value)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temp, path)
|
|
finally:
|
|
if os.path.exists(temp):
|
|
temporary = Path(temp)
|
|
if temporary.resolve().parent != path.parent.resolve() or not temporary.name.startswith('.hope-'):
|
|
raise ProvisioningError('Refusing unexpected temporary-file cleanup path')
|
|
temporary.unlink()
|
|
|
|
def _paths(self, user):
|
|
container_name(user)
|
|
# Every ancestor under the configured root is control-owned, not mounted
|
|
# into Hermes. HOME cannot access the sibling API credential file.
|
|
return self.state_dir / user['id'], self.state_dir / user['id'] / 'home', self.state_dir / user['id'] / 'workspace'
|
|
|
|
def _read_key(self, user):
|
|
private, _, _ = self._paths(user)
|
|
path = private / 'api-key'
|
|
if path.is_symlink():
|
|
raise ProvisioningError('Private runtime file cannot be a symlink')
|
|
try:
|
|
key = path.read_text().strip()
|
|
except FileNotFoundError:
|
|
return None
|
|
if len(key) < 32 or '\n' in key:
|
|
raise ProvisioningError('Invalid persisted Hermes API credential')
|
|
return key
|
|
|
|
@staticmethod
|
|
def _declare_auto_vision(value):
|
|
"""Patch only the verified managed auto model, preserving other metadata."""
|
|
providers = value.get('providers', {})
|
|
if not isinstance(providers, dict):
|
|
raise ValueError('Invalid provider configuration')
|
|
provider = providers.get('hope-upstream')
|
|
if provider is None:
|
|
return False
|
|
if not isinstance(provider, dict):
|
|
raise ValueError('Invalid managed provider')
|
|
models = provider.get('models', {})
|
|
if isinstance(models, list):
|
|
if not all(isinstance(model, str) for model in models):
|
|
raise ValueError('Invalid model list')
|
|
if 'auto' not in models:
|
|
return False
|
|
models = {model: {} for model in models}
|
|
provider['models'] = models
|
|
if not isinstance(models, dict):
|
|
raise ValueError('Invalid model metadata')
|
|
if 'auto' not in models:
|
|
return False
|
|
metadata = models['auto']
|
|
if not isinstance(metadata, dict):
|
|
raise ValueError('Invalid auto metadata')
|
|
if metadata.get('supports_vision') is True:
|
|
return False
|
|
metadata['supports_vision'] = True
|
|
return True
|
|
|
|
def _reload_marker(self, user):
|
|
return self._paths(user)[0] / 'config-reload-pending'
|
|
|
|
def _prepare(self, user, exists, port):
|
|
private, home, workspace = self._paths(user)
|
|
self._directory(self.state_dir)
|
|
for path in (private, home, workspace):
|
|
self._directory(path)
|
|
key = self._read_key(user)
|
|
if not key:
|
|
if exists:
|
|
raise ProvisioningError('Owned container has no persisted API credential; refusing unsafe adoption')
|
|
key = secrets.token_urlsafe(48)
|
|
self._write_private(private / 'api-key', key + '\n')
|
|
config = home / 'config.yaml'
|
|
if not config.exists():
|
|
# JSON is a YAML subset; no credentials are placed in configuration.
|
|
self._write_private(config, json.dumps(self.config(port), indent=2) + '\n')
|
|
elif config.is_symlink():
|
|
raise ProvisioningError('Hermes config cannot be a symlink')
|
|
else:
|
|
# Preserve user configuration except managed listeners/verified auto capability.
|
|
try:
|
|
value = yaml.safe_load(config.read_text())
|
|
if not isinstance(value, dict):
|
|
raise ValueError()
|
|
server = value.setdefault('gateway', {}).setdefault('api_server', {})
|
|
if not isinstance(server, dict):
|
|
raise ValueError()
|
|
changed = self._declare_auto_vision(value)
|
|
if server.get('host') != '127.0.0.1' or server.get('port') != port:
|
|
server.update(host='127.0.0.1', port=port)
|
|
changed = True
|
|
if changed:
|
|
# Durable before writing: retry must reload even after a crash.
|
|
self._write_private(self._reload_marker(user), 'pending\n')
|
|
self._write_private(config, yaml.safe_dump(value, sort_keys=False))
|
|
except (ValueError, TypeError, AttributeError, yaml.YAMLError) as exc:
|
|
raise ProvisioningError('Invalid Hermes runtime configuration') from exc
|
|
if os.geteuid() == 0:
|
|
# Only seeded directory roots/config, never recursive retained data.
|
|
for path in (home, workspace, config):
|
|
os.chown(path, self.runtime_uid, self.runtime_gid)
|
|
elif home.stat().st_uid != self.runtime_uid:
|
|
raise ProvisioningError('Control service must prepare private directories for the Hermes runtime UID')
|
|
return key, home, workspace
|
|
|
|
@staticmethod
|
|
def config(port=20000):
|
|
model = os.environ.get('HERMES_UPSTREAM_MODEL', 'auto')
|
|
return {
|
|
'model': {'default': model, 'provider': 'custom:hope-upstream'},
|
|
'providers': {'hope-upstream': {'api': os.environ.get('HERMES_UPSTREAM_URL', 'http://192.168.86.5:2099/v1'),
|
|
'key_env': 'HERMES_UPSTREAM_API_KEY', 'transport': 'codex_responses', 'default_model': model,
|
|
'models': {'auto': {'supports_vision': True}} if model == 'auto' else [model], 'discover_models': False}},
|
|
'agent': {'reasoning_effort': 'none'},
|
|
'terminal': {'backend': 'local', 'cwd': '/workspace'},
|
|
'platform_toolsets': {'api_server': ['terminal', 'file', 'web', 'browser', 'memory', 'skills', 'delegation']},
|
|
'gateway': {'multiplex_profiles': False,
|
|
'api_server': {'enabled': True, 'host': '127.0.0.1', 'port': port,
|
|
'max_concurrent_runs': 1, 'tool_progress_events': True}},
|
|
}
|
|
|
|
async def _remove_legacy_network(self, user):
|
|
name = 'hope-hermes-net-' + user['id']
|
|
response = await self._request('GET', f'/networks/{name}', allowed=(200, 404))
|
|
if response.status_code == 200:
|
|
network = response.json()
|
|
self._owned(network, user)
|
|
await self._request('DELETE', f'/networks/{network["Id"]}', allowed=(204, 404))
|
|
|
|
def _spec(self, user, key, home, workspace, port):
|
|
extra_env = []
|
|
if self.webui_enabled:
|
|
private, _, _ = self._paths(user)
|
|
password_path = private / 'webui-password'
|
|
if password_path.is_symlink():
|
|
raise ProvisioningError('WebUI password cannot be a symlink')
|
|
if not password_path.exists():
|
|
self._write_private(password_path, secrets.token_urlsafe(32) + '\n')
|
|
password = password_path.read_text().strip()
|
|
if not password or '\n' in password or '\r' in password:
|
|
raise ProvisioningError('WebUI password must be nonempty and single-line')
|
|
extra_env = ['HOPE_WEBUI_ENABLED=true', f'HERMES_WEBUI_HOST={self.webui_host}',
|
|
f'HERMES_WEBUI_PORT={self._ports(True)[user["id"]]}',
|
|
f'HERMES_WEBUI_PASSWORD={password}', f'HERMES_WEBUI_COOKIE_NAME=hope_webui_{user["id"]}',
|
|
'HERMES_WEBUI_SECURE=1', 'HERMES_WEBUI_SKIP_ONBOARDING=1',
|
|
'HERMES_WEBUI_PRESERVE_ENV=1', 'HERMES_WEBUI_AUTO_INSTALL=0',
|
|
'HERMES_WEBUI_SESSION_TTL=86400', 'HERMES_CONFIG_PATH=/opt/data/config.yaml',
|
|
'HERMES_WEBUI_DEFAULT_WORKSPACE=/workspace', 'HERMES_WEBUI_SERVER_CWD=/workspace',
|
|
'HERMES_WEBUI_STATE_DIR=/opt/data/webui',
|
|
'HERMES_WEBUI_TLS_CERT=/opt/data/webui-tls/cert.pem',
|
|
'HERMES_WEBUI_TLS_KEY=/opt/data/webui-tls/key.pem',
|
|
'HERMES_WEBUI_CHAT_BACKEND=gateway', f'HERMES_WEBUI_GATEWAY_BASE_URL=http://127.0.0.1:{port}',
|
|
f'HERMES_WEBUI_GATEWAY_API_KEY={key}', 'HERMES_WEBUI_GATEWAY_USE_RUNS_API=true']
|
|
return {
|
|
'Image': self.image, 'User': f'{self.runtime_uid}:{self.runtime_gid}',
|
|
'Entrypoint': ['/opt/hermes/.venv/bin/python'], 'Cmd': ['-c', _BOOTSTRAP],
|
|
'WorkingDir': '/workspace',
|
|
'Labels': {'hope.managed': 'true', 'hope.user_id': user['id'], **({'hope.webui': '62e4d7b'} if self.webui_enabled else {})},
|
|
'Env': ['HOME=/opt/data/home', 'HERMES_HOME=/opt/data',
|
|
'HERMES_GATEWAY_NO_SUPERVISE=1', 'API_SERVER_ENABLED=true',
|
|
'API_SERVER_HOST=127.0.0.1', f'API_SERVER_PORT={port}', f'API_SERVER_KEY={key}',
|
|
f'HERMES_UPSTREAM_API_KEY={self.upstream_api_key}'] + extra_env,
|
|
'HostConfig': {
|
|
'Binds': [f'{home}:/opt/data:rw', f'{workspace}:/workspace:rw'],
|
|
'NetworkMode': 'host',
|
|
'Init': True, 'Privileged': False, 'CapDrop': ['ALL'],
|
|
'SecurityOpt': ['no-new-privileges:true'], 'PidsLimit': 256,
|
|
'Memory': self.memory_bytes, 'MemorySwap': self.memory_bytes, 'NanoCpus': self.nano_cpus,
|
|
'ShmSize': 256 * 1024 ** 2,
|
|
'RestartPolicy': {'Name': 'unless-stopped'},
|
|
'LogConfig': {'Type': 'json-file', 'Config': {'max-size': '10m', 'max-file': '3'}},
|
|
},
|
|
}
|
|
|
|
def _connection(self, user, info, key):
|
|
self._owned(info, user)
|
|
env = info.get('Config', {}).get('Env') or []
|
|
if f'API_SERVER_KEY={key}' not in env:
|
|
raise ProvisioningError('Owned container API credential does not match private state')
|
|
if info.get('HostConfig', {}).get('NetworkMode') != 'host':
|
|
raise ProvisioningError('Hermes requires host networking')
|
|
port = self._ports().get(user['id'])
|
|
if not port or f'API_SERVER_HOST=127.0.0.1' not in env or f'API_SERVER_PORT={port}' not in env:
|
|
raise ProvisioningError('Hermes API must use its assigned loopback listener')
|
|
return {'container_name': container_name(user), 'hermes_url': f'http://127.0.0.1:{port}', 'hermes_api_key': key}
|
|
|
|
async def _healthy(self, connection):
|
|
try:
|
|
response = await self.health.get(connection['hermes_url'] + '/v1/models',
|
|
headers={'Authorization': 'Bearer ' + connection['hermes_api_key']})
|
|
if response.status_code != 200:
|
|
return False
|
|
if self.webui_enabled:
|
|
api_port = int(connection['hermes_url'].rsplit(':', 1)[1])
|
|
uid = next((uid for uid, port in self._ports().items() if port == api_port), None)
|
|
if not uid:
|
|
return False
|
|
async with httpx.AsyncClient(verify=False, trust_env=False, timeout=3) as client:
|
|
webui = await client.get(f'https://{self.webui_host}:{self._ports(True)[uid]}/health')
|
|
return webui.status_code == 200 and webui.json().get('status') == 'ok'
|
|
return True
|
|
except httpx.HTTPError:
|
|
return False
|
|
|
|
async def ensure(self, user: dict) -> dict:
|
|
async with self._lock(user):
|
|
if not self.upstream_api_key:
|
|
raise ProvisioningError('HERMES_UPSTREAM_API_KEY must be provided at runtime')
|
|
name = container_name(user)
|
|
info = await self._inspect(name)
|
|
if info:
|
|
self._owned(info, user)
|
|
port = await self._port(user)
|
|
if self.webui_enabled:
|
|
await self._port(user, True)
|
|
key, home, workspace = self._prepare(user, info is not None, port)
|
|
# Validate credentials/spec before deleting the previous container.
|
|
spec = self._spec(user, key, home, workspace, port)
|
|
if info and (info.get('HostConfig', {}).get('NetworkMode') != 'host' or
|
|
(self.webui_enabled and info.get('Config', {}).get('Labels', {}).get('hope.webui') != '62e4d7b')):
|
|
# Verify retained credentials before replacing a legacy container.
|
|
if f'API_SERVER_KEY={key}' not in (info.get('Config', {}).get('Env') or []):
|
|
raise ProvisioningError('Owned container API credential does not match private state')
|
|
await self._request('DELETE', f'/containers/{info["Id"]}',
|
|
params={'force': 'true', 'v': 'false'}, allowed=(204, 404))
|
|
info = None
|
|
if info is None:
|
|
await self._request('POST', '/containers/create', params={'name': name},
|
|
json=spec, allowed=(201, 409))
|
|
info = await self._inspect(name)
|
|
if info is None:
|
|
raise ProvisioningError('Docker container creation did not persist; retry')
|
|
self._owned(info, user)
|
|
# Never start a host-networked container with an unexpected listener.
|
|
self._connection(user, info, key)
|
|
await self._remove_legacy_network(user)
|
|
marker = self._reload_marker(user)
|
|
if marker.is_symlink():
|
|
raise ProvisioningError('Reload marker cannot be a symlink')
|
|
if not info.get('State', {}).get('Running'):
|
|
await self._request('POST', f'/containers/{info["Id"]}/start', allowed=(204, 304))
|
|
elif marker.exists():
|
|
await self._request('POST', f'/containers/{info["Id"]}/restart', allowed=(204,))
|
|
# One overall deadline also bounds Docker inspect / health I/O; no
|
|
# deletion on failed health, so a retry starts/adopts the same state.
|
|
try:
|
|
async with asyncio.timeout(self.ready_timeout):
|
|
while True:
|
|
info = await self._inspect(name)
|
|
if info is None:
|
|
raise ProvisioningError('Hermes container disappeared; retry')
|
|
self._owned(info, user)
|
|
if info.get('State', {}).get('Running'):
|
|
connection = self._connection(user, info, key)
|
|
if await self._healthy(connection):
|
|
if marker.exists():
|
|
if marker.is_symlink() or marker.resolve() != self._paths(user)[0].resolve() / 'config-reload-pending':
|
|
raise ProvisioningError('Invalid reload marker path')
|
|
marker.unlink()
|
|
return connection
|
|
await asyncio.sleep(self.poll_interval)
|
|
except TimeoutError as exc:
|
|
raise ProvisioningError('Hermes API readiness timed out; retained container and data for retry') from exc
|
|
|
|
async def ready(self, user: dict) -> bool:
|
|
try:
|
|
info = await self._inspect(container_name(user))
|
|
if not info or not info.get('State', {}).get('Running'):
|
|
return False
|
|
marker = self._reload_marker(user)
|
|
if marker.exists() or marker.is_symlink():
|
|
return False
|
|
config = self._paths(user)[1] / 'config.yaml'
|
|
if config.is_symlink():
|
|
return False
|
|
value = yaml.safe_load(config.read_text())
|
|
if not isinstance(value, dict) or self._declare_auto_vision(value):
|
|
return False
|
|
key = self._read_key(user)
|
|
return bool(key) and await self._healthy(self._connection(user, info, key))
|
|
except (ProvisioningError, ValueError, OSError, yaml.YAMLError):
|
|
return False
|
|
|
|
async def remove(self, user: dict):
|
|
async with self._lock(user):
|
|
info = await self._inspect(container_name(user))
|
|
if info:
|
|
self._owned(info, user)
|
|
# Operate on inspected immutable ID, never a raced name. Labels
|
|
# are immutable for the lifetime of that Docker container ID.
|
|
await self._request('DELETE', f'/containers/{info["Id"]}',
|
|
params={'force': 'true', 'v': 'false'}, allowed=(204, 404))
|
|
await self._remove_legacy_network(user)
|
|
# Retain port assignments alongside retained keys/data for stable recovery.
|
|
# Intentionally retain private credential, home, workspace and memory.
|
|
|
|
|
|
DockerProvisioner = Provisioner
|