"""Offline destructive-step failures restore both original container IDs.""" import json import os from pathlib import Path import subprocess import stat import tarfile import pytest FAKE = r'''#!/usr/bin/env python3 import json,os,sys p=os.environ['STATE'];s=json.load(open(p));a=sys.argv[1:] s.setdefault('operations',[]).append(a) def save():open(p,'w').write(json.dumps(s)) save() def item(n):return next((x for x in s['containers'] if x['name']==n or x['id']==n),None) cmd=a[0];n=a[-1];c=item(n) if cmd=='inspect': if not c:sys.exit(1) if '--format' in a: fmt=a[a.index('--format')+1] print(c['id'] if '.Id' in fmt else '/'+c['name'] if '.Name' in fmt else 'healthy') else:print(json.dumps([c])) elif cmd=='stop': c['running']=False;save() if n=='hope-control' and s['failure']=='stop':sys.exit(7) elif cmd=='rename': c=item(a[1]) if a[1]=='hope-control' and s['failure']=='rename':sys.exit(8) c['name']=a[2];save() elif cmd=='start':c['running']=True;save() elif cmd=='rm': assert c['id'] not in ['original-voice','original-control'] s['containers'].remove(c);save() elif cmd=='run': if '--entrypoint' in a: entry=a[a.index('--entrypoint')+1] if entry=='curl' and '%{http_code}' in a:print('401') sys.exit(0) name=a[a.index('--name')+1] if name=='voice' and s['failure'] in ('run','post-schema'): if s['failure']=='post-schema': from pathlib import Path (Path(p).parent/'accounts'/'post-deploy-message.txt').write_text('New user message must survive rollback') sys.exit(9) s['containers'].append({'name':name,'id':'new-'+name,'running':True});save() else:sys.exit(2) ''' @pytest.mark.parametrize('failure',['stop','rename','run','post-schema']) def test_partial_deployment_restores_originals(tmp_path,failure): root=tmp_path/'state';root.mkdir() for n in ['runtime.env','livekit.yaml','voice.env']:(root/n).touch() for n in ['accounts','hermes','certs']:(root/n).mkdir() state=root/'fake.json' state.write_text(json.dumps({'failure':failure,'containers':[{'name':'voice','id':'original-voice','running':True},{'name':'hope-control','id':'original-control','running':True}]})) bindir=tmp_path/'bin';bindir.mkdir() for name,text in [('docker',FAKE),('curl','#!/bin/sh\nexit 0\n')]: p=bindir/name;p.write_text(text);p.chmod(0o755) source=(Path(__file__).parents[2]/'scripts/deploy-unraid.sh').read_text() source=source.replace('ROOT=/mnt/user/appdata/hope-voice',f'ROOT={root}').replace('/mnt/user/dev/hope-voice-api/certs',str(root/'certs')) script=tmp_path/'deploy.sh';script.write_text(source) env=dict(os.environ,STATE=str(state),PATH=str(bindir)+os.pathsep+os.environ['PATH']) result=subprocess.run(['bash',str(script)],env=env,capture_output=True,text=True) assert result.returncode!=0,result.stdout+result.stderr data=json.loads(state.read_text()) assert sorted((x['name'],x['id'],x['running']) for x in data['containers'])==[('hope-control','original-control',True),('voice','original-voice',True)] if failure == 'post-schema': assert (root/'accounts'/'post-deploy-message.txt').read_text() == 'New user message must survive rollback' backup = next((root/'rollback').iterdir()) with tarfile.open(backup/'account-state.tgz') as archive: assert 'accounts/post-deploy-message.txt' not in archive.getnames() def test_success_retains_quiesced_account_snapshot_and_secret_permissions(tmp_path): root = tmp_path/'state' root.mkdir() for directory in ('accounts', 'hermes', 'certs'): (root/directory).mkdir() files = { 'runtime.env': b'CONTROL_SERVICE_SECRET=local-test-secret\n', 'voice.env': b'AZURE_SPEECH_KEY=local-test-secret\n', 'livekit.yaml': b'keys: {test: private-test-key}\n', 'accounts/accounts.sqlite3': b'offline database snapshot', 'accounts/accounts.sqlite3-wal': b'offline sqlite wal snapshot', 'accounts/features/uploads/private-file': b'private upload bytes', 'hermes/private-home/memory.txt': b'keep original account home', 'certs/server.crt': b'keep original certificate', } for name, content in files.items(): path = root/name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) state = root/'fake.json' state.write_text(json.dumps({'failure': 'none', 'containers': [ {'name': 'voice', 'id': 'original-voice', 'running': True}, {'name': 'hope-control', 'id': 'original-control', 'running': True}]})) bindir = tmp_path/'bin' bindir.mkdir() docker = bindir/'docker' docker.write_text(FAKE) docker.chmod(0o755) # A real tar creates the archive after checking the exact mocked containers # are stopped and renamed, and before either replacement has started. tar = bindir/'tar' tar.write_text('''#!/usr/bin/env python3 import json,os,sys s=json.load(open(os.environ['STATE'])) assert sorted((x['id'],x['running']) for x in s['containers']) == [('original-control',False),('original-voice',False)] assert all('-rollback-' in x['name'] for x in s['containers']) os.execv('/usr/bin/tar',['/usr/bin/tar']+sys.argv[1:]) ''') tar.chmod(0o755) source = (Path(__file__).parents[2]/'scripts/deploy-unraid.sh').read_text() source = source.replace('ROOT=/mnt/user/appdata/hope-voice', f'ROOT={root}').replace('/mnt/user/dev/hope-voice-api/certs', str(root/'certs')) script = tmp_path/'deploy.sh' script.write_text(source) env = dict(os.environ, STATE=str(state), PATH=str(bindir)+os.pathsep+os.environ['PATH']) result = subprocess.run(['bash', str(script)], env=env, capture_output=True, text=True) assert result.returncode == 0, result.stdout+result.stderr data = json.loads(state.read_text()) backups = list((root/'rollback').iterdir()) assert len(backups) == 1 backup = backups[0] assert stat.S_IMODE(backup.stat().st_mode) == 0o700 for name in ('account-state.tgz', 'runtime.env', 'voice.env', 'livekit.yaml', 'voice.json', 'hope-control.json'): assert stat.S_IMODE((backup/name).stat().st_mode) == 0o600 for name in ('runtime.env', 'voice.env', 'livekit.yaml'): assert (backup/name).read_bytes() == files[name] with tarfile.open(backup/'account-state.tgz') as archive: for name, content in files.items(): if name.startswith('accounts/'): assert archive.extractfile(name).read() == content assert not any(name.startswith(('hermes/', 'certs/')) for name in archive.getnames()) for name, identifier in (('voice', 'original-voice'), ('hope-control', 'original-control')): assert (backup/(name+'-container-id.txt')).read_text().strip() == identifier original = next(container for container in data['containers'] if container['id'] == identifier) assert original['name'] == name+'-rollback-'+backup.name assert original['running'] is False replacement = next(container for container in data['containers'] if container['name'] == name) assert replacement['id'] == 'new-'+name and replacement['running'] is True assert not any(operation[0] == 'rm' for operation in data['operations']) for name, content in files.items(): assert (root/name).read_bytes() == content assert 'local-test-secret' not in result.stdout+result.stderr control_run = next(operation for operation in data['operations'] if operation[0] == 'run' and '--name' in operation and operation[operation.index('--name')+1] == 'hope-control') assert str(root/'accounts')+':/var/lib/hope' in control_run assert str(root/'hermes')+':'+str(root/'hermes') in control_run