150 lines
7.5 KiB
Python
150 lines
7.5 KiB
Python
"""Offline destructive-step failures restore both original container IDs."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import stat
|
|
import tarfile
|
|
|
|
import pytest
|
|
|
|
FAKE = r'''#!/usr/bin/env python3
|
|
import json,os,sys
|
|
p=os.environ['STATE'];s=json.load(open(p));a=sys.argv[1:]
|
|
s.setdefault('operations',[]).append(a)
|
|
def save():open(p,'w').write(json.dumps(s))
|
|
save()
|
|
def item(n):return next((x for x in s['containers'] if x['name']==n or x['id']==n),None)
|
|
cmd=a[0];n=a[-1];c=item(n)
|
|
if cmd=='inspect':
|
|
if not c:sys.exit(1)
|
|
if '--format' in a:
|
|
fmt=a[a.index('--format')+1]
|
|
print(c['id'] if '.Id' in fmt else '/'+c['name'] if '.Name' in fmt else 'healthy')
|
|
else:print(json.dumps([c]))
|
|
elif cmd=='stop':
|
|
c['running']=False;save()
|
|
if n=='hope-control' and s['failure']=='stop':sys.exit(7)
|
|
elif cmd=='rename':
|
|
c=item(a[1])
|
|
if a[1]=='hope-control' and s['failure']=='rename':sys.exit(8)
|
|
c['name']=a[2];save()
|
|
elif cmd=='start':c['running']=True;save()
|
|
elif cmd=='rm':
|
|
assert c['id'] not in ['original-voice','original-control']
|
|
s['containers'].remove(c);save()
|
|
elif cmd=='run':
|
|
if '--entrypoint' in a:
|
|
entry=a[a.index('--entrypoint')+1]
|
|
if entry=='curl' and '%{http_code}' in a:print('401')
|
|
sys.exit(0)
|
|
name=a[a.index('--name')+1]
|
|
if name=='voice' and s['failure'] in ('run','post-schema'):
|
|
if s['failure']=='post-schema':
|
|
from pathlib import Path
|
|
(Path(p).parent/'accounts'/'post-deploy-message.txt').write_text('New user message must survive rollback')
|
|
sys.exit(9)
|
|
s['containers'].append({'name':name,'id':'new-'+name,'running':True});save()
|
|
else:sys.exit(2)
|
|
'''
|
|
|
|
@pytest.mark.parametrize('failure',['stop','rename','run','post-schema'])
|
|
def test_partial_deployment_restores_originals(tmp_path,failure):
|
|
root=tmp_path/'state';root.mkdir()
|
|
for n in ['runtime.env','livekit.yaml','voice.env']:(root/n).touch()
|
|
for n in ['accounts','hermes','certs']:(root/n).mkdir()
|
|
state=root/'fake.json'
|
|
state.write_text(json.dumps({'failure':failure,'containers':[{'name':'voice','id':'original-voice','running':True},{'name':'hope-control','id':'original-control','running':True}]}))
|
|
bindir=tmp_path/'bin';bindir.mkdir()
|
|
for name,text in [('docker',FAKE),('curl','#!/bin/sh\nexit 0\n')]:
|
|
p=bindir/name;p.write_text(text);p.chmod(0o755)
|
|
source=(Path(__file__).parents[2]/'scripts/deploy-unraid.sh').read_text()
|
|
source=source.replace('ROOT=/mnt/user/appdata/hope-voice',f'ROOT={root}').replace('/mnt/user/dev/hope-voice-api/certs',str(root/'certs'))
|
|
script=tmp_path/'deploy.sh';script.write_text(source)
|
|
env=dict(os.environ,STATE=str(state),PATH=str(bindir)+os.pathsep+os.environ['PATH'])
|
|
result=subprocess.run(['bash',str(script)],env=env,capture_output=True,text=True)
|
|
assert result.returncode!=0,result.stdout+result.stderr
|
|
data=json.loads(state.read_text())
|
|
assert sorted((x['name'],x['id'],x['running']) for x in data['containers'])==[('hope-control','original-control',True),('voice','original-voice',True)]
|
|
if failure == 'post-schema':
|
|
assert (root/'accounts'/'post-deploy-message.txt').read_text() == 'New user message must survive rollback'
|
|
backup = next((root/'rollback').iterdir())
|
|
with tarfile.open(backup/'account-state.tgz') as archive:
|
|
assert 'accounts/post-deploy-message.txt' not in archive.getnames()
|
|
|
|
|
|
def test_success_retains_quiesced_account_snapshot_and_secret_permissions(tmp_path):
|
|
root = tmp_path/'state'
|
|
root.mkdir()
|
|
for directory in ('accounts', 'hermes', 'certs'):
|
|
(root/directory).mkdir()
|
|
files = {
|
|
'runtime.env': b'CONTROL_SERVICE_SECRET=local-test-secret\n',
|
|
'voice.env': b'AZURE_SPEECH_KEY=local-test-secret\n',
|
|
'livekit.yaml': b'keys: {test: private-test-key}\n',
|
|
'accounts/accounts.sqlite3': b'offline database snapshot',
|
|
'accounts/accounts.sqlite3-wal': b'offline sqlite wal snapshot',
|
|
'accounts/features/uploads/private-file': b'private upload bytes',
|
|
'hermes/private-home/memory.txt': b'keep original account home',
|
|
'certs/server.crt': b'keep original certificate',
|
|
}
|
|
for name, content in files.items():
|
|
path = root/name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(content)
|
|
state = root/'fake.json'
|
|
state.write_text(json.dumps({'failure': 'none', 'containers': [
|
|
{'name': 'voice', 'id': 'original-voice', 'running': True},
|
|
{'name': 'hope-control', 'id': 'original-control', 'running': True}]}))
|
|
bindir = tmp_path/'bin'
|
|
bindir.mkdir()
|
|
docker = bindir/'docker'
|
|
docker.write_text(FAKE)
|
|
docker.chmod(0o755)
|
|
# A real tar creates the archive after checking the exact mocked containers
|
|
# are stopped and renamed, and before either replacement has started.
|
|
tar = bindir/'tar'
|
|
tar.write_text('''#!/usr/bin/env python3
|
|
import json,os,sys
|
|
s=json.load(open(os.environ['STATE']))
|
|
assert sorted((x['id'],x['running']) for x in s['containers']) == [('original-control',False),('original-voice',False)]
|
|
assert all('-rollback-' in x['name'] for x in s['containers'])
|
|
os.execv('/usr/bin/tar',['/usr/bin/tar']+sys.argv[1:])
|
|
''')
|
|
tar.chmod(0o755)
|
|
source = (Path(__file__).parents[2]/'scripts/deploy-unraid.sh').read_text()
|
|
source = source.replace('ROOT=/mnt/user/appdata/hope-voice', f'ROOT={root}').replace('/mnt/user/dev/hope-voice-api/certs', str(root/'certs'))
|
|
script = tmp_path/'deploy.sh'
|
|
script.write_text(source)
|
|
env = dict(os.environ, STATE=str(state), PATH=str(bindir)+os.pathsep+os.environ['PATH'])
|
|
result = subprocess.run(['bash', str(script)], env=env, capture_output=True, text=True)
|
|
assert result.returncode == 0, result.stdout+result.stderr
|
|
data = json.loads(state.read_text())
|
|
backups = list((root/'rollback').iterdir())
|
|
assert len(backups) == 1
|
|
backup = backups[0]
|
|
assert stat.S_IMODE(backup.stat().st_mode) == 0o700
|
|
for name in ('account-state.tgz', 'runtime.env', 'voice.env', 'livekit.yaml', 'voice.json', 'hope-control.json'):
|
|
assert stat.S_IMODE((backup/name).stat().st_mode) == 0o600
|
|
for name in ('runtime.env', 'voice.env', 'livekit.yaml'):
|
|
assert (backup/name).read_bytes() == files[name]
|
|
with tarfile.open(backup/'account-state.tgz') as archive:
|
|
for name, content in files.items():
|
|
if name.startswith('accounts/'):
|
|
assert archive.extractfile(name).read() == content
|
|
assert not any(name.startswith(('hermes/', 'certs/')) for name in archive.getnames())
|
|
for name, identifier in (('voice', 'original-voice'), ('hope-control', 'original-control')):
|
|
assert (backup/(name+'-container-id.txt')).read_text().strip() == identifier
|
|
original = next(container for container in data['containers'] if container['id'] == identifier)
|
|
assert original['name'] == name+'-rollback-'+backup.name
|
|
assert original['running'] is False
|
|
replacement = next(container for container in data['containers'] if container['name'] == name)
|
|
assert replacement['id'] == 'new-'+name and replacement['running'] is True
|
|
assert not any(operation[0] == 'rm' for operation in data['operations'])
|
|
for name, content in files.items():
|
|
assert (root/name).read_bytes() == content
|
|
assert 'local-test-secret' not in result.stdout+result.stderr
|
|
control_run = next(operation for operation in data['operations'] if operation[0] == 'run' and '--name' in operation and operation[operation.index('--name')+1] == 'hope-control')
|
|
assert str(root/'accounts')+':/var/lib/hope' in control_run
|
|
assert str(root/'hermes')+':'+str(root/'hermes') in control_run
|