test: qualify quiesced deployment snapshots and rollback

tests/unit/test_deploy_rollback.py

@@ -3,13 +3,17 @@ import json
 import os
 from pathlib import Path
 import subprocess
+import stat
+import tarfile
 
 import pytest
 
 FAKE = r'''#!/usr/bin/env python3
 import json,os,sys
 p=os.environ['STATE'];s=json.load(open(p));a=sys.argv[1:]
+s.setdefault('operations',[]).append(a)
 def save():open(p,'w').write(json.dumps(s))
+save()
 def item(n):return next((x for x in s['containers'] if x['name']==n or x['id']==n),None)
 cmd=a[0];n=a[-1];c=item(n)
 if cmd=='inspect':
@@ -35,12 +39,16 @@ elif cmd=='run':
   if entry=='curl' and '%{http_code}' in a:print('401')
   sys.exit(0)
  name=a[a.index('--name')+1]
- if name=='voice' and s['failure']=='run':sys.exit(9)
+ if name=='voice' and s['failure'] in ('run','post-schema'):
+  if s['failure']=='post-schema':
+   from pathlib import Path
+   (Path(p).parent/'accounts'/'post-deploy-message.txt').write_text('New user message must survive rollback')
+  sys.exit(9)
  s['containers'].append({'name':name,'id':'new-'+name,'running':True});save()
 else:sys.exit(2)
 '''
 
-@pytest.mark.parametrize('failure',['stop','rename','run'])
+@pytest.mark.parametrize('failure',['stop','rename','run','post-schema'])
 def test_partial_deployment_restores_originals(tmp_path,failure):
     root=tmp_path/'state';root.mkdir()
     for n in ['runtime.env','livekit.yaml','voice.env']:(root/n).touch()
@@ -58,3 +66,84 @@ def test_partial_deployment_restores_originals(tmp_path,failure):
     assert result.returncode!=0,result.stdout+result.stderr
     data=json.loads(state.read_text())
     assert sorted((x['name'],x['id'],x['running']) for x in data['containers'])==[('hope-control','original-control',True),('voice','original-voice',True)]
+    if failure == 'post-schema':
+        assert (root/'accounts'/'post-deploy-message.txt').read_text() == 'New user message must survive rollback'
+        backup = next((root/'rollback').iterdir())
+        with tarfile.open(backup/'account-state.tgz') as archive:
+            assert 'accounts/post-deploy-message.txt' not in archive.getnames()
+
+
+def test_success_retains_quiesced_account_snapshot_and_secret_permissions(tmp_path):
+    root = tmp_path/'state'
+    root.mkdir()
+    for directory in ('accounts', 'hermes', 'certs'):
+        (root/directory).mkdir()
+    files = {
+        'runtime.env': b'CONTROL_SERVICE_SECRET=local-test-secret\n',
+        'voice.env': b'AZURE_SPEECH_KEY=local-test-secret\n',
+        'livekit.yaml': b'keys: {test: private-test-key}\n',
+        'accounts/accounts.sqlite3': b'offline database snapshot',
+        'accounts/accounts.sqlite3-wal': b'offline sqlite wal snapshot',
+        'accounts/features/uploads/private-file': b'private upload bytes',
+        'hermes/private-home/memory.txt': b'keep original account home',
+        'certs/server.crt': b'keep original certificate',
+    }
+    for name, content in files.items():
+        path = root/name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(content)
+    state = root/'fake.json'
+    state.write_text(json.dumps({'failure': 'none', 'containers': [
+        {'name': 'voice', 'id': 'original-voice', 'running': True},
+        {'name': 'hope-control', 'id': 'original-control', 'running': True}]}))
+    bindir = tmp_path/'bin'
+    bindir.mkdir()
+    docker = bindir/'docker'
+    docker.write_text(FAKE)
+    docker.chmod(0o755)
+    # A real tar creates the archive after checking the exact mocked containers
+    # are stopped and renamed, and before either replacement has started.
+    tar = bindir/'tar'
+    tar.write_text('''#!/usr/bin/env python3
+import json,os,sys
+s=json.load(open(os.environ['STATE']))
+assert sorted((x['id'],x['running']) for x in s['containers']) == [('original-control',False),('original-voice',False)]
+assert all('-rollback-' in x['name'] for x in s['containers'])
+os.execv('/usr/bin/tar',['/usr/bin/tar']+sys.argv[1:])
+''')
+    tar.chmod(0o755)
+    source = (Path(__file__).parents[2]/'scripts/deploy-unraid.sh').read_text()
+    source = source.replace('ROOT=/mnt/user/appdata/hope-voice', f'ROOT={root}').replace('/mnt/user/dev/hope-voice-api/certs', str(root/'certs'))
+    script = tmp_path/'deploy.sh'
+    script.write_text(source)
+    env = dict(os.environ, STATE=str(state), PATH=str(bindir)+os.pathsep+os.environ['PATH'])
+    result = subprocess.run(['bash', str(script)], env=env, capture_output=True, text=True)
+    assert result.returncode == 0, result.stdout+result.stderr
+    data = json.loads(state.read_text())
+    backups = list((root/'rollback').iterdir())
+    assert len(backups) == 1
+    backup = backups[0]
+    assert stat.S_IMODE(backup.stat().st_mode) == 0o700
+    for name in ('account-state.tgz', 'runtime.env', 'voice.env', 'livekit.yaml', 'voice.json', 'hope-control.json'):
+        assert stat.S_IMODE((backup/name).stat().st_mode) == 0o600
+    for name in ('runtime.env', 'voice.env', 'livekit.yaml'):
+        assert (backup/name).read_bytes() == files[name]
+    with tarfile.open(backup/'account-state.tgz') as archive:
+        for name, content in files.items():
+            if name.startswith('accounts/'):
+                assert archive.extractfile(name).read() == content
+        assert not any(name.startswith(('hermes/', 'certs/')) for name in archive.getnames())
+    for name, identifier in (('voice', 'original-voice'), ('hope-control', 'original-control')):
+        assert (backup/(name+'-container-id.txt')).read_text().strip() == identifier
+        original = next(container for container in data['containers'] if container['id'] == identifier)
+        assert original['name'] == name+'-rollback-'+backup.name
+        assert original['running'] is False
+        replacement = next(container for container in data['containers'] if container['name'] == name)
+        assert replacement['id'] == 'new-'+name and replacement['running'] is True
+    assert not any(operation[0] == 'rm' for operation in data['operations'])
+    for name, content in files.items():
+        assert (root/name).read_bytes() == content
+    assert 'local-test-secret' not in result.stdout+result.stderr
+    control_run = next(operation for operation in data['operations'] if operation[0] == 'run' and '--name' in operation and operation[operation.index('--name')+1] == 'hope-control')
+    assert str(root/'accounts')+':/var/lib/hope' in control_run
+    assert str(root/'hermes')+':'+str(root/'hermes') in control_run