fix: restore original services on partial deployment failure
This commit is contained in:
1 parent
2aedea84b5
commit
27853f7c27
2 files changed
+94
-1
No files matched your search
+9
-1
@@ -11,6 +11,8 @@ Target: Unraid `192.168.86.2`, public UI `https://192.168.86.2:9178`.
|
|||||||
- User-facing test login username: `john-doe`. Initial passwords are generated,
|
- User-facing test login username: `john-doe`. Initial passwords are generated,
|
||||||
not committed or printed; obtain them from the protected deployment credential
|
not committed or printed; obtain them from the protected deployment credential
|
||||||
record or use administrator password reset in the website.
|
record or use administrator password reset in the website.
|
||||||
|
- Final voice image: `sha256:ea86ac10377c553f32a70d24f46dea1413c949673ed1b9850a4aabbe8574e8d8`;
|
||||||
|
verified `healthy`. Final control HTTP health returned `{"status":"ok"}`.
|
||||||
- No push performed. The existing dirty NAS checkout was not overwritten; clean
|
- No push performed. The existing dirty NAS checkout was not overwritten; clean
|
||||||
build source was staged separately. Hope Music and unrelated services untouched.
|
build source was staged separately. Hope Music and unrelated services untouched.
|
||||||
|
|
||||||
@@ -47,6 +49,10 @@ also passes `model_options.reasoning.enabled: false` and forwards content only.
|
|||||||
- Administrator deletion immediately invalidated session/bootstrap; lifecycle
|
- Administrator deletion immediately invalidated session/bootstrap; lifecycle
|
||||||
removed the account and its named container. Private data retained deliberately.
|
removed the account and its named container. Private data retained deliberately.
|
||||||
- Logout invalidated the token; room grant revocation also covered offline.
|
- Logout invalidated the token; room grant revocation also covered offline.
|
||||||
|
- Actual Azure out-of-band wait-notice and final answer audio received over
|
||||||
|
WebRTC on two independent tracks (measured nonzero peak levels 0.904 / 0.786).
|
||||||
|
Probe used authenticated LiveKit text input with agent voice output and muted
|
||||||
|
synthetic microphone; this verifies synthesis/transport, not physical STT.
|
||||||
- Actual speech worker registered and final container health passed. Health now
|
- Actual speech worker registered and final container health passed. Health now
|
||||||
checks HTTPS, LiveKit, control and PID-bound worker endpoint, not just web shell.
|
checks HTTPS, LiveKit, control and PID-bound worker endpoint, not just web shell.
|
||||||
|
|
||||||
@@ -108,6 +114,8 @@ contains no usable key. Keep API origins synchronized for new DNS names.
|
|||||||
Real physical microphone recognition, perceived speech quality, barge-in and
|
Real physical microphone recognition, perceived speech quality, barge-in and
|
||||||
an installed iPhone against the NAS have **not** been claimed. Self-signed SDK WSS
|
an installed iPhone against the NAS have **not** been claimed. Self-signed SDK WSS
|
||||||
trust may require installing a trusted private CA on the iPhone; native HTTP pinning
|
trust may require installing a trusted private CA on the iPhone; native HTTP pinning
|
||||||
is not a guarantee the SDK socket trusts that certificate. No trust-all/plaintext
|
is not a guarantee the SDK socket trusts that certificate. The deployed public
|
||||||
|
leaf certificate SHA256 was compared with the native HTTP pin and matched exactly
|
||||||
|
(`07907abde3bb9c685265909927627d9ea4705d7e7b27056eefe06e10b192af46`). No trust-all/plaintext
|
||||||
fallback was added. Browser automation's certificate exception is a test setting,
|
fallback was added. Browser automation's certificate exception is a test setting,
|
||||||
not production TLS policy. See iOS project handoff for test artifacts.
|
not production TLS policy. See iOS project handoff for test artifacts.
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
# Run on Unraid after building both images. Never source secret values into logs.
|
||||||
|
ROOT=/mnt/user/appdata/hope-voice
|
||||||
|
ENV_FILE="$ROOT/runtime.env"
|
||||||
|
test -f "$ENV_FILE"
|
||||||
|
test -d "$ROOT/accounts"
|
||||||
|
test -d "$ROOT/hermes"
|
||||||
|
test -f "$ROOT/livekit.yaml"
|
||||||
|
test -d /mnt/user/dev/hope-voice-api/certs
|
||||||
|
STAMP=$(date -u +%Y%m%dT%H%M%SZ)
|
||||||
|
BACKUP="$ROOT/rollback/$STAMP"
|
||||||
|
mkdir -p "$BACKUP"
|
||||||
|
chmod 700 "$ROOT" "$BACKUP"
|
||||||
|
# Capture original immutable IDs before installing state-aware rollback.
|
||||||
|
declare -A ORIGINAL
|
||||||
|
for name in voice hope-control; do
|
||||||
|
ORIGINAL[$name]=$(docker inspect --format '{{.Id}}' "$name" 2>/dev/null || true)
|
||||||
|
done
|
||||||
|
rollback() {
|
||||||
|
local code=$?
|
||||||
|
trap - ERR
|
||||||
|
set +e
|
||||||
|
for name in voice hope-control; do
|
||||||
|
current=$(docker inspect --format '{{.Id}}' "$name" 2>/dev/null)
|
||||||
|
# Never remove an original that failed before its rename. Remove only the
|
||||||
|
# replacement occupying the verified fixed name, then restore the original.
|
||||||
|
if [ -n "$current" ] && [ "$current" != "${ORIGINAL[$name]}" ]; then
|
||||||
|
if [ "$(docker inspect --format '{{.Name}}' "$current")" = "/$name" ]; then
|
||||||
|
docker rm -f "$current" >/dev/null
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ -n "${ORIGINAL[$name]}" ]; then
|
||||||
|
original_name=$(docker inspect --format '{{.Name}}' "${ORIGINAL[$name]}" 2>/dev/null)
|
||||||
|
if [ "$original_name" = "/$name-rollback-$STAMP" ]; then
|
||||||
|
docker rename "${ORIGINAL[$name]}" "$name"
|
||||||
|
fi
|
||||||
|
if [ "$(docker inspect --format '{{.Name}}' "${ORIGINAL[$name]}" 2>/dev/null)" = "/$name" ]; then
|
||||||
|
docker start "${ORIGINAL[$name]}" >/dev/null
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
exit "$code"
|
||||||
|
}
|
||||||
|
trap rollback ERR
|
||||||
|
# Configuration receipts contain runtime secrets: root-only, never output them.
|
||||||
|
for name in voice hope-control; do
|
||||||
|
if [ -n "${ORIGINAL[$name]}" ]; then
|
||||||
|
docker inspect "$name" > "$BACKUP/$name.json"
|
||||||
|
chmod 600 "$BACKUP/$name.json"
|
||||||
|
test "$(docker inspect --format '{{.Name}}' "$name")" = "/$name"
|
||||||
|
docker stop "$name" >/dev/null
|
||||||
|
docker rename "$name" "$name-rollback-$STAMP"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
docker run -d --name hope-control --network host --restart unless-stopped \
|
||||||
|
--env-file "$ENV_FILE" \
|
||||||
|
-e CONTROL_DB_PATH=/var/lib/hope/accounts.sqlite3 \
|
||||||
|
-e CONTROL_HOST=127.0.0.1 -e CONTROL_PORT=8092 \
|
||||||
|
-v "$ROOT/accounts:/var/lib/hope" \
|
||||||
|
-v "$ROOT/hermes:$ROOT/hermes" \
|
||||||
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||||
|
--log-opt max-size=10m --log-opt max-file=3 hope-voice-control:latest >/dev/null
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
if curl -sf http://127.0.0.1:8092/health -o /dev/null; then break; fi
|
||||||
|
if [ "$attempt" = 30 ]; then false; fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
docker run -d --name voice --network host --restart unless-stopped \
|
||||||
|
--env-file "$ROOT/voice.env" \
|
||||||
|
-v /mnt/user/dev/hope-voice-api/certs:/etc/voice/certs \
|
||||||
|
-v "$ROOT/livekit.yaml:/etc/livekit.yaml:ro" \
|
||||||
|
--health-cmd='/opt/voice-agent/.venv/bin/python /opt/voice-agent/healthcheck.py' \
|
||||||
|
--health-interval=15s --health-timeout=5s --health-retries=3 --health-start-period=20s \
|
||||||
|
--log-opt max-size=10m --log-opt max-file=3 hope-voice-api-voice:latest >/dev/null
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
if [ "$(docker inspect --format '{{.State.Health.Status}}' voice)" = healthy ]; then break; fi
|
||||||
|
if [ "$attempt" = 30 ]; then false; fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
test "$(curl -sk -o /dev/null -w '%{http_code}' -X POST https://127.0.0.1:9178/token)" = 401
|
||||||
|
trap - ERR
|
||||||
|
printf 'Deployment healthy; rollback containers retained with suffix %s\n' "$STAMP"
|
||||||
Reference in new issue
Block a user