feat: hardening, error states, interruption handling, a11y (Phase 8)

This commit is contained in:
snowspeeder committed 2026-08-30 09:37:05 -04:00
1 parent 9f1f26ba54
commit 12ded179eb
6 files changed
+233 -38

No files matched your search

+32 -1
View File
@@ -27,7 +27,7 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X
- [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below)
- [x] Phase 6: media search, choices, library panel (commit cbf1830)
- [x] Phase 7: background tasks panel (commit af296da)
- [ ] Phase 8: polish/hardening/parity pass
- [x] Phase 8: polish/hardening/parity pass (commit pending)
- [ ] Phase 9: docs & handoff
## Phase 2 status
@@ -151,6 +151,37 @@ Notes:
- Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead).
- No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing.
## Phase 8 status (2026-08-30 ~13:40)
Done:
- **Stale-room guards**: `sessionGeneration` in VoiceSession, `pollGeneration` in MediaPlayerController. All async callbacks check generation before mutating state. RoomDelegate methods check `self.room === room`. Prevents post-disconnect mutations.
- **Audio interruption**: `handleAudioInterruption` in VoiceSession (mute mic on begin, resume on end if shouldResume). MediaPlayerController pauses/resumes AVPlayer on interruption.
- **Error states**: `isReconnecting` published property + reconnect banner in ConversationView. `stateError` published property + error banner in MediaPanelView.
- **Accessibility**: OrbView accessibilityLabel/Value (maps state to "Listening", "Thinking", etc.). ConversationView: Send, End, Tasks, Media, Settings labels. MediaPanelView: transport controls + refresh library labels.
- **Server URL change fix**: Changed `session` from `@ObservedObject` to `@State` in ConversationView so it can be reassigned inside async closures. Added `.onReceive(session.objectWillChange)` for observation. `.onChange(of: settings.serverURL)` disconnects old session and creates a new one when idle.
Verified in simulator:
- Build: zero errors, zero warnings.
- App launches cleanly.
- Server URL change (invalid → valid) triggers session recreation without crash.
- Connect/Disconnect/Reconnect matrix: Start → Ready → Hang up → Start again (3x). No crashes, state resets cleanly.
- Logs confirm no post-disconnect mutations (stale callbacks guarded).
Web parity checklist (vs /Users/shane/dev/hope-voice-api/web/app.js):
- [x] Token auth + LiveKit connect
- [x] Agent wait + agent_ready
- [x] Voice/model pickers + sync-on-join
- [x] Text mode toggle + barge-in hint
- [x] Transcript list + history
- [x] Media player (poll, reconcile, transport, seek)
- [x] Media search + choices + library
- [x] Background tasks panel
- [x] Stale-room guards (=== room checks)
- [x] Audio interruption handling
- [x] Error states (reconnect, mic error, media error)
- [x] Accessibility labels
- [ ] Web-only: drag-to-seek on waveform (iOS uses slider — acceptable)
- [ ] Web-only: keyboard shortcuts (N/A on iOS)
## Subagent notes
- Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md
- Watch for loops (repeated identical commands) — kill and redirect if seen
@@ -1,32 +1,29 @@
import SwiftUI
import Combine
struct ConversationView: View {
@ObservedObject var settings: SettingsStore
@StateObject private var session: VoiceSession
@State private var sessionID = UUID()
@State private var session: VoiceSession
@State private var composerText = ""
@State private var connectError: String?
@State private var tick = 0
init(settings: SettingsStore) {
self.settings = settings
_session = State(initialValue: Self.makeSession(settings: settings))
}
private static func makeSession(settings: SettingsStore) -> VoiceSession {
let config = ServerConfig.parse(settings.trimmedServerURL)
if case .success(let value) = config {
_session = StateObject(wrappedValue: VoiceSession(
config: value,
tokenClient: TokenClient(config: value),
settings: settings
))
} else {
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
if case .success(let value) = fallback {
_session = StateObject(wrappedValue: VoiceSession(
config: value,
tokenClient: TokenClient(config: value),
settings: settings
))
} else {
fatalError("Unreachable: default server URL is valid")
}
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
}
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
if case .success(let value) = fallback {
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
}
fatalError("Unreachable: default server URL is valid")
}
var body: some View {
@@ -42,12 +39,25 @@ struct ConversationView: View {
}
.navigationTitle("Hope Voice")
.toolbar { toolbarContent }
.id(sessionID)
.onAppear {
if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) {
connectError = error.localizedDescription
}
}
}
.onReceive(session.objectWillChange.receive(on: RunLoop.main)) { _ in
tick &+= 1
}
.onChange(of: settings.serverURL) { _, _ in
guard session.state == .idle else { return }
let oldSession = session
Task { @MainActor [settings] in
await oldSession.disconnect()
self.sessionID = UUID()
self.session = Self.makeSession(settings: settings)
}
}
}
private var statusHeader: some View {
@@ -93,6 +103,12 @@ struct ConversationView: View {
.toggleStyle(.button)
}
}
if session.state == .ready, session.isReconnecting {
Label("Connection lost — reconnecting…", systemImage: "arrow.triangle.2.circlepath")
.font(.footnote)
.foregroundStyle(.orange)
.frame(maxWidth: .infinity, alignment: .leading)
}
if let micError = session.micError {
Label(micError, systemImage: "exclamationmark.triangle")
.font(.footnote)
@@ -163,10 +179,12 @@ struct ConversationView: View {
Image(systemName: "arrow.up.circle.fill")
.font(.title2)
}
.accessibilityLabel("Send message")
.disabled(!settings.textMode || composerText.isEmpty)
Button(role: .destructive, action: endSession) {
Image(systemName: "phone.down.fill")
}
.accessibilityLabel("End conversation")
}
}
.padding()
@@ -190,6 +208,7 @@ struct ConversationView: View {
}
}
}
.accessibilityLabel("Background tasks")
}
ToolbarItem(placement: .topBarTrailing) {
NavigationLink {
@@ -197,6 +216,7 @@ struct ConversationView: View {
} label: {
Image(systemName: "music.note")
}
.accessibilityLabel("Media player")
}
ToolbarItem(placement: .topBarTrailing) {
NavigationLink {
@@ -204,6 +224,7 @@ struct ConversationView: View {
} label: {
Image(systemName: "gearshape")
}
.accessibilityLabel("Settings")
}
}
@@ -30,6 +30,17 @@ struct OrbView: View {
}
}
private var label: String {
switch state {
case .idle: "Idle"
case .connecting: "Connecting"
case .listening: "Listening"
case .speaking: "Speaking"
case .thinking: "Thinking"
case .failed: "Connection failed"
}
}
var body: some View {
let shape = Circle()
.fill(baseColor.gradient)
@@ -44,6 +55,9 @@ struct OrbView: View {
.animation(anim, value: animating)
.onAppear { animating = true }
.onChange(of: state) { _, _ in animating = true }
.accessibilityElement()
.accessibilityLabel("Conversation status")
.accessibilityValue(label)
}
}
@@ -18,6 +18,14 @@ struct MediaPanelView: View {
var body: some View {
ScrollView {
VStack(spacing: 20) {
if let stateError = playerController.stateError {
Label(stateError, systemImage: "wifi.exclamationmark")
.font(.subheadline)
.foregroundStyle(.red)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(12)
.background(Color.red.opacity(0.1), in: RoundedRectangle(cornerRadius: 14))
}
nowPlayingCard
transportControls
seekSection
@@ -103,6 +111,7 @@ struct MediaPanelView: View {
Image(systemName: "gobackward.10")
.font(.title3)
}
.accessibilityLabel("Seek back 10 seconds")
.disabled(playerController.track == nil)
Button(action: togglePlayPause) {
@@ -110,18 +119,21 @@ struct MediaPanelView: View {
.font(.system(size: 44))
.foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor)
}
.accessibilityLabel(playerController.desired == "playing" && playerController.track != nil ? "Pause" : "Play")
.disabled(playerController.track == nil)
Button(action: seekForward) {
Image(systemName: "goforward.10")
.font(.title3)
}
.accessibilityLabel("Seek forward 10 seconds")
.disabled(playerController.track == nil)
Button(role: .destructive, action: stopPlayback) {
Image(systemName: "stop.circle")
.font(.title2)
}
.accessibilityLabel("Stop playback")
.disabled(playerController.track == nil)
}
}
@@ -276,6 +288,7 @@ struct MediaPanelView: View {
Button(action: refreshLibrary) {
Image(systemName: "arrow.clockwise")
}
.accessibilityLabel("Refresh library")
}
if let libraryError {
Text(libraryError)
+103 -16
View File
@@ -1,4 +1,5 @@
import UIKit
import AVFoundation
import Foundation
import LiveKit
import os
@@ -26,9 +27,14 @@ final class VoiceSession: ObservableObject {
@Published private(set) var agent: AgentInfo?
@Published private(set) var isMuted = false
@Published private(set) var micError: String?
@Published private(set) var isReconnecting = false
@Published private(set) var models: [ControlChannel.ModelInfo] = []
@Published private(set) var selectedModel: String?
/// Identity of the current room. Every async callback that could outlive a
/// disconnect/reconnect compares its captured generation against this before
/// mutating state (port of the web UI's `=== room` checks).
private(set) var sessionGeneration = 0
private(set) var room = Room()
let controlChannel = ControlChannel()
let transcriptHandler = TranscriptHandler()
@@ -57,9 +63,18 @@ final class VoiceSession: ObservableObject {
self?.handleControlEvent(event)
}
}
NotificationCenter.default.addObserver(
self, selector: #selector(handleAudioInterruption),
name: AVAudioSession.interruptionNotification, object: nil
)
}
deinit {
NotificationCenter.default.removeObserver(self)
}
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
guard state != .idle else { return }
switch event {
case .models(let received, let selected):
models = received
@@ -82,6 +97,7 @@ final class VoiceSession: ObservableObject {
var isReady: Bool { state == .ready }
func connect() async {
guard state == .idle else { return }
await start()
}
@@ -95,11 +111,15 @@ final class VoiceSession: ObservableObject {
private func start() async {
state = .connecting
isReconnecting = false
sessionGeneration += 1
let generation = sessionGeneration
connectAttempts = 0
await attemptConnect()
await attemptConnect(generation: generation)
}
private func attemptConnect() async {
private func attemptConnect(generation: Int) async {
guard generation == sessionGeneration else { return }
let result: Result<Void, Error>
do {
let token = try await tokenClient.fetchToken()
@@ -110,6 +130,7 @@ final class VoiceSession: ObservableObject {
} catch {
result = .failure(error)
}
guard generation == sessionGeneration else { return }
switch result {
case .success:
logger.log("room connected")
@@ -123,7 +144,8 @@ final class VoiceSession: ObservableObject {
let attempt = connectAttempts
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
await teardownRoom()
await attemptConnect()
guard generation == sessionGeneration else { return }
await attemptConnect(generation: generation)
return
}
logger.error("connect failed: \(String(describing: error), privacy: .public)")
@@ -144,30 +166,32 @@ final class VoiceSession: ObservableObject {
room = makeRoom()
}
private func handleAgentTimeout() async {
guard state == .waitingAgent, !reconnectAttempted else {
if state == .waitingAgent {
state = .failed("Agent never joined. Is the agent worker running?")
}
private func handleAgentTimeout(generation: Int) async {
guard generation == sessionGeneration, state == .waitingAgent else { return }
guard !reconnectAttempted else {
state = .failed("Agent never joined. Is the agent worker running?")
return
}
reconnectAttempted = true
logger.log("agent did not join in time, retrying with fresh token")
await teardownRoom()
guard generation == sessionGeneration else { return }
await start()
}
private func startAgentWait() {
state = .waitingAgent
let generation = sessionGeneration
agentWaitTask?.cancel()
agentWaitTask = Task { [weak self] in
try? await Task.sleep(for: .seconds(10))
guard !Task.isCancelled else { return }
await self?.handleAgentTimeout()
await self?.handleAgentTimeout(generation: generation)
}
}
func agentReadyReceived() {
guard state == .waitingAgent else {
guard state == .waitingAgent, sessionGeneration > 0 else {
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
return
}
@@ -175,18 +199,21 @@ final class VoiceSession: ObservableObject {
state = .ready
stateModel.start()
logger.log("state=ready, enabling mic")
let generation = sessionGeneration
Task { await enableMicrophone() }
Task { await syncSettingsOnJoin() }
Task { await syncSettingsOnJoin(generation: generation) }
}
private func syncSettingsOnJoin() async {
guard isReady else { return }
private func syncSettingsOnJoin(generation: Int) async {
guard generation == sessionGeneration, isReady else { return }
let voiceID = settings.voiceID
do {
try await controlChannel.setVoice(voiceID, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
if let model = selectedModel ?? settings.modelID {
try await controlChannel.setModel(model, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
}
} catch {
@@ -197,8 +224,10 @@ final class VoiceSession: ObservableObject {
func selectVoice(_ id: String) async {
settings.voiceID = id
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setVoice(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_voice sent: \(id, privacy: .public)")
} catch {
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
@@ -208,8 +237,10 @@ final class VoiceSession: ObservableObject {
func selectModel(_ id: String) async {
settings.modelID = id
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setModel(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_model sent: \(id, privacy: .public)")
} catch {
logger.error("set_model failed: \(String(describing: error), privacy: .public)")
@@ -234,9 +265,11 @@ final class VoiceSession: ObservableObject {
}
func enableMicrophone() async {
let generation = sessionGeneration
for attempt in 0..<3 {
do {
try await room.localParticipant.setMicrophone(enabled: true)
guard generation == sessionGeneration else { return }
isMuted = false
micError = nil
logger.log("mic enabled")
@@ -246,10 +279,12 @@ final class VoiceSession: ObservableObject {
if attempt < 2 {
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
try? await Task.sleep(for: .seconds(1))
guard generation == sessionGeneration else { return }
continue
}
// Non-fatal: keep the session usable (text mode still works).
logger.error("mic enable failed after retries: \(message, privacy: .public)")
guard generation == sessionGeneration else { return }
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
}
}
@@ -257,18 +292,23 @@ final class VoiceSession: ObservableObject {
func setMuted(_ muted: Bool) async {
guard isReady else { return }
let generation = sessionGeneration
do {
try await room.localParticipant.setMicrophone(enabled: !muted)
guard generation == sessionGeneration else { return }
isMuted = muted
} catch {
guard generation == sessionGeneration else { return }
micError = "Microphone error: \(error.localizedDescription)"
}
}
func setMode(_ mode: String) async {
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setMode(mode, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_mode sent: \(mode, privacy: .public)")
} catch {
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
@@ -276,17 +316,20 @@ final class VoiceSession: ObservableObject {
}
func sendText(_ text: String) async throws {
guard isReady else { throw URLError(.notConnectedToInternet) }
logger.log("sendText lk.chat chars=\(text.count)")
try await room.localParticipant.sendText(text, for: "lk.chat")
}
func disconnect() async {
agentWaitTask?.cancel()
sessionGeneration += 1
state = .idle
agent = nil
isMuted = false
micError = nil
models = []
isReconnecting = false
reconnectAttempted = false
stateModel.stop()
UIApplication.shared.isIdleTimerDisabled = false
@@ -294,6 +337,34 @@ final class VoiceSession: ObservableObject {
await teardownRoom()
}
// MARK: Audio session interruptions (phone call, Siri)
@objc private func handleAudioInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
let wasActive = state == .ready && !isMuted && !settings.textMode
if wasActive {
Task { await setMuted(true) }
}
logger.log("audio interruption began, muted=\(wasActive)")
case .ended:
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
if state == .ready, isMuted, !settings.textMode, shouldResume {
Task { await setMuted(false) }
logger.log("audio interruption ended, resuming mic")
} else {
let description = String(describing: state)
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))")
}
@unknown default:
break
}
}
func tickAudioLevels(local: Float, remote: Float) {
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
}
@@ -303,14 +374,14 @@ final class VoiceSession: ObservableObject {
extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
Task { @MainActor [weak self] in
guard let self, self.isAgentParticipant(participant) else { return }
guard let self, self.room === room, self.isAgentParticipant(participant) else { return }
self.foundAgent(participant)
}
}
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
Task { @MainActor [weak self] in
guard let self, self.state == .ready else { return }
guard let self, self.room === room, self.state == .ready else { return }
var local = Float(0)
var remote = Float(0)
for participant in participants {
@@ -327,7 +398,7 @@ extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
Task { @MainActor [weak self] in
guard let self else { return }
guard let self, self.room === room else { return }
// Secondary transcript source (finals only).
if topic == "transcript",
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
@@ -341,4 +412,20 @@ extension VoiceSession: RoomDelegate {
_ = self.controlChannel.handleVoiceControl(data, session: self)
}
}
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
Task { @MainActor [weak self] in
guard let self, self.room === room else { return }
switch (oldConnectionState, connectionState) {
case (.connected, .reconnecting):
self.isReconnecting = true
logger.log("connection lost, waiting for LiveKit reconnect")
case (.reconnecting, .connected):
self.isReconnecting = false
logger.log("connection restored")
default:
break
}
}
}
}
+33 -4
View File
@@ -16,6 +16,7 @@ final class MediaPlayerController: ObservableObject {
@Published private(set) var duration: Double = 0
@Published private(set) var pendingJobs: [MediaPendingJob] = []
@Published private(set) var choices: [MediaChoice] = []
@Published private(set) var stateError: String?
let player = AVPlayer()
@@ -34,6 +35,11 @@ final class MediaPlayerController: ObservableObject {
private var timeObserver: Any?
private var statusCancellable: AnyCancellable?
/// Bumped on every room-scope change so in-flight /state polls from the
/// previous scope cannot reconcile into the new one (port of the web UI's
/// `=== room` checks).
private var pollGeneration = 0
init(api: MediaAPI, settings: SettingsStore) {
self.api = api
self.settings = settings
@@ -52,6 +58,25 @@ final class MediaPlayerController: ObservableObject {
NotificationCenter.default.removeObserver(self)
}
@objc private func handleInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
player.pause()
logger.log("audio interruption began, paused media")
case .ended:
applyVolume()
if currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
player.play()
logger.log("audio interruption ended, resumed media")
}
@unknown default:
break
}
}
// MARK: Lifecycle
func attach(session: VoiceSession) {
@@ -63,6 +88,7 @@ final class MediaPlayerController: ObservableObject {
/// Called just before joining the LiveKit room. Carries whatever is loaded
/// into the new room-scoped player state, paused (talking pauses music).
func enterMediaScope(roomName: String) async {
pollGeneration += 1
let wasPlaying = currentTrackID != nil && !player.rate.isZero
wasPlayingBeforeTalk = wasPlaying
let carriedTrack = currentTrackID
@@ -83,6 +109,7 @@ final class MediaPlayerController: ObservableObject {
/// hand the room back to the player.
func endConversation() async {
stopPolling()
pollGeneration += 1
if wasPlayingBeforeTalk {
wasPlayingBeforeTalk = false
guard let room else { return }
@@ -120,12 +147,17 @@ final class MediaPlayerController: ObservableObject {
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
private func pollState() async {
let generation = pollGeneration
let scope = room ?? "default"
do {
let state = try await api.state(room: scope)
guard generation == pollGeneration else { return }
stateError = nil
reconcile(state)
} catch {
guard generation == pollGeneration else { return }
logger.log("poll failed: \(String(describing: error), privacy: .public)")
stateError = "Media server unreachable: \(error.localizedDescription)"
}
}
@@ -246,10 +278,6 @@ final class MediaPlayerController: ObservableObject {
}
}
@objc private func handleInterruption(_ notification: Notification) {
applyVolume()
}
// MARK: Reporting + publishing
private func report(_ state: String) {
@@ -344,6 +372,7 @@ final class MediaPlayerController: ObservableObject {
player.replaceCurrentItem(with: nil)
currentTime = 0
duration = 0
stateError = nil
}
// MARK: Volume