From 12ded179ebb63481f2fec868b1fabfb23d64fa0d Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sun, 30 Aug 2026 09:37:05 -0400 Subject: [PATCH] feat: hardening, error states, interruption handling, a11y (Phase 8) --- PLAN.md | 33 ++++- .../Conversation/ConversationView.swift | 55 +++++--- Sources/Features/Conversation/OrbView.swift | 14 +++ Sources/Features/Media/MediaPanelView.swift | 13 ++ Sources/LiveKit/VoiceSession.swift | 119 +++++++++++++++--- Sources/Media/MediaPlayerController.swift | 37 +++++- 6 files changed, 233 insertions(+), 38 deletions(-) diff --git a/PLAN.md b/PLAN.md index 085725a..60befbd 100644 --- a/PLAN.md +++ b/PLAN.md @@ -27,7 +27,7 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X - [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below) - [x] Phase 6: media search, choices, library panel (commit cbf1830) - [x] Phase 7: background tasks panel (commit af296da) -- [ ] Phase 8: polish/hardening/parity pass +- [x] Phase 8: polish/hardening/parity pass (commit pending) - [ ] Phase 9: docs & handoff ## Phase 2 status @@ -151,6 +151,37 @@ Notes: - Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead). - No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing. +## Phase 8 status (2026-08-30 ~13:40) +Done: +- **Stale-room guards**: `sessionGeneration` in VoiceSession, `pollGeneration` in MediaPlayerController. All async callbacks check generation before mutating state. RoomDelegate methods check `self.room === room`. Prevents post-disconnect mutations. +- **Audio interruption**: `handleAudioInterruption` in VoiceSession (mute mic on begin, resume on end if shouldResume). MediaPlayerController pauses/resumes AVPlayer on interruption. +- **Error states**: `isReconnecting` published property + reconnect banner in ConversationView. `stateError` published property + error banner in MediaPanelView. +- **Accessibility**: OrbView accessibilityLabel/Value (maps state to "Listening", "Thinking", etc.). ConversationView: Send, End, Tasks, Media, Settings labels. MediaPanelView: transport controls + refresh library labels. +- **Server URL change fix**: Changed `session` from `@ObservedObject` to `@State` in ConversationView so it can be reassigned inside async closures. Added `.onReceive(session.objectWillChange)` for observation. `.onChange(of: settings.serverURL)` disconnects old session and creates a new one when idle. + +Verified in simulator: +- Build: zero errors, zero warnings. +- App launches cleanly. +- Server URL change (invalid → valid) triggers session recreation without crash. +- Connect/Disconnect/Reconnect matrix: Start → Ready → Hang up → Start again (3x). No crashes, state resets cleanly. +- Logs confirm no post-disconnect mutations (stale callbacks guarded). + +Web parity checklist (vs /Users/shane/dev/hope-voice-api/web/app.js): +- [x] Token auth + LiveKit connect +- [x] Agent wait + agent_ready +- [x] Voice/model pickers + sync-on-join +- [x] Text mode toggle + barge-in hint +- [x] Transcript list + history +- [x] Media player (poll, reconcile, transport, seek) +- [x] Media search + choices + library +- [x] Background tasks panel +- [x] Stale-room guards (=== room checks) +- [x] Audio interruption handling +- [x] Error states (reconnect, mic error, media error) +- [x] Accessibility labels +- [ ] Web-only: drag-to-seek on waveform (iOS uses slider — acceptable) +- [ ] Web-only: keyboard shortcuts (N/A on iOS) + ## Subagent notes - Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md - Watch for loops (repeated identical commands) — kill and redirect if seen diff --git a/Sources/Features/Conversation/ConversationView.swift b/Sources/Features/Conversation/ConversationView.swift index ee793d4..016da4b 100644 --- a/Sources/Features/Conversation/ConversationView.swift +++ b/Sources/Features/Conversation/ConversationView.swift @@ -1,32 +1,29 @@ import SwiftUI +import Combine struct ConversationView: View { @ObservedObject var settings: SettingsStore - @StateObject private var session: VoiceSession + @State private var sessionID = UUID() + @State private var session: VoiceSession @State private var composerText = "" @State private var connectError: String? + @State private var tick = 0 init(settings: SettingsStore) { self.settings = settings + _session = State(initialValue: Self.makeSession(settings: settings)) + } + + private static func makeSession(settings: SettingsStore) -> VoiceSession { let config = ServerConfig.parse(settings.trimmedServerURL) if case .success(let value) = config { - _session = StateObject(wrappedValue: VoiceSession( - config: value, - tokenClient: TokenClient(config: value), - settings: settings - )) - } else { - let fallback = ServerConfig.parse("https://192.168.86.2:9178") - if case .success(let value) = fallback { - _session = StateObject(wrappedValue: VoiceSession( - config: value, - tokenClient: TokenClient(config: value), - settings: settings - )) - } else { - fatalError("Unreachable: default server URL is valid") - } + return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings) } + let fallback = ServerConfig.parse("https://192.168.86.2:9178") + if case .success(let value) = fallback { + return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings) + } + fatalError("Unreachable: default server URL is valid") } var body: some View { @@ -42,12 +39,25 @@ struct ConversationView: View { } .navigationTitle("Hope Voice") .toolbar { toolbarContent } + .id(sessionID) .onAppear { if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) { connectError = error.localizedDescription } } } + .onReceive(session.objectWillChange.receive(on: RunLoop.main)) { _ in + tick &+= 1 + } + .onChange(of: settings.serverURL) { _, _ in + guard session.state == .idle else { return } + let oldSession = session + Task { @MainActor [settings] in + await oldSession.disconnect() + self.sessionID = UUID() + self.session = Self.makeSession(settings: settings) + } + } } private var statusHeader: some View { @@ -93,6 +103,12 @@ struct ConversationView: View { .toggleStyle(.button) } } + if session.state == .ready, session.isReconnecting { + Label("Connection lost — reconnecting…", systemImage: "arrow.triangle.2.circlepath") + .font(.footnote) + .foregroundStyle(.orange) + .frame(maxWidth: .infinity, alignment: .leading) + } if let micError = session.micError { Label(micError, systemImage: "exclamationmark.triangle") .font(.footnote) @@ -163,10 +179,12 @@ struct ConversationView: View { Image(systemName: "arrow.up.circle.fill") .font(.title2) } + .accessibilityLabel("Send message") .disabled(!settings.textMode || composerText.isEmpty) Button(role: .destructive, action: endSession) { Image(systemName: "phone.down.fill") } + .accessibilityLabel("End conversation") } } .padding() @@ -190,6 +208,7 @@ struct ConversationView: View { } } } + .accessibilityLabel("Background tasks") } ToolbarItem(placement: .topBarTrailing) { NavigationLink { @@ -197,6 +216,7 @@ struct ConversationView: View { } label: { Image(systemName: "music.note") } + .accessibilityLabel("Media player") } ToolbarItem(placement: .topBarTrailing) { NavigationLink { @@ -204,6 +224,7 @@ struct ConversationView: View { } label: { Image(systemName: "gearshape") } + .accessibilityLabel("Settings") } } diff --git a/Sources/Features/Conversation/OrbView.swift b/Sources/Features/Conversation/OrbView.swift index ce2af8a..6ef3288 100644 --- a/Sources/Features/Conversation/OrbView.swift +++ b/Sources/Features/Conversation/OrbView.swift @@ -30,6 +30,17 @@ struct OrbView: View { } } + private var label: String { + switch state { + case .idle: "Idle" + case .connecting: "Connecting" + case .listening: "Listening" + case .speaking: "Speaking" + case .thinking: "Thinking" + case .failed: "Connection failed" + } + } + var body: some View { let shape = Circle() .fill(baseColor.gradient) @@ -44,6 +55,9 @@ struct OrbView: View { .animation(anim, value: animating) .onAppear { animating = true } .onChange(of: state) { _, _ in animating = true } + .accessibilityElement() + .accessibilityLabel("Conversation status") + .accessibilityValue(label) } } diff --git a/Sources/Features/Media/MediaPanelView.swift b/Sources/Features/Media/MediaPanelView.swift index 9c1216d..e17ead5 100644 --- a/Sources/Features/Media/MediaPanelView.swift +++ b/Sources/Features/Media/MediaPanelView.swift @@ -18,6 +18,14 @@ struct MediaPanelView: View { var body: some View { ScrollView { VStack(spacing: 20) { + if let stateError = playerController.stateError { + Label(stateError, systemImage: "wifi.exclamationmark") + .font(.subheadline) + .foregroundStyle(.red) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(12) + .background(Color.red.opacity(0.1), in: RoundedRectangle(cornerRadius: 14)) + } nowPlayingCard transportControls seekSection @@ -103,6 +111,7 @@ struct MediaPanelView: View { Image(systemName: "gobackward.10") .font(.title3) } + .accessibilityLabel("Seek back 10 seconds") .disabled(playerController.track == nil) Button(action: togglePlayPause) { @@ -110,18 +119,21 @@ struct MediaPanelView: View { .font(.system(size: 44)) .foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor) } + .accessibilityLabel(playerController.desired == "playing" && playerController.track != nil ? "Pause" : "Play") .disabled(playerController.track == nil) Button(action: seekForward) { Image(systemName: "goforward.10") .font(.title3) } + .accessibilityLabel("Seek forward 10 seconds") .disabled(playerController.track == nil) Button(role: .destructive, action: stopPlayback) { Image(systemName: "stop.circle") .font(.title2) } + .accessibilityLabel("Stop playback") .disabled(playerController.track == nil) } } @@ -276,6 +288,7 @@ struct MediaPanelView: View { Button(action: refreshLibrary) { Image(systemName: "arrow.clockwise") } + .accessibilityLabel("Refresh library") } if let libraryError { Text(libraryError) diff --git a/Sources/LiveKit/VoiceSession.swift b/Sources/LiveKit/VoiceSession.swift index bb57f49..04ef083 100644 --- a/Sources/LiveKit/VoiceSession.swift +++ b/Sources/LiveKit/VoiceSession.swift @@ -1,4 +1,5 @@ import UIKit +import AVFoundation import Foundation import LiveKit import os @@ -26,9 +27,14 @@ final class VoiceSession: ObservableObject { @Published private(set) var agent: AgentInfo? @Published private(set) var isMuted = false @Published private(set) var micError: String? + @Published private(set) var isReconnecting = false @Published private(set) var models: [ControlChannel.ModelInfo] = [] @Published private(set) var selectedModel: String? + /// Identity of the current room. Every async callback that could outlive a + /// disconnect/reconnect compares its captured generation against this before + /// mutating state (port of the web UI's `=== room` checks). + private(set) var sessionGeneration = 0 private(set) var room = Room() let controlChannel = ControlChannel() let transcriptHandler = TranscriptHandler() @@ -57,9 +63,18 @@ final class VoiceSession: ObservableObject { self?.handleControlEvent(event) } } + NotificationCenter.default.addObserver( + self, selector: #selector(handleAudioInterruption), + name: AVAudioSession.interruptionNotification, object: nil + ) + } + + deinit { + NotificationCenter.default.removeObserver(self) } private func handleControlEvent(_ event: ControlChannel.InboundEvent) { + guard state != .idle else { return } switch event { case .models(let received, let selected): models = received @@ -82,6 +97,7 @@ final class VoiceSession: ObservableObject { var isReady: Bool { state == .ready } func connect() async { + guard state == .idle else { return } await start() } @@ -95,11 +111,15 @@ final class VoiceSession: ObservableObject { private func start() async { state = .connecting + isReconnecting = false + sessionGeneration += 1 + let generation = sessionGeneration connectAttempts = 0 - await attemptConnect() + await attemptConnect(generation: generation) } - private func attemptConnect() async { + private func attemptConnect(generation: Int) async { + guard generation == sessionGeneration else { return } let result: Result do { let token = try await tokenClient.fetchToken() @@ -110,6 +130,7 @@ final class VoiceSession: ObservableObject { } catch { result = .failure(error) } + guard generation == sessionGeneration else { return } switch result { case .success: logger.log("room connected") @@ -123,7 +144,8 @@ final class VoiceSession: ObservableObject { let attempt = connectAttempts logger.log("connect attempt \(attempt) timed out, retrying with fresh room") await teardownRoom() - await attemptConnect() + guard generation == sessionGeneration else { return } + await attemptConnect(generation: generation) return } logger.error("connect failed: \(String(describing: error), privacy: .public)") @@ -144,30 +166,32 @@ final class VoiceSession: ObservableObject { room = makeRoom() } - private func handleAgentTimeout() async { - guard state == .waitingAgent, !reconnectAttempted else { - if state == .waitingAgent { - state = .failed("Agent never joined. Is the agent worker running?") - } + private func handleAgentTimeout(generation: Int) async { + guard generation == sessionGeneration, state == .waitingAgent else { return } + guard !reconnectAttempted else { + state = .failed("Agent never joined. Is the agent worker running?") return } reconnectAttempted = true + logger.log("agent did not join in time, retrying with fresh token") await teardownRoom() + guard generation == sessionGeneration else { return } await start() } private func startAgentWait() { state = .waitingAgent + let generation = sessionGeneration agentWaitTask?.cancel() agentWaitTask = Task { [weak self] in try? await Task.sleep(for: .seconds(10)) guard !Task.isCancelled else { return } - await self?.handleAgentTimeout() + await self?.handleAgentTimeout(generation: generation) } } func agentReadyReceived() { - guard state == .waitingAgent else { + guard state == .waitingAgent, sessionGeneration > 0 else { logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)") return } @@ -175,18 +199,21 @@ final class VoiceSession: ObservableObject { state = .ready stateModel.start() logger.log("state=ready, enabling mic") + let generation = sessionGeneration Task { await enableMicrophone() } - Task { await syncSettingsOnJoin() } + Task { await syncSettingsOnJoin(generation: generation) } } - private func syncSettingsOnJoin() async { - guard isReady else { return } + private func syncSettingsOnJoin(generation: Int) async { + guard generation == sessionGeneration, isReady else { return } let voiceID = settings.voiceID do { try await controlChannel.setVoice(voiceID, room: room) + guard generation == sessionGeneration else { return } logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)") if let model = selectedModel ?? settings.modelID { try await controlChannel.setModel(model, room: room) + guard generation == sessionGeneration else { return } logger.log("sync-on-join set_model sent: \(model, privacy: .public)") } } catch { @@ -197,8 +224,10 @@ final class VoiceSession: ObservableObject { func selectVoice(_ id: String) async { settings.voiceID = id guard isReady else { return } + let generation = sessionGeneration do { try await controlChannel.setVoice(id, room: room) + guard generation == sessionGeneration else { return } logger.log("set_voice sent: \(id, privacy: .public)") } catch { logger.error("set_voice failed: \(String(describing: error), privacy: .public)") @@ -208,8 +237,10 @@ final class VoiceSession: ObservableObject { func selectModel(_ id: String) async { settings.modelID = id guard isReady else { return } + let generation = sessionGeneration do { try await controlChannel.setModel(id, room: room) + guard generation == sessionGeneration else { return } logger.log("set_model sent: \(id, privacy: .public)") } catch { logger.error("set_model failed: \(String(describing: error), privacy: .public)") @@ -234,9 +265,11 @@ final class VoiceSession: ObservableObject { } func enableMicrophone() async { + let generation = sessionGeneration for attempt in 0..<3 { do { try await room.localParticipant.setMicrophone(enabled: true) + guard generation == sessionGeneration else { return } isMuted = false micError = nil logger.log("mic enabled") @@ -246,10 +279,12 @@ final class VoiceSession: ObservableObject { if attempt < 2 { logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s") try? await Task.sleep(for: .seconds(1)) + guard generation == sessionGeneration else { return } continue } // Non-fatal: keep the session usable (text mode still works). logger.error("mic enable failed after retries: \(message, privacy: .public)") + guard generation == sessionGeneration else { return } micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works." } } @@ -257,18 +292,23 @@ final class VoiceSession: ObservableObject { func setMuted(_ muted: Bool) async { guard isReady else { return } + let generation = sessionGeneration do { try await room.localParticipant.setMicrophone(enabled: !muted) + guard generation == sessionGeneration else { return } isMuted = muted } catch { + guard generation == sessionGeneration else { return } micError = "Microphone error: \(error.localizedDescription)" } } func setMode(_ mode: String) async { guard isReady else { return } + let generation = sessionGeneration do { try await controlChannel.setMode(mode, room: room) + guard generation == sessionGeneration else { return } logger.log("set_mode sent: \(mode, privacy: .public)") } catch { logger.error("set_mode failed: \(String(describing: error), privacy: .public)") @@ -276,17 +316,20 @@ final class VoiceSession: ObservableObject { } func sendText(_ text: String) async throws { + guard isReady else { throw URLError(.notConnectedToInternet) } logger.log("sendText lk.chat chars=\(text.count)") try await room.localParticipant.sendText(text, for: "lk.chat") } func disconnect() async { agentWaitTask?.cancel() + sessionGeneration += 1 state = .idle agent = nil isMuted = false micError = nil models = [] + isReconnecting = false reconnectAttempted = false stateModel.stop() UIApplication.shared.isIdleTimerDisabled = false @@ -294,6 +337,34 @@ final class VoiceSession: ObservableObject { await teardownRoom() } + // MARK: Audio session interruptions (phone call, Siri) + + @objc private func handleAudioInterruption(_ notification: Notification) { + guard let info = notification.userInfo else { return } + guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return } + switch type { + case .began: + let wasActive = state == .ready && !isMuted && !settings.textMode + if wasActive { + Task { await setMuted(true) } + } + logger.log("audio interruption began, muted=\(wasActive)") + case .ended: + let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0 + let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume) + if state == .ready, isMuted, !settings.textMode, shouldResume { + Task { await setMuted(false) } + logger.log("audio interruption ended, resuming mic") + } else { + let description = String(describing: state) + logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))") + } + @unknown default: + break + } + } + func tickAudioLevels(local: Float, remote: Float) { stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote) } @@ -303,14 +374,14 @@ final class VoiceSession: ObservableObject { extension VoiceSession: RoomDelegate { nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) { Task { @MainActor [weak self] in - guard let self, self.isAgentParticipant(participant) else { return } + guard let self, self.room === room, self.isAgentParticipant(participant) else { return } self.foundAgent(participant) } } nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) { Task { @MainActor [weak self] in - guard let self, self.state == .ready else { return } + guard let self, self.room === room, self.state == .ready else { return } var local = Float(0) var remote = Float(0) for participant in participants { @@ -327,7 +398,7 @@ extension VoiceSession: RoomDelegate { nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) { logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "", privacy: .public)") Task { @MainActor [weak self] in - guard let self else { return } + guard let self, self.room === room else { return } // Secondary transcript source (finals only). if topic == "transcript", let json = try? JSONDecoder().decode([String: JSONValue].self, from: data), @@ -341,4 +412,20 @@ extension VoiceSession: RoomDelegate { _ = self.controlChannel.handleVoiceControl(data, session: self) } } + + nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) { + Task { @MainActor [weak self] in + guard let self, self.room === room else { return } + switch (oldConnectionState, connectionState) { + case (.connected, .reconnecting): + self.isReconnecting = true + logger.log("connection lost, waiting for LiveKit reconnect") + case (.reconnecting, .connected): + self.isReconnecting = false + logger.log("connection restored") + default: + break + } + } + } } diff --git a/Sources/Media/MediaPlayerController.swift b/Sources/Media/MediaPlayerController.swift index 4138c07..b38c049 100644 --- a/Sources/Media/MediaPlayerController.swift +++ b/Sources/Media/MediaPlayerController.swift @@ -16,6 +16,7 @@ final class MediaPlayerController: ObservableObject { @Published private(set) var duration: Double = 0 @Published private(set) var pendingJobs: [MediaPendingJob] = [] @Published private(set) var choices: [MediaChoice] = [] + @Published private(set) var stateError: String? let player = AVPlayer() @@ -34,6 +35,11 @@ final class MediaPlayerController: ObservableObject { private var timeObserver: Any? private var statusCancellable: AnyCancellable? + /// Bumped on every room-scope change so in-flight /state polls from the + /// previous scope cannot reconcile into the new one (port of the web UI's + /// `=== room` checks). + private var pollGeneration = 0 + init(api: MediaAPI, settings: SettingsStore) { self.api = api self.settings = settings @@ -52,6 +58,25 @@ final class MediaPlayerController: ObservableObject { NotificationCenter.default.removeObserver(self) } + @objc private func handleInterruption(_ notification: Notification) { + guard let info = notification.userInfo else { return } + guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return } + switch type { + case .began: + player.pause() + logger.log("audio interruption began, paused media") + case .ended: + applyVolume() + if currentTrackID != nil, desired == "playing", loadedAudioPath != nil { + player.play() + logger.log("audio interruption ended, resumed media") + } + @unknown default: + break + } + } + // MARK: Lifecycle func attach(session: VoiceSession) { @@ -63,6 +88,7 @@ final class MediaPlayerController: ObservableObject { /// Called just before joining the LiveKit room. Carries whatever is loaded /// into the new room-scoped player state, paused (talking pauses music). func enterMediaScope(roomName: String) async { + pollGeneration += 1 let wasPlaying = currentTrackID != nil && !player.rate.isZero wasPlayingBeforeTalk = wasPlaying let carriedTrack = currentTrackID @@ -83,6 +109,7 @@ final class MediaPlayerController: ObservableObject { /// hand the room back to the player. func endConversation() async { stopPolling() + pollGeneration += 1 if wasPlayingBeforeTalk { wasPlayingBeforeTalk = false guard let room else { return } @@ -120,12 +147,17 @@ final class MediaPlayerController: ObservableObject { // MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia) private func pollState() async { + let generation = pollGeneration let scope = room ?? "default" do { let state = try await api.state(room: scope) + guard generation == pollGeneration else { return } + stateError = nil reconcile(state) } catch { + guard generation == pollGeneration else { return } logger.log("poll failed: \(String(describing: error), privacy: .public)") + stateError = "Media server unreachable: \(error.localizedDescription)" } } @@ -246,10 +278,6 @@ final class MediaPlayerController: ObservableObject { } } - @objc private func handleInterruption(_ notification: Notification) { - applyVolume() - } - // MARK: Reporting + publishing private func report(_ state: String) { @@ -344,6 +372,7 @@ final class MediaPlayerController: ObservableObject { player.replaceCurrentItem(with: nil) currentTime = 0 duration = 0 + stateError = nil } // MARK: Volume