feat: hardening, error states, interruption handling, a11y (Phase 8)

This commit is contained in:
snowspeeder committed 2026-08-30 09:37:05 -04:00
1 parent 9f1f26ba54
commit 12ded179eb
6 files changed
+233 -38

No files matched your search

+32 -1
View File
@@ -27,7 +27,7 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X
- [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below) - [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below)
- [x] Phase 6: media search, choices, library panel (commit cbf1830) - [x] Phase 6: media search, choices, library panel (commit cbf1830)
- [x] Phase 7: background tasks panel (commit af296da) - [x] Phase 7: background tasks panel (commit af296da)
- [ ] Phase 8: polish/hardening/parity pass - [x] Phase 8: polish/hardening/parity pass (commit pending)
- [ ] Phase 9: docs & handoff - [ ] Phase 9: docs & handoff
## Phase 2 status ## Phase 2 status
@@ -151,6 +151,37 @@ Notes:
- Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead). - Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead).
- No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing. - No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing.
## Phase 8 status (2026-08-30 ~13:40)
Done:
- **Stale-room guards**: `sessionGeneration` in VoiceSession, `pollGeneration` in MediaPlayerController. All async callbacks check generation before mutating state. RoomDelegate methods check `self.room === room`. Prevents post-disconnect mutations.
- **Audio interruption**: `handleAudioInterruption` in VoiceSession (mute mic on begin, resume on end if shouldResume). MediaPlayerController pauses/resumes AVPlayer on interruption.
- **Error states**: `isReconnecting` published property + reconnect banner in ConversationView. `stateError` published property + error banner in MediaPanelView.
- **Accessibility**: OrbView accessibilityLabel/Value (maps state to "Listening", "Thinking", etc.). ConversationView: Send, End, Tasks, Media, Settings labels. MediaPanelView: transport controls + refresh library labels.
- **Server URL change fix**: Changed `session` from `@ObservedObject` to `@State` in ConversationView so it can be reassigned inside async closures. Added `.onReceive(session.objectWillChange)` for observation. `.onChange(of: settings.serverURL)` disconnects old session and creates a new one when idle.
Verified in simulator:
- Build: zero errors, zero warnings.
- App launches cleanly.
- Server URL change (invalid → valid) triggers session recreation without crash.
- Connect/Disconnect/Reconnect matrix: Start → Ready → Hang up → Start again (3x). No crashes, state resets cleanly.
- Logs confirm no post-disconnect mutations (stale callbacks guarded).
Web parity checklist (vs /Users/shane/dev/hope-voice-api/web/app.js):
- [x] Token auth + LiveKit connect
- [x] Agent wait + agent_ready
- [x] Voice/model pickers + sync-on-join
- [x] Text mode toggle + barge-in hint
- [x] Transcript list + history
- [x] Media player (poll, reconcile, transport, seek)
- [x] Media search + choices + library
- [x] Background tasks panel
- [x] Stale-room guards (=== room checks)
- [x] Audio interruption handling
- [x] Error states (reconnect, mic error, media error)
- [x] Accessibility labels
- [ ] Web-only: drag-to-seek on waveform (iOS uses slider — acceptable)
- [ ] Web-only: keyboard shortcuts (N/A on iOS)
## Subagent notes ## Subagent notes
- Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md - Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md
- Watch for loops (repeated identical commands) — kill and redirect if seen - Watch for loops (repeated identical commands) — kill and redirect if seen
@@ -1,32 +1,29 @@
import SwiftUI import SwiftUI
import Combine
struct ConversationView: View { struct ConversationView: View {
@ObservedObject var settings: SettingsStore @ObservedObject var settings: SettingsStore
@StateObject private var session: VoiceSession @State private var sessionID = UUID()
@State private var session: VoiceSession
@State private var composerText = "" @State private var composerText = ""
@State private var connectError: String? @State private var connectError: String?
@State private var tick = 0
init(settings: SettingsStore) { init(settings: SettingsStore) {
self.settings = settings self.settings = settings
_session = State(initialValue: Self.makeSession(settings: settings))
}
private static func makeSession(settings: SettingsStore) -> VoiceSession {
let config = ServerConfig.parse(settings.trimmedServerURL) let config = ServerConfig.parse(settings.trimmedServerURL)
if case .success(let value) = config { if case .success(let value) = config {
_session = StateObject(wrappedValue: VoiceSession( return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
config: value,
tokenClient: TokenClient(config: value),
settings: settings
))
} else {
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
if case .success(let value) = fallback {
_session = StateObject(wrappedValue: VoiceSession(
config: value,
tokenClient: TokenClient(config: value),
settings: settings
))
} else {
fatalError("Unreachable: default server URL is valid")
}
} }
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
if case .success(let value) = fallback {
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
}
fatalError("Unreachable: default server URL is valid")
} }
var body: some View { var body: some View {
@@ -42,12 +39,25 @@ struct ConversationView: View {
} }
.navigationTitle("Hope Voice") .navigationTitle("Hope Voice")
.toolbar { toolbarContent } .toolbar { toolbarContent }
.id(sessionID)
.onAppear { .onAppear {
if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) { if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) {
connectError = error.localizedDescription connectError = error.localizedDescription
} }
} }
} }
.onReceive(session.objectWillChange.receive(on: RunLoop.main)) { _ in
tick &+= 1
}
.onChange(of: settings.serverURL) { _, _ in
guard session.state == .idle else { return }
let oldSession = session
Task { @MainActor [settings] in
await oldSession.disconnect()
self.sessionID = UUID()
self.session = Self.makeSession(settings: settings)
}
}
} }
private var statusHeader: some View { private var statusHeader: some View {
@@ -93,6 +103,12 @@ struct ConversationView: View {
.toggleStyle(.button) .toggleStyle(.button)
} }
} }
if session.state == .ready, session.isReconnecting {
Label("Connection lost — reconnecting…", systemImage: "arrow.triangle.2.circlepath")
.font(.footnote)
.foregroundStyle(.orange)
.frame(maxWidth: .infinity, alignment: .leading)
}
if let micError = session.micError { if let micError = session.micError {
Label(micError, systemImage: "exclamationmark.triangle") Label(micError, systemImage: "exclamationmark.triangle")
.font(.footnote) .font(.footnote)
@@ -163,10 +179,12 @@ struct ConversationView: View {
Image(systemName: "arrow.up.circle.fill") Image(systemName: "arrow.up.circle.fill")
.font(.title2) .font(.title2)
} }
.accessibilityLabel("Send message")
.disabled(!settings.textMode || composerText.isEmpty) .disabled(!settings.textMode || composerText.isEmpty)
Button(role: .destructive, action: endSession) { Button(role: .destructive, action: endSession) {
Image(systemName: "phone.down.fill") Image(systemName: "phone.down.fill")
} }
.accessibilityLabel("End conversation")
} }
} }
.padding() .padding()
@@ -190,6 +208,7 @@ struct ConversationView: View {
} }
} }
} }
.accessibilityLabel("Background tasks")
} }
ToolbarItem(placement: .topBarTrailing) { ToolbarItem(placement: .topBarTrailing) {
NavigationLink { NavigationLink {
@@ -197,6 +216,7 @@ struct ConversationView: View {
} label: { } label: {
Image(systemName: "music.note") Image(systemName: "music.note")
} }
.accessibilityLabel("Media player")
} }
ToolbarItem(placement: .topBarTrailing) { ToolbarItem(placement: .topBarTrailing) {
NavigationLink { NavigationLink {
@@ -204,6 +224,7 @@ struct ConversationView: View {
} label: { } label: {
Image(systemName: "gearshape") Image(systemName: "gearshape")
} }
.accessibilityLabel("Settings")
} }
} }
@@ -30,6 +30,17 @@ struct OrbView: View {
} }
} }
private var label: String {
switch state {
case .idle: "Idle"
case .connecting: "Connecting"
case .listening: "Listening"
case .speaking: "Speaking"
case .thinking: "Thinking"
case .failed: "Connection failed"
}
}
var body: some View { var body: some View {
let shape = Circle() let shape = Circle()
.fill(baseColor.gradient) .fill(baseColor.gradient)
@@ -44,6 +55,9 @@ struct OrbView: View {
.animation(anim, value: animating) .animation(anim, value: animating)
.onAppear { animating = true } .onAppear { animating = true }
.onChange(of: state) { _, _ in animating = true } .onChange(of: state) { _, _ in animating = true }
.accessibilityElement()
.accessibilityLabel("Conversation status")
.accessibilityValue(label)
} }
} }
@@ -18,6 +18,14 @@ struct MediaPanelView: View {
var body: some View { var body: some View {
ScrollView { ScrollView {
VStack(spacing: 20) { VStack(spacing: 20) {
if let stateError = playerController.stateError {
Label(stateError, systemImage: "wifi.exclamationmark")
.font(.subheadline)
.foregroundStyle(.red)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(12)
.background(Color.red.opacity(0.1), in: RoundedRectangle(cornerRadius: 14))
}
nowPlayingCard nowPlayingCard
transportControls transportControls
seekSection seekSection
@@ -103,6 +111,7 @@ struct MediaPanelView: View {
Image(systemName: "gobackward.10") Image(systemName: "gobackward.10")
.font(.title3) .font(.title3)
} }
.accessibilityLabel("Seek back 10 seconds")
.disabled(playerController.track == nil) .disabled(playerController.track == nil)
Button(action: togglePlayPause) { Button(action: togglePlayPause) {
@@ -110,18 +119,21 @@ struct MediaPanelView: View {
.font(.system(size: 44)) .font(.system(size: 44))
.foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor) .foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor)
} }
.accessibilityLabel(playerController.desired == "playing" && playerController.track != nil ? "Pause" : "Play")
.disabled(playerController.track == nil) .disabled(playerController.track == nil)
Button(action: seekForward) { Button(action: seekForward) {
Image(systemName: "goforward.10") Image(systemName: "goforward.10")
.font(.title3) .font(.title3)
} }
.accessibilityLabel("Seek forward 10 seconds")
.disabled(playerController.track == nil) .disabled(playerController.track == nil)
Button(role: .destructive, action: stopPlayback) { Button(role: .destructive, action: stopPlayback) {
Image(systemName: "stop.circle") Image(systemName: "stop.circle")
.font(.title2) .font(.title2)
} }
.accessibilityLabel("Stop playback")
.disabled(playerController.track == nil) .disabled(playerController.track == nil)
} }
} }
@@ -276,6 +288,7 @@ struct MediaPanelView: View {
Button(action: refreshLibrary) { Button(action: refreshLibrary) {
Image(systemName: "arrow.clockwise") Image(systemName: "arrow.clockwise")
} }
.accessibilityLabel("Refresh library")
} }
if let libraryError { if let libraryError {
Text(libraryError) Text(libraryError)
+103 -16
View File
@@ -1,4 +1,5 @@
import UIKit import UIKit
import AVFoundation
import Foundation import Foundation
import LiveKit import LiveKit
import os import os
@@ -26,9 +27,14 @@ final class VoiceSession: ObservableObject {
@Published private(set) var agent: AgentInfo? @Published private(set) var agent: AgentInfo?
@Published private(set) var isMuted = false @Published private(set) var isMuted = false
@Published private(set) var micError: String? @Published private(set) var micError: String?
@Published private(set) var isReconnecting = false
@Published private(set) var models: [ControlChannel.ModelInfo] = [] @Published private(set) var models: [ControlChannel.ModelInfo] = []
@Published private(set) var selectedModel: String? @Published private(set) var selectedModel: String?
/// Identity of the current room. Every async callback that could outlive a
/// disconnect/reconnect compares its captured generation against this before
/// mutating state (port of the web UI's `=== room` checks).
private(set) var sessionGeneration = 0
private(set) var room = Room() private(set) var room = Room()
let controlChannel = ControlChannel() let controlChannel = ControlChannel()
let transcriptHandler = TranscriptHandler() let transcriptHandler = TranscriptHandler()
@@ -57,9 +63,18 @@ final class VoiceSession: ObservableObject {
self?.handleControlEvent(event) self?.handleControlEvent(event)
} }
} }
NotificationCenter.default.addObserver(
self, selector: #selector(handleAudioInterruption),
name: AVAudioSession.interruptionNotification, object: nil
)
}
deinit {
NotificationCenter.default.removeObserver(self)
} }
private func handleControlEvent(_ event: ControlChannel.InboundEvent) { private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
guard state != .idle else { return }
switch event { switch event {
case .models(let received, let selected): case .models(let received, let selected):
models = received models = received
@@ -82,6 +97,7 @@ final class VoiceSession: ObservableObject {
var isReady: Bool { state == .ready } var isReady: Bool { state == .ready }
func connect() async { func connect() async {
guard state == .idle else { return }
await start() await start()
} }
@@ -95,11 +111,15 @@ final class VoiceSession: ObservableObject {
private func start() async { private func start() async {
state = .connecting state = .connecting
isReconnecting = false
sessionGeneration += 1
let generation = sessionGeneration
connectAttempts = 0 connectAttempts = 0
await attemptConnect() await attemptConnect(generation: generation)
} }
private func attemptConnect() async { private func attemptConnect(generation: Int) async {
guard generation == sessionGeneration else { return }
let result: Result<Void, Error> let result: Result<Void, Error>
do { do {
let token = try await tokenClient.fetchToken() let token = try await tokenClient.fetchToken()
@@ -110,6 +130,7 @@ final class VoiceSession: ObservableObject {
} catch { } catch {
result = .failure(error) result = .failure(error)
} }
guard generation == sessionGeneration else { return }
switch result { switch result {
case .success: case .success:
logger.log("room connected") logger.log("room connected")
@@ -123,7 +144,8 @@ final class VoiceSession: ObservableObject {
let attempt = connectAttempts let attempt = connectAttempts
logger.log("connect attempt \(attempt) timed out, retrying with fresh room") logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
await teardownRoom() await teardownRoom()
await attemptConnect() guard generation == sessionGeneration else { return }
await attemptConnect(generation: generation)
return return
} }
logger.error("connect failed: \(String(describing: error), privacy: .public)") logger.error("connect failed: \(String(describing: error), privacy: .public)")
@@ -144,30 +166,32 @@ final class VoiceSession: ObservableObject {
room = makeRoom() room = makeRoom()
} }
private func handleAgentTimeout() async { private func handleAgentTimeout(generation: Int) async {
guard state == .waitingAgent, !reconnectAttempted else { guard generation == sessionGeneration, state == .waitingAgent else { return }
if state == .waitingAgent { guard !reconnectAttempted else {
state = .failed("Agent never joined. Is the agent worker running?") state = .failed("Agent never joined. Is the agent worker running?")
}
return return
} }
reconnectAttempted = true reconnectAttempted = true
logger.log("agent did not join in time, retrying with fresh token")
await teardownRoom() await teardownRoom()
guard generation == sessionGeneration else { return }
await start() await start()
} }
private func startAgentWait() { private func startAgentWait() {
state = .waitingAgent state = .waitingAgent
let generation = sessionGeneration
agentWaitTask?.cancel() agentWaitTask?.cancel()
agentWaitTask = Task { [weak self] in agentWaitTask = Task { [weak self] in
try? await Task.sleep(for: .seconds(10)) try? await Task.sleep(for: .seconds(10))
guard !Task.isCancelled else { return } guard !Task.isCancelled else { return }
await self?.handleAgentTimeout() await self?.handleAgentTimeout(generation: generation)
} }
} }
func agentReadyReceived() { func agentReadyReceived() {
guard state == .waitingAgent else { guard state == .waitingAgent, sessionGeneration > 0 else {
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)") logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
return return
} }
@@ -175,18 +199,21 @@ final class VoiceSession: ObservableObject {
state = .ready state = .ready
stateModel.start() stateModel.start()
logger.log("state=ready, enabling mic") logger.log("state=ready, enabling mic")
let generation = sessionGeneration
Task { await enableMicrophone() } Task { await enableMicrophone() }
Task { await syncSettingsOnJoin() } Task { await syncSettingsOnJoin(generation: generation) }
} }
private func syncSettingsOnJoin() async { private func syncSettingsOnJoin(generation: Int) async {
guard isReady else { return } guard generation == sessionGeneration, isReady else { return }
let voiceID = settings.voiceID let voiceID = settings.voiceID
do { do {
try await controlChannel.setVoice(voiceID, room: room) try await controlChannel.setVoice(voiceID, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)") logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
if let model = selectedModel ?? settings.modelID { if let model = selectedModel ?? settings.modelID {
try await controlChannel.setModel(model, room: room) try await controlChannel.setModel(model, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_model sent: \(model, privacy: .public)") logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
} }
} catch { } catch {
@@ -197,8 +224,10 @@ final class VoiceSession: ObservableObject {
func selectVoice(_ id: String) async { func selectVoice(_ id: String) async {
settings.voiceID = id settings.voiceID = id
guard isReady else { return } guard isReady else { return }
let generation = sessionGeneration
do { do {
try await controlChannel.setVoice(id, room: room) try await controlChannel.setVoice(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_voice sent: \(id, privacy: .public)") logger.log("set_voice sent: \(id, privacy: .public)")
} catch { } catch {
logger.error("set_voice failed: \(String(describing: error), privacy: .public)") logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
@@ -208,8 +237,10 @@ final class VoiceSession: ObservableObject {
func selectModel(_ id: String) async { func selectModel(_ id: String) async {
settings.modelID = id settings.modelID = id
guard isReady else { return } guard isReady else { return }
let generation = sessionGeneration
do { do {
try await controlChannel.setModel(id, room: room) try await controlChannel.setModel(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_model sent: \(id, privacy: .public)") logger.log("set_model sent: \(id, privacy: .public)")
} catch { } catch {
logger.error("set_model failed: \(String(describing: error), privacy: .public)") logger.error("set_model failed: \(String(describing: error), privacy: .public)")
@@ -234,9 +265,11 @@ final class VoiceSession: ObservableObject {
} }
func enableMicrophone() async { func enableMicrophone() async {
let generation = sessionGeneration
for attempt in 0..<3 { for attempt in 0..<3 {
do { do {
try await room.localParticipant.setMicrophone(enabled: true) try await room.localParticipant.setMicrophone(enabled: true)
guard generation == sessionGeneration else { return }
isMuted = false isMuted = false
micError = nil micError = nil
logger.log("mic enabled") logger.log("mic enabled")
@@ -246,10 +279,12 @@ final class VoiceSession: ObservableObject {
if attempt < 2 { if attempt < 2 {
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s") logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
try? await Task.sleep(for: .seconds(1)) try? await Task.sleep(for: .seconds(1))
guard generation == sessionGeneration else { return }
continue continue
} }
// Non-fatal: keep the session usable (text mode still works). // Non-fatal: keep the session usable (text mode still works).
logger.error("mic enable failed after retries: \(message, privacy: .public)") logger.error("mic enable failed after retries: \(message, privacy: .public)")
guard generation == sessionGeneration else { return }
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works." micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
} }
} }
@@ -257,18 +292,23 @@ final class VoiceSession: ObservableObject {
func setMuted(_ muted: Bool) async { func setMuted(_ muted: Bool) async {
guard isReady else { return } guard isReady else { return }
let generation = sessionGeneration
do { do {
try await room.localParticipant.setMicrophone(enabled: !muted) try await room.localParticipant.setMicrophone(enabled: !muted)
guard generation == sessionGeneration else { return }
isMuted = muted isMuted = muted
} catch { } catch {
guard generation == sessionGeneration else { return }
micError = "Microphone error: \(error.localizedDescription)" micError = "Microphone error: \(error.localizedDescription)"
} }
} }
func setMode(_ mode: String) async { func setMode(_ mode: String) async {
guard isReady else { return } guard isReady else { return }
let generation = sessionGeneration
do { do {
try await controlChannel.setMode(mode, room: room) try await controlChannel.setMode(mode, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_mode sent: \(mode, privacy: .public)") logger.log("set_mode sent: \(mode, privacy: .public)")
} catch { } catch {
logger.error("set_mode failed: \(String(describing: error), privacy: .public)") logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
@@ -276,17 +316,20 @@ final class VoiceSession: ObservableObject {
} }
func sendText(_ text: String) async throws { func sendText(_ text: String) async throws {
guard isReady else { throw URLError(.notConnectedToInternet) }
logger.log("sendText lk.chat chars=\(text.count)") logger.log("sendText lk.chat chars=\(text.count)")
try await room.localParticipant.sendText(text, for: "lk.chat") try await room.localParticipant.sendText(text, for: "lk.chat")
} }
func disconnect() async { func disconnect() async {
agentWaitTask?.cancel() agentWaitTask?.cancel()
sessionGeneration += 1
state = .idle state = .idle
agent = nil agent = nil
isMuted = false isMuted = false
micError = nil micError = nil
models = [] models = []
isReconnecting = false
reconnectAttempted = false reconnectAttempted = false
stateModel.stop() stateModel.stop()
UIApplication.shared.isIdleTimerDisabled = false UIApplication.shared.isIdleTimerDisabled = false
@@ -294,6 +337,34 @@ final class VoiceSession: ObservableObject {
await teardownRoom() await teardownRoom()
} }
// MARK: Audio session interruptions (phone call, Siri)
@objc private func handleAudioInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
let wasActive = state == .ready && !isMuted && !settings.textMode
if wasActive {
Task { await setMuted(true) }
}
logger.log("audio interruption began, muted=\(wasActive)")
case .ended:
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
if state == .ready, isMuted, !settings.textMode, shouldResume {
Task { await setMuted(false) }
logger.log("audio interruption ended, resuming mic")
} else {
let description = String(describing: state)
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))")
}
@unknown default:
break
}
}
func tickAudioLevels(local: Float, remote: Float) { func tickAudioLevels(local: Float, remote: Float) {
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote) stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
} }
@@ -303,14 +374,14 @@ final class VoiceSession: ObservableObject {
extension VoiceSession: RoomDelegate { extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) { nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
guard let self, self.isAgentParticipant(participant) else { return } guard let self, self.room === room, self.isAgentParticipant(participant) else { return }
self.foundAgent(participant) self.foundAgent(participant)
} }
} }
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) { nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
guard let self, self.state == .ready else { return } guard let self, self.room === room, self.state == .ready else { return }
var local = Float(0) var local = Float(0)
var remote = Float(0) var remote = Float(0)
for participant in participants { for participant in participants {
@@ -327,7 +398,7 @@ extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) { nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)") logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
guard let self else { return } guard let self, self.room === room else { return }
// Secondary transcript source (finals only). // Secondary transcript source (finals only).
if topic == "transcript", if topic == "transcript",
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data), let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
@@ -341,4 +412,20 @@ extension VoiceSession: RoomDelegate {
_ = self.controlChannel.handleVoiceControl(data, session: self) _ = self.controlChannel.handleVoiceControl(data, session: self)
} }
} }
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
Task { @MainActor [weak self] in
guard let self, self.room === room else { return }
switch (oldConnectionState, connectionState) {
case (.connected, .reconnecting):
self.isReconnecting = true
logger.log("connection lost, waiting for LiveKit reconnect")
case (.reconnecting, .connected):
self.isReconnecting = false
logger.log("connection restored")
default:
break
}
}
}
} }
+33 -4
View File
@@ -16,6 +16,7 @@ final class MediaPlayerController: ObservableObject {
@Published private(set) var duration: Double = 0 @Published private(set) var duration: Double = 0
@Published private(set) var pendingJobs: [MediaPendingJob] = [] @Published private(set) var pendingJobs: [MediaPendingJob] = []
@Published private(set) var choices: [MediaChoice] = [] @Published private(set) var choices: [MediaChoice] = []
@Published private(set) var stateError: String?
let player = AVPlayer() let player = AVPlayer()
@@ -34,6 +35,11 @@ final class MediaPlayerController: ObservableObject {
private var timeObserver: Any? private var timeObserver: Any?
private var statusCancellable: AnyCancellable? private var statusCancellable: AnyCancellable?
/// Bumped on every room-scope change so in-flight /state polls from the
/// previous scope cannot reconcile into the new one (port of the web UI's
/// `=== room` checks).
private var pollGeneration = 0
init(api: MediaAPI, settings: SettingsStore) { init(api: MediaAPI, settings: SettingsStore) {
self.api = api self.api = api
self.settings = settings self.settings = settings
@@ -52,6 +58,25 @@ final class MediaPlayerController: ObservableObject {
NotificationCenter.default.removeObserver(self) NotificationCenter.default.removeObserver(self)
} }
@objc private func handleInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
player.pause()
logger.log("audio interruption began, paused media")
case .ended:
applyVolume()
if currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
player.play()
logger.log("audio interruption ended, resumed media")
}
@unknown default:
break
}
}
// MARK: Lifecycle // MARK: Lifecycle
func attach(session: VoiceSession) { func attach(session: VoiceSession) {
@@ -63,6 +88,7 @@ final class MediaPlayerController: ObservableObject {
/// Called just before joining the LiveKit room. Carries whatever is loaded /// Called just before joining the LiveKit room. Carries whatever is loaded
/// into the new room-scoped player state, paused (talking pauses music). /// into the new room-scoped player state, paused (talking pauses music).
func enterMediaScope(roomName: String) async { func enterMediaScope(roomName: String) async {
pollGeneration += 1
let wasPlaying = currentTrackID != nil && !player.rate.isZero let wasPlaying = currentTrackID != nil && !player.rate.isZero
wasPlayingBeforeTalk = wasPlaying wasPlayingBeforeTalk = wasPlaying
let carriedTrack = currentTrackID let carriedTrack = currentTrackID
@@ -83,6 +109,7 @@ final class MediaPlayerController: ObservableObject {
/// hand the room back to the player. /// hand the room back to the player.
func endConversation() async { func endConversation() async {
stopPolling() stopPolling()
pollGeneration += 1
if wasPlayingBeforeTalk { if wasPlayingBeforeTalk {
wasPlayingBeforeTalk = false wasPlayingBeforeTalk = false
guard let room else { return } guard let room else { return }
@@ -120,12 +147,17 @@ final class MediaPlayerController: ObservableObject {
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia) // MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
private func pollState() async { private func pollState() async {
let generation = pollGeneration
let scope = room ?? "default" let scope = room ?? "default"
do { do {
let state = try await api.state(room: scope) let state = try await api.state(room: scope)
guard generation == pollGeneration else { return }
stateError = nil
reconcile(state) reconcile(state)
} catch { } catch {
guard generation == pollGeneration else { return }
logger.log("poll failed: \(String(describing: error), privacy: .public)") logger.log("poll failed: \(String(describing: error), privacy: .public)")
stateError = "Media server unreachable: \(error.localizedDescription)"
} }
} }
@@ -246,10 +278,6 @@ final class MediaPlayerController: ObservableObject {
} }
} }
@objc private func handleInterruption(_ notification: Notification) {
applyVolume()
}
// MARK: Reporting + publishing // MARK: Reporting + publishing
private func report(_ state: String) { private func report(_ state: String) {
@@ -344,6 +372,7 @@ final class MediaPlayerController: ObservableObject {
player.replaceCurrentItem(with: nil) player.replaceCurrentItem(with: nil)
currentTime = 0 currentTime = 0
duration = 0 duration = 0
stateError = nil
} }
// MARK: Volume // MARK: Volume