snowspeeder/hope-voice-api
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: hardening, error states, interruption handling, a11y (Phase 8)

Browse Source
This commit is contained in:
snowspeeder committed 2026-08-30 09:37:05 -04:00
1 parent 9f1f26ba54
commit 12ded179eb
6 files changed
+233 -38
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Sources/LiveKit/VoiceSession.swift
+103 -16
View File
@@ -1,4 +1,5 @@
import UIKit
import AVFoundation
import Foundation
import LiveKit
import os
@@ -26,9 +27,14 @@ final class VoiceSession: ObservableObject {
@Published private(set) var agent: AgentInfo?
@Published private(set) var isMuted = false
@Published private(set) var micError: String?
@Published private(set) var isReconnecting = false
@Published private(set) var models: [ControlChannel.ModelInfo] = []
@Published private(set) var selectedModel: String?
/// Identity of the current room. Every async callback that could outlive a
/// disconnect/reconnect compares its captured generation against this before
/// mutating state (port of the web UI's `=== room` checks).
private(set) var sessionGeneration = 0
private(set) var room = Room()
let controlChannel = ControlChannel()
let transcriptHandler = TranscriptHandler()
@@ -57,9 +63,18 @@ final class VoiceSession: ObservableObject {
self?.handleControlEvent(event)
}
}
NotificationCenter.default.addObserver(
self, selector: #selector(handleAudioInterruption),
name: AVAudioSession.interruptionNotification, object: nil
)
}
deinit {
NotificationCenter.default.removeObserver(self)
}
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
guard state != .idle else { return }
switch event {
case .models(let received, let selected):
models = received
@@ -82,6 +97,7 @@ final class VoiceSession: ObservableObject {
var isReady: Bool { state == .ready }
func connect() async {
guard state == .idle else { return }
await start()
}
@@ -95,11 +111,15 @@ final class VoiceSession: ObservableObject {
private func start() async {
state = .connecting
isReconnecting = false
sessionGeneration += 1
let generation = sessionGeneration
connectAttempts = 0
await attemptConnect()
await attemptConnect(generation: generation)
}
private func attemptConnect() async {
private func attemptConnect(generation: Int) async {
guard generation == sessionGeneration else { return }
let result: Result<Void, Error>
do {
let token = try await tokenClient.fetchToken()
@@ -110,6 +130,7 @@ final class VoiceSession: ObservableObject {
} catch {
result = .failure(error)
}
guard generation == sessionGeneration else { return }
switch result {
case .success:
logger.log("room connected")
@@ -123,7 +144,8 @@ final class VoiceSession: ObservableObject {
let attempt = connectAttempts
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
await teardownRoom()
await attemptConnect()
guard generation == sessionGeneration else { return }
await attemptConnect(generation: generation)
return
}
logger.error("connect failed: \(String(describing: error), privacy: .public)")
@@ -144,30 +166,32 @@ final class VoiceSession: ObservableObject {
room = makeRoom()
}
private func handleAgentTimeout() async {
guard state == .waitingAgent, !reconnectAttempted else {
if state == .waitingAgent {
state = .failed("Agent never joined. Is the agent worker running?")
}
private func handleAgentTimeout(generation: Int) async {
guard generation == sessionGeneration, state == .waitingAgent else { return }
guard !reconnectAttempted else {
state = .failed("Agent never joined. Is the agent worker running?")
return
}
reconnectAttempted = true
logger.log("agent did not join in time, retrying with fresh token")
await teardownRoom()
guard generation == sessionGeneration else { return }
await start()
}
private func startAgentWait() {
state = .waitingAgent
let generation = sessionGeneration
agentWaitTask?.cancel()
agentWaitTask = Task { [weak self] in
try? await Task.sleep(for: .seconds(10))
guard !Task.isCancelled else { return }
await self?.handleAgentTimeout()
await self?.handleAgentTimeout(generation: generation)
}
}
func agentReadyReceived() {
guard state == .waitingAgent else {
guard state == .waitingAgent, sessionGeneration > 0 else {
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
return
}
@@ -175,18 +199,21 @@ final class VoiceSession: ObservableObject {
state = .ready
stateModel.start()
logger.log("state=ready, enabling mic")
let generation = sessionGeneration
Task { await enableMicrophone() }
Task { await syncSettingsOnJoin() }
Task { await syncSettingsOnJoin(generation: generation) }
}
private func syncSettingsOnJoin() async {
guard isReady else { return }
private func syncSettingsOnJoin(generation: Int) async {
guard generation == sessionGeneration, isReady else { return }
let voiceID = settings.voiceID
do {
try await controlChannel.setVoice(voiceID, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
if let model = selectedModel ?? settings.modelID {
try await controlChannel.setModel(model, room: room)
guard generation == sessionGeneration else { return }
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
}
} catch {
@@ -197,8 +224,10 @@ final class VoiceSession: ObservableObject {
func selectVoice(_ id: String) async {
settings.voiceID = id
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setVoice(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_voice sent: \(id, privacy: .public)")
} catch {
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
@@ -208,8 +237,10 @@ final class VoiceSession: ObservableObject {
func selectModel(_ id: String) async {
settings.modelID = id
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setModel(id, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_model sent: \(id, privacy: .public)")
} catch {
logger.error("set_model failed: \(String(describing: error), privacy: .public)")
@@ -234,9 +265,11 @@ final class VoiceSession: ObservableObject {
}
func enableMicrophone() async {
let generation = sessionGeneration
for attempt in 0..<3 {
do {
try await room.localParticipant.setMicrophone(enabled: true)
guard generation == sessionGeneration else { return }
isMuted = false
micError = nil
logger.log("mic enabled")
@@ -246,10 +279,12 @@ final class VoiceSession: ObservableObject {
if attempt < 2 {
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
try? await Task.sleep(for: .seconds(1))
guard generation == sessionGeneration else { return }
continue
}
// Non-fatal: keep the session usable (text mode still works).
logger.error("mic enable failed after retries: \(message, privacy: .public)")
guard generation == sessionGeneration else { return }
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
}
}
@@ -257,18 +292,23 @@ final class VoiceSession: ObservableObject {
func setMuted(_ muted: Bool) async {
guard isReady else { return }
let generation = sessionGeneration
do {
try await room.localParticipant.setMicrophone(enabled: !muted)
guard generation == sessionGeneration else { return }
isMuted = muted
} catch {
guard generation == sessionGeneration else { return }
micError = "Microphone error: \(error.localizedDescription)"
}
}
func setMode(_ mode: String) async {
guard isReady else { return }
let generation = sessionGeneration
do {
try await controlChannel.setMode(mode, room: room)
guard generation == sessionGeneration else { return }
logger.log("set_mode sent: \(mode, privacy: .public)")
} catch {
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
@@ -276,17 +316,20 @@ final class VoiceSession: ObservableObject {
}
func sendText(_ text: String) async throws {
guard isReady else { throw URLError(.notConnectedToInternet) }
logger.log("sendText lk.chat chars=\(text.count)")
try await room.localParticipant.sendText(text, for: "lk.chat")
}
func disconnect() async {
agentWaitTask?.cancel()
sessionGeneration += 1
state = .idle
agent = nil
isMuted = false
micError = nil
models = []
isReconnecting = false
reconnectAttempted = false
stateModel.stop()
UIApplication.shared.isIdleTimerDisabled = false
@@ -294,6 +337,34 @@ final class VoiceSession: ObservableObject {
await teardownRoom()
}
// MARK: Audio session interruptions (phone call, Siri)
@objc private func handleAudioInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
let wasActive = state == .ready && !isMuted && !settings.textMode
if wasActive {
Task { await setMuted(true) }
}
logger.log("audio interruption began, muted=\(wasActive)")
case .ended:
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
if state == .ready, isMuted, !settings.textMode, shouldResume {
Task { await setMuted(false) }
logger.log("audio interruption ended, resuming mic")
} else {
let description = String(describing: state)
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))")
}
@unknown default:
break
}
}
func tickAudioLevels(local: Float, remote: Float) {
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
}
@@ -303,14 +374,14 @@ final class VoiceSession: ObservableObject {
extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
Task { @MainActor [weak self] in
guard let self, self.isAgentParticipant(participant) else { return }
guard let self, self.room === room, self.isAgentParticipant(participant) else { return }
self.foundAgent(participant)
}
}
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
Task { @MainActor [weak self] in
guard let self, self.state == .ready else { return }
guard let self, self.room === room, self.state == .ready else { return }
var local = Float(0)
var remote = Float(0)
for participant in participants {
@@ -327,7 +398,7 @@ extension VoiceSession: RoomDelegate {
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
Task { @MainActor [weak self] in
guard let self else { return }
guard let self, self.room === room else { return }
// Secondary transcript source (finals only).
if topic == "transcript",
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
@@ -341,4 +412,20 @@ extension VoiceSession: RoomDelegate {
_ = self.controlChannel.handleVoiceControl(data, session: self)
}
}
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
Task { @MainActor [weak self] in
guard let self, self.room === room else { return }
switch (oldConnectionState, connectionState) {
case (.connected, .reconnecting):
self.isReconnecting = true
logger.log("connection lost, waiting for LiveKit reconnect")
case (.reconnecting, .connected):
self.isReconnecting = false
logger.log("connection restored")
default:
break
}
}
}
}