Archived
823 lines
38 KiB
Swift
823 lines
38 KiB
Swift
import UIKit
|
|
import AVFoundation
|
|
import Foundation
|
|
import LiveKit
|
|
import os
|
|
|
|
private let logger = Logger(subsystem: "dev.shane.voice", category: "session")
|
|
|
|
enum VoiceSessionState: Equatable {
|
|
case idle
|
|
case connecting
|
|
case waitingAgent
|
|
case ready
|
|
/// Failure teardown in progress. Start is unavailable; the session is not
|
|
/// yet safe for a fresh connect. Transitions to `.failed` when cleanup completes.
|
|
case tearingDown
|
|
case failed(String)
|
|
|
|
var isFailed: Bool {
|
|
if case .failed = self { return true }
|
|
return false
|
|
}
|
|
}
|
|
|
|
struct AgentInfo: Equatable {
|
|
let identity: String
|
|
let name: String?
|
|
}
|
|
|
|
/// Room lifecycle: connect, agent discovery, agent_ready wait, reconnect-once,
|
|
/// disconnect teardown. Ports the web UI's connect flow.
|
|
@MainActor
|
|
final class VoiceSession: ObservableObject {
|
|
@Published private(set) var state: VoiceSessionState = .idle
|
|
@Published private(set) var agent: AgentInfo?
|
|
@Published private(set) var isMuted = false
|
|
@Published private(set) var micError: String?
|
|
@Published private(set) var isReconnecting = false
|
|
@Published private(set) var assistantActivity: String?
|
|
private var workingRequests: [String: String] = [:]
|
|
|
|
/// Identity of the current room. Every async callback that could outlive a
|
|
/// disconnect/reconnect compares its captured generation against this before
|
|
/// mutating state (port of the web UI's `=== room` checks).
|
|
private(set) var sessionGeneration = 0
|
|
private(set) var room = Room()
|
|
let controlChannel = ControlChannel()
|
|
let transcriptHandler: TranscriptHandler
|
|
let stateModel = ConversationStateModel()
|
|
let taskStore = TaskStore()
|
|
private let config: ServerConfig
|
|
private let tokenClient: TokenClient
|
|
private let settings: SettingsStore
|
|
private var reconnectAttempted = false
|
|
private var connectAttempts = 0
|
|
private var agentWaitTask: Task<Void, Never>?
|
|
/// Whether the user (or session mode) intentionally wants the microphone
|
|
/// muted, independent of interruption-owned hardware state. Set by public
|
|
/// mute actions (`setMuted(true)`, text-mode enable) after the operation
|
|
/// succeeds; cleared by a successful normal mic enable (`enableMicrophone`).
|
|
/// Interruption begin operations use this *after* awaiting the predecessor
|
|
/// to decide whether to mute, rather than snapshotting transient `isMuted`
|
|
/// before the predecessor completes (Iteration 08 Item 1).
|
|
private(set) var userWantsMicMuted = false
|
|
/// Whether the current audio interruption actually muted the mic. Set only
|
|
/// *after* the mute operation completes successfully; cleared on `.ended`
|
|
/// and `disconnect()`.
|
|
private var micMutedByInterruption = false
|
|
/// Sequence number for interruption-owned operations. Incremented on every
|
|
/// `.began`; a `.ended` handler captures the sequence at `.began` time and
|
|
/// only resumes if its sequence is still current (no newer `.began` has
|
|
/// superseded it). This prevents out-of-order mute/unmute when interruptions
|
|
/// overlap or arrive in rapid succession.
|
|
private var interruptionSequence = 0
|
|
/// The retained operation chain for microphone interruption operations.
|
|
/// Both `.began` (mute) and `.ended` (unmute) are assigned to this chain so
|
|
/// every new operation awaits its predecessor. This guarantees:
|
|
/// - A newer `.began` orders after an in-flight old end/unmute and finishes
|
|
/// by muting again (the old unmute cannot leave hardware enabled).
|
|
/// - An `.ended` never launches unretained; it is part of the chain.
|
|
/// Cancelled on disconnect/failure.
|
|
private var micOpTask: Task<Void, Never>?
|
|
/// Set before awaiting `room.disconnect()` during intentional teardown so
|
|
/// the resulting `.disconnected` callback is recognized as expected (not a
|
|
/// terminal failure) and ignored. Cleared when the new room connects.
|
|
private var isTearingDown = false
|
|
|
|
init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore, userID: String? = nil) {
|
|
transcriptHandler = TranscriptHandler(history: TranscriptHistory(origin: config.baseURL.absoluteString, userID: userID))
|
|
self.config = config
|
|
self.tokenClient = tokenClient
|
|
self.settings = settings
|
|
transcriptHandler.acceptsRoom = { [weak self] source in
|
|
guard let self else { return false }
|
|
return self.room === source && !self.isTearingDown && self.state == .ready
|
|
}
|
|
room.add(delegate: self)
|
|
room.add(delegate: controlChannel)
|
|
room.add(delegate: transcriptHandler)
|
|
controlChannel.onEvent = { [weak self] event in
|
|
Task { @MainActor [weak self] in
|
|
self?.handleControlEvent(event)
|
|
}
|
|
}
|
|
NotificationCenter.default.addObserver(
|
|
self, selector: #selector(handleAudioInterruption),
|
|
name: AVAudioSession.interruptionNotification, object: nil
|
|
)
|
|
}
|
|
|
|
deinit {
|
|
NotificationCenter.default.removeObserver(self)
|
|
}
|
|
|
|
/// Explicit chat-to-Talk navigation tears down the prior room first.
|
|
func bindConversation(_ id: String?) async {
|
|
await disconnect()
|
|
tokenClient.conversationID = id
|
|
}
|
|
|
|
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
|
|
// Restrict inbound control-event application to active lifecycle states
|
|
// so late events from a failed/tearing-down room cannot repopulate stale
|
|
// fields (agent, models, mic state) into a fresh retry (Iteration 08 Item 3).
|
|
guard !isTearingDown, state == .connecting || state == .waitingAgent || state == .ready else { return }
|
|
switch event {
|
|
case .taskEvent(let payload):
|
|
taskStore.apply(payload)
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
|
|
var onUnauthorized: (() async -> Void)? {
|
|
get { tokenClient.onUnauthorized }
|
|
set { tokenClient.onUnauthorized = newValue }
|
|
}
|
|
|
|
var bootstrapBearerToken: String { tokenClient.bearerToken }
|
|
|
|
var isReady: Bool { state == .ready }
|
|
|
|
/// Clear all per-room/session ephemeral published fields so the failed or
|
|
/// idle UI and any subsequent retry do not display stale identity, error,
|
|
/// mute, model, or reconnect data from a torn-down room.
|
|
///
|
|
/// Deliberately preserved across reset: `settings` (durable user
|
|
/// preferences), `transcriptHandler` events (conversation history), and
|
|
/// `taskStore` (task history). These are not per-room ephemeral state.
|
|
///
|
|
/// Reused from `teardownForFailure` (both the initial pass and the final
|
|
/// pass immediately before `.failed` publication) and `disconnect()`.
|
|
private func resetEphemeralFields() {
|
|
agent = nil
|
|
isMuted = false
|
|
userWantsMicMuted = false
|
|
micError = nil
|
|
workingRequests.removeAll()
|
|
assistantActivity = nil
|
|
isReconnecting = false
|
|
reconnectAttempted = false
|
|
micMutedByInterruption = false
|
|
}
|
|
|
|
/// Pure predicate: whether a direct `RoomDelegate` callback that mutates
|
|
/// published ephemeral fields (agent discovery, connection-state changes)
|
|
/// should be accepted given the current lifecycle state and teardown flag.
|
|
///
|
|
/// Callbacks are accepted only for active lifecycle states (`.connecting`,
|
|
/// `.waitingAgent`, `.ready`) AND only when `isTearingDown` is false. This
|
|
/// prevents old-room callbacks delivered during failure teardown from
|
|
/// repopulating ephemeral fields that were just cleared (Iteration 09 Item 1).
|
|
func shouldAcceptRoomCallback(state: VoiceSessionState, isTearingDown: Bool) -> Bool {
|
|
!isTearingDown && (state == .connecting || state == .waitingAgent || state == .ready)
|
|
}
|
|
|
|
/// Start (or restart) a session. Accepts `.idle` and `.failed` so the
|
|
/// Start button can retry after a failure without first resetting to idle.
|
|
/// Rejects `.tearingDown`: failure cleanup must complete before a fresh
|
|
/// connect is allowed, preventing the old failure task from tearing down
|
|
/// the new room.
|
|
func connect() async {
|
|
guard state == .idle || state.isFailed else { return }
|
|
reconnectAttempted = false
|
|
await start()
|
|
}
|
|
|
|
/// Whether a failure teardown is currently in progress. Exposed for UI and
|
|
/// test purposes; `connect()` rejects while this is true.
|
|
var isFailureTearingDown: Bool { state == .tearingDown }
|
|
|
|
private func makeRoom() -> Room {
|
|
let room = Room()
|
|
room.add(delegate: self)
|
|
room.add(delegate: controlChannel)
|
|
room.add(delegate: transcriptHandler)
|
|
return room
|
|
}
|
|
|
|
private func start() async {
|
|
state = .connecting
|
|
isReconnecting = false
|
|
sessionGeneration += 1
|
|
let generation = sessionGeneration
|
|
connectAttempts = 0
|
|
await attemptConnect(generation: generation)
|
|
}
|
|
|
|
private func attemptConnect(generation: Int) async {
|
|
guard generation == sessionGeneration else { return }
|
|
let result: Result<Void, Error>
|
|
do {
|
|
let token = try await tokenClient.fetchToken()
|
|
guard generation == sessionGeneration, !Task.isCancelled, !isTearingDown else { return }
|
|
try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions())
|
|
result = .success(())
|
|
} catch {
|
|
result = .failure(error)
|
|
}
|
|
guard generation == sessionGeneration else { return }
|
|
switch result {
|
|
case .success:
|
|
logger.log("room connected")
|
|
UIApplication.shared.isIdleTimerDisabled = true
|
|
scanForAgent()
|
|
startAgentWait()
|
|
case .failure(let error):
|
|
let isTimeout = (error as? LiveKitError)?.type == .timedOut
|
|
if isTimeout, connectAttempts < 2 {
|
|
connectAttempts += 1
|
|
let attempt = connectAttempts
|
|
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
|
await teardownRoom()
|
|
guard generation == sessionGeneration else { return }
|
|
await attemptConnect(generation: generation)
|
|
return
|
|
}
|
|
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
|
await teardownForFailure(message: error.localizedDescription)
|
|
}
|
|
}
|
|
|
|
private func makeConnectOptions() -> ConnectOptions {
|
|
return ConnectOptions(
|
|
socketConnectTimeoutInterval: 15,
|
|
primaryTransportConnectTimeout: 20,
|
|
publisherTransportConnectTimeout: 20
|
|
)
|
|
}
|
|
|
|
/// Intentional, idempotent room teardown. Sets the teardown flag *before*
|
|
/// awaiting `room.disconnect()` so the resulting `.disconnected` callback is
|
|
/// recognized as expected and ignored (not treated as a terminal failure).
|
|
/// Safe to call when the room is already disconnected: LiveKit's
|
|
/// `disconnect()` is a no-op in that case, and the flag is cleared either way.
|
|
private func teardownRoom() async {
|
|
isTearingDown = true
|
|
await room.disconnect()
|
|
room = makeRoom()
|
|
isTearingDown = false
|
|
}
|
|
|
|
private func handleAgentTimeout(generation: Int) async {
|
|
guard generation == sessionGeneration, state == .waitingAgent else { return }
|
|
guard !reconnectAttempted else {
|
|
// Final failure: tear down the live room and restore the idle timer.
|
|
logger.log("agent timeout final, tearing down")
|
|
await teardownForFailure(message: "Agent never joined. Is the agent worker running?")
|
|
return
|
|
}
|
|
reconnectAttempted = true
|
|
logger.log("agent did not join in time, retrying with fresh token")
|
|
await teardownRoom()
|
|
guard generation == sessionGeneration else { return }
|
|
await start()
|
|
}
|
|
|
|
/// Centralized failure teardown: mark teardown active before any await so
|
|
/// `connect()` is rejected during cleanup, cancel wait work, invalidate the
|
|
/// current generation (so stale callbacks become no-ops), clear all
|
|
/// per-room/session ephemeral published fields (agent, mute state, model
|
|
/// list, error state, interruption ownership), disconnect and recreate the
|
|
/// room, stop conversation state, restore the idle timer, and publish
|
|
/// `.failed(message)`, making retry safe: the old room is fully torn down
|
|
/// before Start becomes available again.
|
|
///
|
|
/// Deliberately preserved across failure: `settings` (durable user
|
|
/// preferences), `transcriptHandler` events (conversation history), and
|
|
/// `taskStore` (task history). These are not per-room ephemeral state.
|
|
///
|
|
/// Idempotent: if already tearing down or failed, keeps the first message.
|
|
private func teardownForFailure(message: String) async {
|
|
// Mark teardown active *before* any await so connect() rejects during
|
|
// cleanup and the UI does not expose Start prematurely.
|
|
if state == .tearingDown || state.isFailed { return }
|
|
state = .tearingDown
|
|
|
|
agentWaitTask?.cancel()
|
|
agentWaitTask = nil
|
|
sessionGeneration += 1
|
|
resetEphemeralFields()
|
|
// Cancel any in-flight interruption operation and invalidate the
|
|
// sequence so a pending .ended cannot resume the mic after teardown.
|
|
micOpTask?.cancel()
|
|
micOpTask = nil
|
|
interruptionSequence += 1
|
|
stateModel.stop()
|
|
UIApplication.shared.isIdleTimerDisabled = false
|
|
await teardownRoom()
|
|
|
|
// Final reset immediately before publishing .failed: any old-room
|
|
// callback delivered during the teardownRoom() await (participantDidConnect,
|
|
// connection-state changes) may have repopulated ephemeral fields. This
|
|
// second pass erases them so the published .failed state is clean
|
|
// (Iteration 09 Item 1).
|
|
resetEphemeralFields()
|
|
|
|
// Publish .failed only after all cleanup has completed. This makes the
|
|
// state a truthful signal: retry is safe because the old room is gone.
|
|
state = .failed(message)
|
|
}
|
|
|
|
private func startAgentWait() {
|
|
state = .waitingAgent
|
|
let generation = sessionGeneration
|
|
agentWaitTask?.cancel()
|
|
agentWaitTask = Task { [weak self] in
|
|
try? await Task.sleep(for: .seconds(10))
|
|
guard !Task.isCancelled else { return }
|
|
await self?.handleAgentTimeout(generation: generation)
|
|
}
|
|
}
|
|
|
|
func agentReadyReceived() {
|
|
guard !isTearingDown, state == .waitingAgent, sessionGeneration > 0 else {
|
|
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
|
return
|
|
}
|
|
agentWaitTask?.cancel()
|
|
state = .ready
|
|
stateModel.start()
|
|
let generation = sessionGeneration
|
|
// Honor the persisted interaction mode: text mode joins with the mic
|
|
// off and the agent in text mode; voice mode enables the mic.
|
|
if settings.textMode {
|
|
isMuted = true
|
|
userWantsMicMuted = true
|
|
logger.log("state=ready (text mode), mic stays off")
|
|
} else {
|
|
logger.log("state=ready, enabling mic")
|
|
Task { await enableMicrophone() }
|
|
}
|
|
Task { await syncSettingsOnJoin(generation: generation) }
|
|
}
|
|
|
|
private func syncSettingsOnJoin(generation: Int) async {
|
|
guard generation == sessionGeneration, isReady else { return }
|
|
// Each sync operation is independent and best-effort: a failed set_voice
|
|
// must not prevent set_mode from being sent. Hermes is server-managed.
|
|
let voiceID = settings.voiceID
|
|
do {
|
|
try await controlChannel.setVoice(voiceID, room: room)
|
|
guard generation == sessionGeneration else { return }
|
|
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
|
|
} catch {
|
|
logger.error("sync-on-join set_voice failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
guard generation == sessionGeneration else { return }
|
|
// Sync the agent's conversation mode with the persisted setting.
|
|
let mode = settings.textMode ? "text" : "voice"
|
|
do {
|
|
try await controlChannel.setMode(mode, room: room)
|
|
guard generation == sessionGeneration else { return }
|
|
logger.log("sync-on-join set_mode sent: \(mode, privacy: .public)")
|
|
} catch {
|
|
logger.error("sync-on-join set_mode failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
|
|
@discardableResult
|
|
func selectVoice(_ id: String) async -> Bool {
|
|
let previous = settings.voiceID
|
|
guard isReady else { settings.voiceID = id; return true }
|
|
let generation = sessionGeneration
|
|
do {
|
|
try await controlChannel.setVoice(id, room: room)
|
|
guard generation == sessionGeneration else { return false }
|
|
settings.voiceID = id
|
|
logger.log("set_voice sent: \(id, privacy: .public)")
|
|
return true
|
|
} catch {
|
|
settings.voiceID = previous
|
|
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
|
|
return false
|
|
}
|
|
}
|
|
|
|
private func scanForAgent() {
|
|
for participant in room.remoteParticipants.values {
|
|
if isAgentParticipant(participant) {
|
|
foundAgent(participant)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
private func isAgentParticipant(_ participant: RemoteParticipant) -> Bool {
|
|
participant.kind == .agent || participant.identity?.stringValue == "voice-assistant"
|
|
}
|
|
|
|
private func foundAgent(_ participant: RemoteParticipant) {
|
|
agent = AgentInfo(identity: participant.identity?.stringValue ?? "", name: participant.name)
|
|
}
|
|
|
|
func enableMicrophone() async {
|
|
let predecessor = micOpTask
|
|
let generation = sessionGeneration
|
|
let task = Task { @MainActor in
|
|
await predecessor?.value
|
|
guard !Task.isCancelled, generation == sessionGeneration, isReady, !settings.textMode else { return }
|
|
await enableMicrophoneSerialized()
|
|
}
|
|
micOpTask = task
|
|
await task.value
|
|
}
|
|
|
|
private func enableMicrophoneSerialized() async {
|
|
// Text mode invariant: the microphone must never be enabled while text
|
|
// mode is active. This enforces the privacy contract at the API level,
|
|
// not just in the view layer (Finding 4).
|
|
guard !settings.textMode else {
|
|
logger.log("enableMicrophone blocked: text mode active")
|
|
return
|
|
}
|
|
let generation = sessionGeneration
|
|
for attempt in 0..<3 {
|
|
guard generation == sessionGeneration, isReady, !settings.textMode, !Task.isCancelled else { return }
|
|
do {
|
|
try await room.localParticipant.setMicrophone(enabled: true)
|
|
guard generation == sessionGeneration else { return }
|
|
if settings.textMode || Task.isCancelled {
|
|
try? await room.localParticipant.setMicrophone(enabled: false)
|
|
isMuted = true
|
|
return
|
|
}
|
|
isMuted = false
|
|
userWantsMicMuted = false
|
|
micError = nil
|
|
logger.log("mic enabled")
|
|
return
|
|
} catch {
|
|
let message = String(describing: error)
|
|
if attempt < 2 {
|
|
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
|
try? await Task.sleep(for: .seconds(1))
|
|
guard generation == sessionGeneration else { return }
|
|
continue
|
|
}
|
|
// Non-fatal: keep the session usable (text mode still works).
|
|
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
|
guard generation == sessionGeneration else { return }
|
|
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
|
|
}
|
|
}
|
|
}
|
|
|
|
func setMuted(_ muted: Bool) async {
|
|
let predecessor = micOpTask
|
|
let generation = sessionGeneration
|
|
let task = Task { @MainActor in
|
|
await predecessor?.value
|
|
guard !Task.isCancelled, generation == sessionGeneration else { return }
|
|
await setMutedSerialized(muted)
|
|
}
|
|
micOpTask = task
|
|
await task.value
|
|
}
|
|
|
|
private func setMutedSerialized(_ muted: Bool) async {
|
|
guard isReady else { return }
|
|
// Text mode invariant: unmuting (enabling the mic) is not permitted
|
|
// while text mode is active. Muting is always allowed.
|
|
if !muted && settings.textMode {
|
|
logger.log("setMuted(false) blocked: text mode active")
|
|
return
|
|
}
|
|
let generation = sessionGeneration
|
|
do {
|
|
try await room.localParticipant.setMicrophone(enabled: !muted)
|
|
guard generation == sessionGeneration else { return }
|
|
if !muted && settings.textMode {
|
|
try? await room.localParticipant.setMicrophone(enabled: false)
|
|
isMuted = true
|
|
return
|
|
}
|
|
isMuted = muted
|
|
// Track explicit user mute intent separately from hardware state.
|
|
// Interruption begin operations use this after awaiting the
|
|
// predecessor to decide whether to mute (Iteration 08 Item 1).
|
|
userWantsMicMuted = muted
|
|
} catch {
|
|
guard generation == sessionGeneration else { return }
|
|
micError = "Microphone error: \(error.localizedDescription)"
|
|
}
|
|
}
|
|
|
|
func setMode(_ mode: String) async {
|
|
guard isReady else { return }
|
|
let generation = sessionGeneration
|
|
do {
|
|
try await controlChannel.setMode(mode, room: room)
|
|
guard generation == sessionGeneration else { return }
|
|
logger.log("set_mode sent: \(mode, privacy: .public)")
|
|
} catch {
|
|
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
|
|
/// Apply a persisted text-mode change to the live session: mute + set_mode
|
|
/// when enabling, set_mode + mic on when disabling. Called from
|
|
/// ConversationView's `.onChange(of: settings.textMode)` so changes from
|
|
/// either the conversation toggle or the Settings sheet apply to the
|
|
/// current session.
|
|
func applyTextMode(_ enabled: Bool) async {
|
|
guard isReady else { return }
|
|
guard enabled == settings.textMode else { return }
|
|
if enabled {
|
|
await setMuted(true)
|
|
guard settings.textMode else { return }
|
|
await setMode("text")
|
|
} else {
|
|
await setMode("voice")
|
|
guard !settings.textMode else { return }
|
|
await enableMicrophone()
|
|
}
|
|
}
|
|
|
|
func sendText(_ text: String) async throws {
|
|
guard isReady else { throw URLError(.notConnectedToInternet) }
|
|
logger.log("sendText lk.chat chars=\(text.count)")
|
|
try await room.localParticipant.sendText(text, for: "lk.chat")
|
|
}
|
|
|
|
func disconnect() async {
|
|
tokenClient.cancel()
|
|
// Set the teardown guard first so any .disconnected callback from our
|
|
// own room.disconnect() is ignored (not treated as a terminal failure).
|
|
isTearingDown = true
|
|
agentWaitTask?.cancel()
|
|
agentWaitTask = nil
|
|
sessionGeneration += 1
|
|
resetEphemeralFields()
|
|
// Cancel any in-flight interruption operation and invalidate the
|
|
// sequence so a pending .ended cannot resume the mic after disconnect.
|
|
micOpTask?.cancel()
|
|
micOpTask = nil
|
|
interruptionSequence += 1
|
|
stateModel.stop()
|
|
UIApplication.shared.isIdleTimerDisabled = false
|
|
await teardownRoom()
|
|
// Final reset before publishing .idle: any old-room callback delivered
|
|
// during the teardownRoom() await may have repopulated ephemeral fields.
|
|
// This ensures the idle state is clean for server replacement (Iteration 09).
|
|
resetEphemeralFields()
|
|
// Publish .idle only after room teardown has completed: the old session
|
|
// is fully torn down before the UI shows "ready to start".
|
|
state = .idle
|
|
}
|
|
|
|
// MARK: Audio session interruptions (phone call, Siri)
|
|
|
|
/// Handles audio session interruptions with a single retained operation
|
|
/// chain (`micOpTask`) for both begin and end operations.
|
|
///
|
|
/// **Serialization invariants (Iteration 07 Item 2):**
|
|
/// - Every new operation captures the prior `micOpTask` and awaits it before
|
|
/// executing. This guarantees strict ordering: a newer `.began` orders
|
|
/// after an in-flight old end/unmute and finishes by muting again, so a
|
|
/// stale unmute cannot leave hardware enabled during a newer interruption.
|
|
/// - `.began` increments `interruptionSequence`, then after predecessor
|
|
/// completion mutes and records ownership (`micMutedByInterruption`) only
|
|
/// after the mute succeeds.
|
|
/// - `.ended` is also assigned to the retained chain (never unretained),
|
|
/// awaits begin, and un-mutes only for matching sequence/generation/
|
|
/// ownership/shouldResume.
|
|
/// - Cancel/invalidate on disconnect or failure teardown.
|
|
@objc private func handleAudioInterruption(_ notification: Notification) {
|
|
guard let info = notification.userInfo else { return }
|
|
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
|
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
|
switch type {
|
|
case .began:
|
|
// Increment the sequence so any pending `.ended` from a prior
|
|
// interruption sees a stale sequence and does not resume.
|
|
interruptionSequence += 1
|
|
let seq = interruptionSequence
|
|
let generation = sessionGeneration
|
|
let predecessor = micOpTask
|
|
|
|
// Assign the new begin operation to the retained chain. It awaits
|
|
// the predecessor (which may be an in-flight end/unmute) *before*
|
|
// deciding whether to mute. The mute decision uses the user's
|
|
// explicit intent (`userWantsMicMuted`) and current text mode, NOT
|
|
// a pre-await snapshot of transient `isMuted` which may still
|
|
// reflect the predecessor's in-flight state (Iteration 08 Item 1).
|
|
//
|
|
// This guarantees: begin A → end A/unmute in flight → begin B.
|
|
// B awaits A's unmute (which sets isMuted=false, userWantsMicMuted
|
|
// already false from the successful enable). B then sees
|
|
// userWantsMicMuted == false and textMode == false, so it mutes —
|
|
// the final state is muted during interruption B.
|
|
micOpTask = Task { [weak self] in
|
|
guard let self else { return }
|
|
if let pred = predecessor { await pred.value }
|
|
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
|
// Reset ownership: a new interruption supersedes any prior claim.
|
|
self.micMutedByInterruption = false
|
|
// Decide whether to mute AFTER the predecessor completes, using
|
|
// user intent (not a pre-await isMuted snapshot). The mic should
|
|
// be muted during the interruption unless the user explicitly
|
|
// wanted it muted already (in which case hardware is already off)
|
|
// or text mode is active (mic must stay off by invariant).
|
|
let shouldMute = !self.userWantsMicMuted && !self.settings.textMode
|
|
&& self.state == .ready
|
|
guard shouldMute else {
|
|
logger.log("audio interruption began, no mute needed (userMuted=\(self.userWantsMicMuted), textMode=\(self.settings.textMode), state=\(String(describing: self.state)), seq=\(seq))")
|
|
return
|
|
}
|
|
do {
|
|
try await self.room.localParticipant.setMicrophone(enabled: false)
|
|
// Mark ownership only after the mute actually completed and
|
|
// the sequence is still current (no even newer .began).
|
|
guard !Task.isCancelled,
|
|
self.sessionGeneration == generation,
|
|
seq == self.interruptionSequence else { return }
|
|
self.isMuted = true
|
|
self.micMutedByInterruption = true
|
|
logger.log("interruption mute succeeded (seq=\(seq))")
|
|
} catch {
|
|
logger.log("interruption mute failed: \(error.localizedDescription) (seq=\(seq))")
|
|
}
|
|
}
|
|
logger.log("audio interruption began, seq=\(seq)")
|
|
case .ended:
|
|
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
|
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
|
// Capture the current sequence and generation before any async hop.
|
|
let currentSeq = interruptionSequence
|
|
let generation = sessionGeneration
|
|
let predecessor = micOpTask
|
|
|
|
// Assign the end operation to the retained chain (never unretained).
|
|
// It awaits the predecessor (the begin's mute) before deciding.
|
|
micOpTask = Task { [weak self] in
|
|
guard let self else { return }
|
|
if let pred = predecessor { await pred.value }
|
|
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
|
// Re-check all conditions after the await:
|
|
// 1. The mute succeeded (flag set only on success).
|
|
// 2. No newer .began has superseded this sequence.
|
|
// 3. The session generation still matches.
|
|
// 4. Text mode is off.
|
|
// 5. .shouldResume is present.
|
|
let shouldUnmute = self.micMutedByInterruption
|
|
&& currentSeq == self.interruptionSequence
|
|
&& self.sessionGeneration == generation
|
|
&& self.state == .ready
|
|
&& !self.settings.textMode
|
|
&& shouldResume
|
|
|
|
// Clear the flag regardless of whether we resume, so a stale
|
|
// true cannot cause an incorrect future resume.
|
|
self.micMutedByInterruption = false
|
|
|
|
if shouldUnmute {
|
|
do {
|
|
try await self.room.localParticipant.setMicrophone(enabled: true)
|
|
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
|
if self.settings.textMode {
|
|
try? await self.room.localParticipant.setMicrophone(enabled: false)
|
|
self.isMuted = true
|
|
return
|
|
}
|
|
self.isMuted = false
|
|
logger.log("audio interruption ended, resuming mic (seq=\(currentSeq))")
|
|
} catch {
|
|
logger.log("interruption resume failed: \(error.localizedDescription) (seq=\(currentSeq))")
|
|
}
|
|
} else {
|
|
let description = String(describing: self.state)
|
|
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public), shouldResume=\(shouldResume), seq=\(currentSeq))")
|
|
}
|
|
}
|
|
@unknown default:
|
|
break
|
|
}
|
|
}
|
|
|
|
func receiveAssistantProgress(_ data: Data) {
|
|
struct Progress: Decodable { let type: String; let request_id: String; let status: String; let message: String }
|
|
guard let progress = try? JSONDecoder().decode(Progress.self, from: data),
|
|
progress.type == "assistant_progress", !progress.request_id.isEmpty else { return }
|
|
switch progress.status {
|
|
case "working": workingRequests[progress.request_id] = progress.message.isEmpty ? "Hope is working on your request…" : progress.message
|
|
case "completed": workingRequests.removeValue(forKey: progress.request_id)
|
|
case "failed": workingRequests.removeValue(forKey: progress.request_id)
|
|
default: return
|
|
}
|
|
assistantActivity = workingRequests.values.sorted().first
|
|
if progress.status == "failed" { assistantActivity = "Hope could not complete that request. Please try again." }
|
|
}
|
|
|
|
func tickAudioLevels(local: Float, remote: Float) {
|
|
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
|
}
|
|
|
|
/// Pure decision seam for interruption-begin mute logic (Iteration 08 Item 1).
|
|
/// Extracted from `handleAudioInterruption` so the begin A → end A/unmute →
|
|
/// begin B transient-state case can be tested deterministically without a
|
|
/// live microphone or AVAudioSession.
|
|
///
|
|
/// Returns `true` if the begin operation should mute the mic, given the
|
|
/// user's explicit intent and current session state *after* the predecessor
|
|
/// has completed (not a pre-await snapshot).
|
|
func interruptionBeginShouldMute(
|
|
userWantsMicMuted: Bool,
|
|
textMode: Bool,
|
|
state: VoiceSessionState
|
|
) -> Bool {
|
|
!userWantsMicMuted && !textMode && state == .ready
|
|
}
|
|
|
|
}
|
|
|
|
extension VoiceSession: RoomDelegate {
|
|
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
|
Task { @MainActor [weak self] in
|
|
guard let self, self.room === room else { return }
|
|
// Gate on active lifecycle states and teardown flag so old-room
|
|
// callbacks delivered during failure teardown cannot repopulate
|
|
// the agent field after it was cleared (Iteration 09 Item 1).
|
|
guard self.shouldAcceptRoomCallback(state: self.state, isTearingDown: self.isTearingDown) else { return }
|
|
guard self.isAgentParticipant(participant) else { return }
|
|
self.foundAgent(participant)
|
|
}
|
|
}
|
|
|
|
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
|
|
Task { @MainActor [weak self] in
|
|
guard let self, self.room === room, self.state == .ready else { return }
|
|
var local = Float(0)
|
|
var remote = Float(0)
|
|
for participant in participants {
|
|
if participant.identity?.stringValue == room.localParticipant.identity?.stringValue {
|
|
local = max(local, participant.audioLevel)
|
|
} else {
|
|
remote = max(remote, participant.audioLevel)
|
|
}
|
|
}
|
|
self.tickAudioLevels(local: local, remote: remote)
|
|
}
|
|
}
|
|
|
|
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
|
Task { @MainActor [weak self] in
|
|
guard let self, self.room === room, !self.isTearingDown, self.state == .ready || self.state == .waitingAgent else { return }
|
|
if topic == "assistant-progress" {
|
|
self.receiveAssistantProgress(data)
|
|
return
|
|
}
|
|
// Secondary transcript source (finals only).
|
|
if topic == "transcript",
|
|
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
|
json["type"]?.stringValue == "transcript",
|
|
let text = json["text"]?.stringValue {
|
|
let role: TranscriptEvent.Role = json["role"]?.stringValue == "agent" ? .agent : .user
|
|
self.transcriptHandler.receiveDataTranscript(role: role, text: text)
|
|
return
|
|
}
|
|
if self.controlChannel.handleTaskEvent(data) { return }
|
|
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
|
}
|
|
}
|
|
|
|
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
|
|
Task { @MainActor [weak self] in
|
|
guard let self, self.room === room else { return }
|
|
// Ignore all connection-state updates while intentional teardown is
|
|
// in progress. The caller (teardownForFailure or disconnect) has
|
|
// already set the correct terminal/transitional state; any callback
|
|
// from the old room during the disconnect await must not mutate
|
|
// published fields (Iteration 09 Item 1).
|
|
if self.isTearingDown {
|
|
logger.log("connection-state callback ignored during teardown: \(String(describing: connectionState))")
|
|
return
|
|
}
|
|
switch (oldConnectionState, connectionState) {
|
|
case (.connected, .reconnecting):
|
|
// Only mutate published fields for active lifecycle states.
|
|
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
|
self.isReconnecting = true
|
|
logger.log("connection lost, waiting for LiveKit reconnect")
|
|
case (.reconnecting, .connected):
|
|
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
|
self.isReconnecting = false
|
|
logger.log("connection restored")
|
|
case (_, .disconnected):
|
|
// Terminal disconnect (not a transient reconnect). If the
|
|
// session was active, tear down cleanly so no stale room or
|
|
// wake lock survives. The user can retry via Start.
|
|
if self.state == .ready || self.state == .waitingAgent {
|
|
logger.log("terminal disconnect, tearing down session")
|
|
await self.teardownForFailure(message: "Connection lost.")
|
|
} else {
|
|
self.isReconnecting = false
|
|
}
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|