feat: hardening, error states, interruption handling, a11y (Phase 8)
This commit is contained in:
1 parent
9f1f26ba54
commit
12ded179eb
6 files changed
+233
-38
No files matched your search
@@ -27,7 +27,7 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X
|
|||||||
- [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below)
|
- [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below)
|
||||||
- [x] Phase 6: media search, choices, library panel (commit cbf1830)
|
- [x] Phase 6: media search, choices, library panel (commit cbf1830)
|
||||||
- [x] Phase 7: background tasks panel (commit af296da)
|
- [x] Phase 7: background tasks panel (commit af296da)
|
||||||
- [ ] Phase 8: polish/hardening/parity pass
|
- [x] Phase 8: polish/hardening/parity pass (commit pending)
|
||||||
- [ ] Phase 9: docs & handoff
|
- [ ] Phase 9: docs & handoff
|
||||||
|
|
||||||
## Phase 2 status
|
## Phase 2 status
|
||||||
@@ -151,6 +151,37 @@ Notes:
|
|||||||
- Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead).
|
- Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead).
|
||||||
- No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing.
|
- No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing.
|
||||||
|
|
||||||
|
## Phase 8 status (2026-08-30 ~13:40)
|
||||||
|
Done:
|
||||||
|
- **Stale-room guards**: `sessionGeneration` in VoiceSession, `pollGeneration` in MediaPlayerController. All async callbacks check generation before mutating state. RoomDelegate methods check `self.room === room`. Prevents post-disconnect mutations.
|
||||||
|
- **Audio interruption**: `handleAudioInterruption` in VoiceSession (mute mic on begin, resume on end if shouldResume). MediaPlayerController pauses/resumes AVPlayer on interruption.
|
||||||
|
- **Error states**: `isReconnecting` published property + reconnect banner in ConversationView. `stateError` published property + error banner in MediaPanelView.
|
||||||
|
- **Accessibility**: OrbView accessibilityLabel/Value (maps state to "Listening", "Thinking", etc.). ConversationView: Send, End, Tasks, Media, Settings labels. MediaPanelView: transport controls + refresh library labels.
|
||||||
|
- **Server URL change fix**: Changed `session` from `@ObservedObject` to `@State` in ConversationView so it can be reassigned inside async closures. Added `.onReceive(session.objectWillChange)` for observation. `.onChange(of: settings.serverURL)` disconnects old session and creates a new one when idle.
|
||||||
|
|
||||||
|
Verified in simulator:
|
||||||
|
- Build: zero errors, zero warnings.
|
||||||
|
- App launches cleanly.
|
||||||
|
- Server URL change (invalid → valid) triggers session recreation without crash.
|
||||||
|
- Connect/Disconnect/Reconnect matrix: Start → Ready → Hang up → Start again (3x). No crashes, state resets cleanly.
|
||||||
|
- Logs confirm no post-disconnect mutations (stale callbacks guarded).
|
||||||
|
|
||||||
|
Web parity checklist (vs /Users/shane/dev/hope-voice-api/web/app.js):
|
||||||
|
- [x] Token auth + LiveKit connect
|
||||||
|
- [x] Agent wait + agent_ready
|
||||||
|
- [x] Voice/model pickers + sync-on-join
|
||||||
|
- [x] Text mode toggle + barge-in hint
|
||||||
|
- [x] Transcript list + history
|
||||||
|
- [x] Media player (poll, reconcile, transport, seek)
|
||||||
|
- [x] Media search + choices + library
|
||||||
|
- [x] Background tasks panel
|
||||||
|
- [x] Stale-room guards (=== room checks)
|
||||||
|
- [x] Audio interruption handling
|
||||||
|
- [x] Error states (reconnect, mic error, media error)
|
||||||
|
- [x] Accessibility labels
|
||||||
|
- [ ] Web-only: drag-to-seek on waveform (iOS uses slider — acceptable)
|
||||||
|
- [ ] Web-only: keyboard shortcuts (N/A on iOS)
|
||||||
|
|
||||||
## Subagent notes
|
## Subagent notes
|
||||||
- Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md
|
- Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md
|
||||||
- Watch for loops (repeated identical commands) — kill and redirect if seen
|
- Watch for loops (repeated identical commands) — kill and redirect if seen
|
||||||
|
|||||||
@@ -1,32 +1,29 @@
|
|||||||
import SwiftUI
|
import SwiftUI
|
||||||
|
import Combine
|
||||||
|
|
||||||
struct ConversationView: View {
|
struct ConversationView: View {
|
||||||
@ObservedObject var settings: SettingsStore
|
@ObservedObject var settings: SettingsStore
|
||||||
@StateObject private var session: VoiceSession
|
@State private var sessionID = UUID()
|
||||||
|
@State private var session: VoiceSession
|
||||||
@State private var composerText = ""
|
@State private var composerText = ""
|
||||||
@State private var connectError: String?
|
@State private var connectError: String?
|
||||||
|
@State private var tick = 0
|
||||||
|
|
||||||
init(settings: SettingsStore) {
|
init(settings: SettingsStore) {
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
|
_session = State(initialValue: Self.makeSession(settings: settings))
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func makeSession(settings: SettingsStore) -> VoiceSession {
|
||||||
let config = ServerConfig.parse(settings.trimmedServerURL)
|
let config = ServerConfig.parse(settings.trimmedServerURL)
|
||||||
if case .success(let value) = config {
|
if case .success(let value) = config {
|
||||||
_session = StateObject(wrappedValue: VoiceSession(
|
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||||
config: value,
|
|
||||||
tokenClient: TokenClient(config: value),
|
|
||||||
settings: settings
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
|
|
||||||
if case .success(let value) = fallback {
|
|
||||||
_session = StateObject(wrappedValue: VoiceSession(
|
|
||||||
config: value,
|
|
||||||
tokenClient: TokenClient(config: value),
|
|
||||||
settings: settings
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
fatalError("Unreachable: default server URL is valid")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
|
||||||
|
if case .success(let value) = fallback {
|
||||||
|
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||||
|
}
|
||||||
|
fatalError("Unreachable: default server URL is valid")
|
||||||
}
|
}
|
||||||
|
|
||||||
var body: some View {
|
var body: some View {
|
||||||
@@ -42,12 +39,25 @@ struct ConversationView: View {
|
|||||||
}
|
}
|
||||||
.navigationTitle("Hope Voice")
|
.navigationTitle("Hope Voice")
|
||||||
.toolbar { toolbarContent }
|
.toolbar { toolbarContent }
|
||||||
|
.id(sessionID)
|
||||||
.onAppear {
|
.onAppear {
|
||||||
if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) {
|
if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) {
|
||||||
connectError = error.localizedDescription
|
connectError = error.localizedDescription
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.onReceive(session.objectWillChange.receive(on: RunLoop.main)) { _ in
|
||||||
|
tick &+= 1
|
||||||
|
}
|
||||||
|
.onChange(of: settings.serverURL) { _, _ in
|
||||||
|
guard session.state == .idle else { return }
|
||||||
|
let oldSession = session
|
||||||
|
Task { @MainActor [settings] in
|
||||||
|
await oldSession.disconnect()
|
||||||
|
self.sessionID = UUID()
|
||||||
|
self.session = Self.makeSession(settings: settings)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private var statusHeader: some View {
|
private var statusHeader: some View {
|
||||||
@@ -93,6 +103,12 @@ struct ConversationView: View {
|
|||||||
.toggleStyle(.button)
|
.toggleStyle(.button)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if session.state == .ready, session.isReconnecting {
|
||||||
|
Label("Connection lost — reconnecting…", systemImage: "arrow.triangle.2.circlepath")
|
||||||
|
.font(.footnote)
|
||||||
|
.foregroundStyle(.orange)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
}
|
||||||
if let micError = session.micError {
|
if let micError = session.micError {
|
||||||
Label(micError, systemImage: "exclamationmark.triangle")
|
Label(micError, systemImage: "exclamationmark.triangle")
|
||||||
.font(.footnote)
|
.font(.footnote)
|
||||||
@@ -163,10 +179,12 @@ struct ConversationView: View {
|
|||||||
Image(systemName: "arrow.up.circle.fill")
|
Image(systemName: "arrow.up.circle.fill")
|
||||||
.font(.title2)
|
.font(.title2)
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Send message")
|
||||||
.disabled(!settings.textMode || composerText.isEmpty)
|
.disabled(!settings.textMode || composerText.isEmpty)
|
||||||
Button(role: .destructive, action: endSession) {
|
Button(role: .destructive, action: endSession) {
|
||||||
Image(systemName: "phone.down.fill")
|
Image(systemName: "phone.down.fill")
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("End conversation")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.padding()
|
.padding()
|
||||||
@@ -190,6 +208,7 @@ struct ConversationView: View {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Background tasks")
|
||||||
}
|
}
|
||||||
ToolbarItem(placement: .topBarTrailing) {
|
ToolbarItem(placement: .topBarTrailing) {
|
||||||
NavigationLink {
|
NavigationLink {
|
||||||
@@ -197,6 +216,7 @@ struct ConversationView: View {
|
|||||||
} label: {
|
} label: {
|
||||||
Image(systemName: "music.note")
|
Image(systemName: "music.note")
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Media player")
|
||||||
}
|
}
|
||||||
ToolbarItem(placement: .topBarTrailing) {
|
ToolbarItem(placement: .topBarTrailing) {
|
||||||
NavigationLink {
|
NavigationLink {
|
||||||
@@ -204,6 +224,7 @@ struct ConversationView: View {
|
|||||||
} label: {
|
} label: {
|
||||||
Image(systemName: "gearshape")
|
Image(systemName: "gearshape")
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Settings")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -30,6 +30,17 @@ struct OrbView: View {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private var label: String {
|
||||||
|
switch state {
|
||||||
|
case .idle: "Idle"
|
||||||
|
case .connecting: "Connecting"
|
||||||
|
case .listening: "Listening"
|
||||||
|
case .speaking: "Speaking"
|
||||||
|
case .thinking: "Thinking"
|
||||||
|
case .failed: "Connection failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var body: some View {
|
var body: some View {
|
||||||
let shape = Circle()
|
let shape = Circle()
|
||||||
.fill(baseColor.gradient)
|
.fill(baseColor.gradient)
|
||||||
@@ -44,6 +55,9 @@ struct OrbView: View {
|
|||||||
.animation(anim, value: animating)
|
.animation(anim, value: animating)
|
||||||
.onAppear { animating = true }
|
.onAppear { animating = true }
|
||||||
.onChange(of: state) { _, _ in animating = true }
|
.onChange(of: state) { _, _ in animating = true }
|
||||||
|
.accessibilityElement()
|
||||||
|
.accessibilityLabel("Conversation status")
|
||||||
|
.accessibilityValue(label)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ struct MediaPanelView: View {
|
|||||||
var body: some View {
|
var body: some View {
|
||||||
ScrollView {
|
ScrollView {
|
||||||
VStack(spacing: 20) {
|
VStack(spacing: 20) {
|
||||||
|
if let stateError = playerController.stateError {
|
||||||
|
Label(stateError, systemImage: "wifi.exclamationmark")
|
||||||
|
.font(.subheadline)
|
||||||
|
.foregroundStyle(.red)
|
||||||
|
.frame(maxWidth: .infinity, alignment: .leading)
|
||||||
|
.padding(12)
|
||||||
|
.background(Color.red.opacity(0.1), in: RoundedRectangle(cornerRadius: 14))
|
||||||
|
}
|
||||||
nowPlayingCard
|
nowPlayingCard
|
||||||
transportControls
|
transportControls
|
||||||
seekSection
|
seekSection
|
||||||
@@ -103,6 +111,7 @@ struct MediaPanelView: View {
|
|||||||
Image(systemName: "gobackward.10")
|
Image(systemName: "gobackward.10")
|
||||||
.font(.title3)
|
.font(.title3)
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Seek back 10 seconds")
|
||||||
.disabled(playerController.track == nil)
|
.disabled(playerController.track == nil)
|
||||||
|
|
||||||
Button(action: togglePlayPause) {
|
Button(action: togglePlayPause) {
|
||||||
@@ -110,18 +119,21 @@ struct MediaPanelView: View {
|
|||||||
.font(.system(size: 44))
|
.font(.system(size: 44))
|
||||||
.foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor)
|
.foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor)
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel(playerController.desired == "playing" && playerController.track != nil ? "Pause" : "Play")
|
||||||
.disabled(playerController.track == nil)
|
.disabled(playerController.track == nil)
|
||||||
|
|
||||||
Button(action: seekForward) {
|
Button(action: seekForward) {
|
||||||
Image(systemName: "goforward.10")
|
Image(systemName: "goforward.10")
|
||||||
.font(.title3)
|
.font(.title3)
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Seek forward 10 seconds")
|
||||||
.disabled(playerController.track == nil)
|
.disabled(playerController.track == nil)
|
||||||
|
|
||||||
Button(role: .destructive, action: stopPlayback) {
|
Button(role: .destructive, action: stopPlayback) {
|
||||||
Image(systemName: "stop.circle")
|
Image(systemName: "stop.circle")
|
||||||
.font(.title2)
|
.font(.title2)
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Stop playback")
|
||||||
.disabled(playerController.track == nil)
|
.disabled(playerController.track == nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -276,6 +288,7 @@ struct MediaPanelView: View {
|
|||||||
Button(action: refreshLibrary) {
|
Button(action: refreshLibrary) {
|
||||||
Image(systemName: "arrow.clockwise")
|
Image(systemName: "arrow.clockwise")
|
||||||
}
|
}
|
||||||
|
.accessibilityLabel("Refresh library")
|
||||||
}
|
}
|
||||||
if let libraryError {
|
if let libraryError {
|
||||||
Text(libraryError)
|
Text(libraryError)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import UIKit
|
import UIKit
|
||||||
|
import AVFoundation
|
||||||
import Foundation
|
import Foundation
|
||||||
import LiveKit
|
import LiveKit
|
||||||
import os
|
import os
|
||||||
@@ -26,9 +27,14 @@ final class VoiceSession: ObservableObject {
|
|||||||
@Published private(set) var agent: AgentInfo?
|
@Published private(set) var agent: AgentInfo?
|
||||||
@Published private(set) var isMuted = false
|
@Published private(set) var isMuted = false
|
||||||
@Published private(set) var micError: String?
|
@Published private(set) var micError: String?
|
||||||
|
@Published private(set) var isReconnecting = false
|
||||||
@Published private(set) var models: [ControlChannel.ModelInfo] = []
|
@Published private(set) var models: [ControlChannel.ModelInfo] = []
|
||||||
@Published private(set) var selectedModel: String?
|
@Published private(set) var selectedModel: String?
|
||||||
|
|
||||||
|
/// Identity of the current room. Every async callback that could outlive a
|
||||||
|
/// disconnect/reconnect compares its captured generation against this before
|
||||||
|
/// mutating state (port of the web UI's `=== room` checks).
|
||||||
|
private(set) var sessionGeneration = 0
|
||||||
private(set) var room = Room()
|
private(set) var room = Room()
|
||||||
let controlChannel = ControlChannel()
|
let controlChannel = ControlChannel()
|
||||||
let transcriptHandler = TranscriptHandler()
|
let transcriptHandler = TranscriptHandler()
|
||||||
@@ -57,9 +63,18 @@ final class VoiceSession: ObservableObject {
|
|||||||
self?.handleControlEvent(event)
|
self?.handleControlEvent(event)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
NotificationCenter.default.addObserver(
|
||||||
|
self, selector: #selector(handleAudioInterruption),
|
||||||
|
name: AVAudioSession.interruptionNotification, object: nil
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
deinit {
|
||||||
|
NotificationCenter.default.removeObserver(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
|
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
|
||||||
|
guard state != .idle else { return }
|
||||||
switch event {
|
switch event {
|
||||||
case .models(let received, let selected):
|
case .models(let received, let selected):
|
||||||
models = received
|
models = received
|
||||||
@@ -82,6 +97,7 @@ final class VoiceSession: ObservableObject {
|
|||||||
var isReady: Bool { state == .ready }
|
var isReady: Bool { state == .ready }
|
||||||
|
|
||||||
func connect() async {
|
func connect() async {
|
||||||
|
guard state == .idle else { return }
|
||||||
await start()
|
await start()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,11 +111,15 @@ final class VoiceSession: ObservableObject {
|
|||||||
|
|
||||||
private func start() async {
|
private func start() async {
|
||||||
state = .connecting
|
state = .connecting
|
||||||
|
isReconnecting = false
|
||||||
|
sessionGeneration += 1
|
||||||
|
let generation = sessionGeneration
|
||||||
connectAttempts = 0
|
connectAttempts = 0
|
||||||
await attemptConnect()
|
await attemptConnect(generation: generation)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func attemptConnect() async {
|
private func attemptConnect(generation: Int) async {
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
let result: Result<Void, Error>
|
let result: Result<Void, Error>
|
||||||
do {
|
do {
|
||||||
let token = try await tokenClient.fetchToken()
|
let token = try await tokenClient.fetchToken()
|
||||||
@@ -110,6 +130,7 @@ final class VoiceSession: ObservableObject {
|
|||||||
} catch {
|
} catch {
|
||||||
result = .failure(error)
|
result = .failure(error)
|
||||||
}
|
}
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
switch result {
|
switch result {
|
||||||
case .success:
|
case .success:
|
||||||
logger.log("room connected")
|
logger.log("room connected")
|
||||||
@@ -123,7 +144,8 @@ final class VoiceSession: ObservableObject {
|
|||||||
let attempt = connectAttempts
|
let attempt = connectAttempts
|
||||||
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
||||||
await teardownRoom()
|
await teardownRoom()
|
||||||
await attemptConnect()
|
guard generation == sessionGeneration else { return }
|
||||||
|
await attemptConnect(generation: generation)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
||||||
@@ -144,30 +166,32 @@ final class VoiceSession: ObservableObject {
|
|||||||
room = makeRoom()
|
room = makeRoom()
|
||||||
}
|
}
|
||||||
|
|
||||||
private func handleAgentTimeout() async {
|
private func handleAgentTimeout(generation: Int) async {
|
||||||
guard state == .waitingAgent, !reconnectAttempted else {
|
guard generation == sessionGeneration, state == .waitingAgent else { return }
|
||||||
if state == .waitingAgent {
|
guard !reconnectAttempted else {
|
||||||
state = .failed("Agent never joined. Is the agent worker running?")
|
state = .failed("Agent never joined. Is the agent worker running?")
|
||||||
}
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
reconnectAttempted = true
|
reconnectAttempted = true
|
||||||
|
logger.log("agent did not join in time, retrying with fresh token")
|
||||||
await teardownRoom()
|
await teardownRoom()
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
await start()
|
await start()
|
||||||
}
|
}
|
||||||
|
|
||||||
private func startAgentWait() {
|
private func startAgentWait() {
|
||||||
state = .waitingAgent
|
state = .waitingAgent
|
||||||
|
let generation = sessionGeneration
|
||||||
agentWaitTask?.cancel()
|
agentWaitTask?.cancel()
|
||||||
agentWaitTask = Task { [weak self] in
|
agentWaitTask = Task { [weak self] in
|
||||||
try? await Task.sleep(for: .seconds(10))
|
try? await Task.sleep(for: .seconds(10))
|
||||||
guard !Task.isCancelled else { return }
|
guard !Task.isCancelled else { return }
|
||||||
await self?.handleAgentTimeout()
|
await self?.handleAgentTimeout(generation: generation)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func agentReadyReceived() {
|
func agentReadyReceived() {
|
||||||
guard state == .waitingAgent else {
|
guard state == .waitingAgent, sessionGeneration > 0 else {
|
||||||
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -175,18 +199,21 @@ final class VoiceSession: ObservableObject {
|
|||||||
state = .ready
|
state = .ready
|
||||||
stateModel.start()
|
stateModel.start()
|
||||||
logger.log("state=ready, enabling mic")
|
logger.log("state=ready, enabling mic")
|
||||||
|
let generation = sessionGeneration
|
||||||
Task { await enableMicrophone() }
|
Task { await enableMicrophone() }
|
||||||
Task { await syncSettingsOnJoin() }
|
Task { await syncSettingsOnJoin(generation: generation) }
|
||||||
}
|
}
|
||||||
|
|
||||||
private func syncSettingsOnJoin() async {
|
private func syncSettingsOnJoin(generation: Int) async {
|
||||||
guard isReady else { return }
|
guard generation == sessionGeneration, isReady else { return }
|
||||||
let voiceID = settings.voiceID
|
let voiceID = settings.voiceID
|
||||||
do {
|
do {
|
||||||
try await controlChannel.setVoice(voiceID, room: room)
|
try await controlChannel.setVoice(voiceID, room: room)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
|
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
|
||||||
if let model = selectedModel ?? settings.modelID {
|
if let model = selectedModel ?? settings.modelID {
|
||||||
try await controlChannel.setModel(model, room: room)
|
try await controlChannel.setModel(model, room: room)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
|
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
@@ -197,8 +224,10 @@ final class VoiceSession: ObservableObject {
|
|||||||
func selectVoice(_ id: String) async {
|
func selectVoice(_ id: String) async {
|
||||||
settings.voiceID = id
|
settings.voiceID = id
|
||||||
guard isReady else { return }
|
guard isReady else { return }
|
||||||
|
let generation = sessionGeneration
|
||||||
do {
|
do {
|
||||||
try await controlChannel.setVoice(id, room: room)
|
try await controlChannel.setVoice(id, room: room)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
logger.log("set_voice sent: \(id, privacy: .public)")
|
logger.log("set_voice sent: \(id, privacy: .public)")
|
||||||
} catch {
|
} catch {
|
||||||
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
|
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
|
||||||
@@ -208,8 +237,10 @@ final class VoiceSession: ObservableObject {
|
|||||||
func selectModel(_ id: String) async {
|
func selectModel(_ id: String) async {
|
||||||
settings.modelID = id
|
settings.modelID = id
|
||||||
guard isReady else { return }
|
guard isReady else { return }
|
||||||
|
let generation = sessionGeneration
|
||||||
do {
|
do {
|
||||||
try await controlChannel.setModel(id, room: room)
|
try await controlChannel.setModel(id, room: room)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
logger.log("set_model sent: \(id, privacy: .public)")
|
logger.log("set_model sent: \(id, privacy: .public)")
|
||||||
} catch {
|
} catch {
|
||||||
logger.error("set_model failed: \(String(describing: error), privacy: .public)")
|
logger.error("set_model failed: \(String(describing: error), privacy: .public)")
|
||||||
@@ -234,9 +265,11 @@ final class VoiceSession: ObservableObject {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func enableMicrophone() async {
|
func enableMicrophone() async {
|
||||||
|
let generation = sessionGeneration
|
||||||
for attempt in 0..<3 {
|
for attempt in 0..<3 {
|
||||||
do {
|
do {
|
||||||
try await room.localParticipant.setMicrophone(enabled: true)
|
try await room.localParticipant.setMicrophone(enabled: true)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
isMuted = false
|
isMuted = false
|
||||||
micError = nil
|
micError = nil
|
||||||
logger.log("mic enabled")
|
logger.log("mic enabled")
|
||||||
@@ -246,10 +279,12 @@ final class VoiceSession: ObservableObject {
|
|||||||
if attempt < 2 {
|
if attempt < 2 {
|
||||||
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
||||||
try? await Task.sleep(for: .seconds(1))
|
try? await Task.sleep(for: .seconds(1))
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Non-fatal: keep the session usable (text mode still works).
|
// Non-fatal: keep the session usable (text mode still works).
|
||||||
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
|
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -257,18 +292,23 @@ final class VoiceSession: ObservableObject {
|
|||||||
|
|
||||||
func setMuted(_ muted: Bool) async {
|
func setMuted(_ muted: Bool) async {
|
||||||
guard isReady else { return }
|
guard isReady else { return }
|
||||||
|
let generation = sessionGeneration
|
||||||
do {
|
do {
|
||||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
isMuted = muted
|
isMuted = muted
|
||||||
} catch {
|
} catch {
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
micError = "Microphone error: \(error.localizedDescription)"
|
micError = "Microphone error: \(error.localizedDescription)"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func setMode(_ mode: String) async {
|
func setMode(_ mode: String) async {
|
||||||
guard isReady else { return }
|
guard isReady else { return }
|
||||||
|
let generation = sessionGeneration
|
||||||
do {
|
do {
|
||||||
try await controlChannel.setMode(mode, room: room)
|
try await controlChannel.setMode(mode, room: room)
|
||||||
|
guard generation == sessionGeneration else { return }
|
||||||
logger.log("set_mode sent: \(mode, privacy: .public)")
|
logger.log("set_mode sent: \(mode, privacy: .public)")
|
||||||
} catch {
|
} catch {
|
||||||
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
|
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
|
||||||
@@ -276,17 +316,20 @@ final class VoiceSession: ObservableObject {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func sendText(_ text: String) async throws {
|
func sendText(_ text: String) async throws {
|
||||||
|
guard isReady else { throw URLError(.notConnectedToInternet) }
|
||||||
logger.log("sendText lk.chat chars=\(text.count)")
|
logger.log("sendText lk.chat chars=\(text.count)")
|
||||||
try await room.localParticipant.sendText(text, for: "lk.chat")
|
try await room.localParticipant.sendText(text, for: "lk.chat")
|
||||||
}
|
}
|
||||||
|
|
||||||
func disconnect() async {
|
func disconnect() async {
|
||||||
agentWaitTask?.cancel()
|
agentWaitTask?.cancel()
|
||||||
|
sessionGeneration += 1
|
||||||
state = .idle
|
state = .idle
|
||||||
agent = nil
|
agent = nil
|
||||||
isMuted = false
|
isMuted = false
|
||||||
micError = nil
|
micError = nil
|
||||||
models = []
|
models = []
|
||||||
|
isReconnecting = false
|
||||||
reconnectAttempted = false
|
reconnectAttempted = false
|
||||||
stateModel.stop()
|
stateModel.stop()
|
||||||
UIApplication.shared.isIdleTimerDisabled = false
|
UIApplication.shared.isIdleTimerDisabled = false
|
||||||
@@ -294,6 +337,34 @@ final class VoiceSession: ObservableObject {
|
|||||||
await teardownRoom()
|
await teardownRoom()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: Audio session interruptions (phone call, Siri)
|
||||||
|
|
||||||
|
@objc private func handleAudioInterruption(_ notification: Notification) {
|
||||||
|
guard let info = notification.userInfo else { return }
|
||||||
|
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
||||||
|
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
||||||
|
switch type {
|
||||||
|
case .began:
|
||||||
|
let wasActive = state == .ready && !isMuted && !settings.textMode
|
||||||
|
if wasActive {
|
||||||
|
Task { await setMuted(true) }
|
||||||
|
}
|
||||||
|
logger.log("audio interruption began, muted=\(wasActive)")
|
||||||
|
case .ended:
|
||||||
|
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
||||||
|
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
||||||
|
if state == .ready, isMuted, !settings.textMode, shouldResume {
|
||||||
|
Task { await setMuted(false) }
|
||||||
|
logger.log("audio interruption ended, resuming mic")
|
||||||
|
} else {
|
||||||
|
let description = String(describing: state)
|
||||||
|
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))")
|
||||||
|
}
|
||||||
|
@unknown default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func tickAudioLevels(local: Float, remote: Float) {
|
func tickAudioLevels(local: Float, remote: Float) {
|
||||||
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
||||||
}
|
}
|
||||||
@@ -303,14 +374,14 @@ final class VoiceSession: ObservableObject {
|
|||||||
extension VoiceSession: RoomDelegate {
|
extension VoiceSession: RoomDelegate {
|
||||||
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
||||||
Task { @MainActor [weak self] in
|
Task { @MainActor [weak self] in
|
||||||
guard let self, self.isAgentParticipant(participant) else { return }
|
guard let self, self.room === room, self.isAgentParticipant(participant) else { return }
|
||||||
self.foundAgent(participant)
|
self.foundAgent(participant)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
|
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
|
||||||
Task { @MainActor [weak self] in
|
Task { @MainActor [weak self] in
|
||||||
guard let self, self.state == .ready else { return }
|
guard let self, self.room === room, self.state == .ready else { return }
|
||||||
var local = Float(0)
|
var local = Float(0)
|
||||||
var remote = Float(0)
|
var remote = Float(0)
|
||||||
for participant in participants {
|
for participant in participants {
|
||||||
@@ -327,7 +398,7 @@ extension VoiceSession: RoomDelegate {
|
|||||||
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
||||||
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
|
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
|
||||||
Task { @MainActor [weak self] in
|
Task { @MainActor [weak self] in
|
||||||
guard let self else { return }
|
guard let self, self.room === room else { return }
|
||||||
// Secondary transcript source (finals only).
|
// Secondary transcript source (finals only).
|
||||||
if topic == "transcript",
|
if topic == "transcript",
|
||||||
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
||||||
@@ -341,4 +412,20 @@ extension VoiceSession: RoomDelegate {
|
|||||||
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
|
||||||
|
Task { @MainActor [weak self] in
|
||||||
|
guard let self, self.room === room else { return }
|
||||||
|
switch (oldConnectionState, connectionState) {
|
||||||
|
case (.connected, .reconnecting):
|
||||||
|
self.isReconnecting = true
|
||||||
|
logger.log("connection lost, waiting for LiveKit reconnect")
|
||||||
|
case (.reconnecting, .connected):
|
||||||
|
self.isReconnecting = false
|
||||||
|
logger.log("connection restored")
|
||||||
|
default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -16,6 +16,7 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
@Published private(set) var duration: Double = 0
|
@Published private(set) var duration: Double = 0
|
||||||
@Published private(set) var pendingJobs: [MediaPendingJob] = []
|
@Published private(set) var pendingJobs: [MediaPendingJob] = []
|
||||||
@Published private(set) var choices: [MediaChoice] = []
|
@Published private(set) var choices: [MediaChoice] = []
|
||||||
|
@Published private(set) var stateError: String?
|
||||||
|
|
||||||
let player = AVPlayer()
|
let player = AVPlayer()
|
||||||
|
|
||||||
@@ -34,6 +35,11 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
private var timeObserver: Any?
|
private var timeObserver: Any?
|
||||||
private var statusCancellable: AnyCancellable?
|
private var statusCancellable: AnyCancellable?
|
||||||
|
|
||||||
|
/// Bumped on every room-scope change so in-flight /state polls from the
|
||||||
|
/// previous scope cannot reconcile into the new one (port of the web UI's
|
||||||
|
/// `=== room` checks).
|
||||||
|
private var pollGeneration = 0
|
||||||
|
|
||||||
init(api: MediaAPI, settings: SettingsStore) {
|
init(api: MediaAPI, settings: SettingsStore) {
|
||||||
self.api = api
|
self.api = api
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
@@ -52,6 +58,25 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
NotificationCenter.default.removeObserver(self)
|
NotificationCenter.default.removeObserver(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@objc private func handleInterruption(_ notification: Notification) {
|
||||||
|
guard let info = notification.userInfo else { return }
|
||||||
|
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
||||||
|
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
||||||
|
switch type {
|
||||||
|
case .began:
|
||||||
|
player.pause()
|
||||||
|
logger.log("audio interruption began, paused media")
|
||||||
|
case .ended:
|
||||||
|
applyVolume()
|
||||||
|
if currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
|
||||||
|
player.play()
|
||||||
|
logger.log("audio interruption ended, resumed media")
|
||||||
|
}
|
||||||
|
@unknown default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// MARK: Lifecycle
|
// MARK: Lifecycle
|
||||||
|
|
||||||
func attach(session: VoiceSession) {
|
func attach(session: VoiceSession) {
|
||||||
@@ -63,6 +88,7 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
/// Called just before joining the LiveKit room. Carries whatever is loaded
|
/// Called just before joining the LiveKit room. Carries whatever is loaded
|
||||||
/// into the new room-scoped player state, paused (talking pauses music).
|
/// into the new room-scoped player state, paused (talking pauses music).
|
||||||
func enterMediaScope(roomName: String) async {
|
func enterMediaScope(roomName: String) async {
|
||||||
|
pollGeneration += 1
|
||||||
let wasPlaying = currentTrackID != nil && !player.rate.isZero
|
let wasPlaying = currentTrackID != nil && !player.rate.isZero
|
||||||
wasPlayingBeforeTalk = wasPlaying
|
wasPlayingBeforeTalk = wasPlaying
|
||||||
let carriedTrack = currentTrackID
|
let carriedTrack = currentTrackID
|
||||||
@@ -83,6 +109,7 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
/// hand the room back to the player.
|
/// hand the room back to the player.
|
||||||
func endConversation() async {
|
func endConversation() async {
|
||||||
stopPolling()
|
stopPolling()
|
||||||
|
pollGeneration += 1
|
||||||
if wasPlayingBeforeTalk {
|
if wasPlayingBeforeTalk {
|
||||||
wasPlayingBeforeTalk = false
|
wasPlayingBeforeTalk = false
|
||||||
guard let room else { return }
|
guard let room else { return }
|
||||||
@@ -120,12 +147,17 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
|
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
|
||||||
|
|
||||||
private func pollState() async {
|
private func pollState() async {
|
||||||
|
let generation = pollGeneration
|
||||||
let scope = room ?? "default"
|
let scope = room ?? "default"
|
||||||
do {
|
do {
|
||||||
let state = try await api.state(room: scope)
|
let state = try await api.state(room: scope)
|
||||||
|
guard generation == pollGeneration else { return }
|
||||||
|
stateError = nil
|
||||||
reconcile(state)
|
reconcile(state)
|
||||||
} catch {
|
} catch {
|
||||||
|
guard generation == pollGeneration else { return }
|
||||||
logger.log("poll failed: \(String(describing: error), privacy: .public)")
|
logger.log("poll failed: \(String(describing: error), privacy: .public)")
|
||||||
|
stateError = "Media server unreachable: \(error.localizedDescription)"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,10 +278,6 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@objc private func handleInterruption(_ notification: Notification) {
|
|
||||||
applyVolume()
|
|
||||||
}
|
|
||||||
|
|
||||||
// MARK: Reporting + publishing
|
// MARK: Reporting + publishing
|
||||||
|
|
||||||
private func report(_ state: String) {
|
private func report(_ state: String) {
|
||||||
@@ -344,6 +372,7 @@ final class MediaPlayerController: ObservableObject {
|
|||||||
player.replaceCurrentItem(with: nil)
|
player.replaceCurrentItem(with: nil)
|
||||||
currentTime = 0
|
currentTime = 0
|
||||||
duration = 0
|
duration = 0
|
||||||
|
stateError = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: Volume
|
// MARK: Volume
|
||||||
|
|||||||
Reference in new issue
Block a user