From ca2d26f171de505e35c5c427799245db0d47ce3c Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sun, 30 Aug 2026 07:50:45 -0400 Subject: [PATCH] feat: voice and model pickers with sync-on-join (Phase 4) - Voices.swift: 13 Azure voices, default en-US-Ava - SettingsStore: persist voiceID/modelID, fallback to default voice - VoiceSession: sync set_voice/set_model on agent_ready, selectVoice/selectModel, track models + model_changed - SettingsView: voice and model picker sheets - ControlChannel: publish via publish(data:) reliable data channel instead of sendText DataStream (agent listens on legacy data_received; DataStream text was dropped) --- PLAN.md | 25 +++- Sources/App/SettingsStore.swift | 3 +- Sources/App/Voices.swift | 34 +++++ .../Conversation/ConversationView.swift | 13 +- Sources/Features/Settings/SettingsView.swift | 118 +++++++++++++++++- Sources/LiveKit/ControlChannel.swift | 4 +- Sources/LiveKit/VoiceSession.swift | 69 +++++++++- Voice.xcodeproj/project.pbxproj | 4 + 8 files changed, 256 insertions(+), 14 deletions(-) create mode 100644 Sources/App/Voices.swift diff --git a/PLAN.md b/PLAN.md index 4b1ebab..6285583 100644 --- a/PLAN.md +++ b/PLAN.md @@ -22,10 +22,10 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X - [x] Phase 0: scaffold (commit 297fbaa) - [x] Phase 1: ServerConfig/TokenClient/TLSPinning/ConnectionTest (commit 378b8cb) - [x] Phase 2: LiveKit session spike — commit 4986b6d (mic -4010 is sim env limitation; verify on device) -- [x] Phase 3: state machine, audio levels, wake lock, history (commit 70aef02) -- [ ] Phase 4: media playback reconciliation -- [ ] Phase 5: AVPlayer + local reverse proxy for self-signed https media -- [ ] Phase 6: settings parity (voice/model pickers, text mode) +- [x] Phase 3: state machine, audio levels, wake lock, history (commit 5483e6e) +- [x] Phase 4: voice/model pickers with sync-on-join (commit pending) +- [ ] Phase 5: media playback reconciliation + AVPlayer + local reverse proxy for self-signed https media +- [ ] Phase 6: settings parity (text mode done in Phase 3; remaining polish) - [ ] Phase 7: polish/edge cases - [ ] Phase 8: real-device qualification checklist - [ ] Phase 9: docs/final @@ -75,6 +75,23 @@ Needs device verification: - Live voice-driven states (listening while user speaks, thinking gap >2s chip, barge-in) — sim mic is silent (-4010), so local audio level never rises in sim; only remote (agent) audio drove the machine here. - Mic meter movement with real speech; wake lock behavior on physical screen-off. +## Phase 4 status (2026-08-30 ~11:50) +Done: +- `Voices.swift`: 13 Azure voices from web app.js; default en-US-Ava:DragonHDOmniLatestNeural. +- `SettingsStore`: voiceID/modelID persisted to UserDefaults; invalid stored voice falls back to default. +- `VoiceSession`: sync-on-join on agent_ready sends set_voice + set_model (if known); selectVoice/selectModel send immediately and persist; models list + model_changed acks tracked (`@Published models`, `selectedModel`). +- `SettingsView`: VoicePickerView sheet (13 voices), ModelPickerView sheet (models from session, "Start a conversation..." when not connected). +- `ConversationView`: shows current voice/model in status header when ready. + +### SENDTEXT BUG — FIXED (2026-08-30 ~11:47) +Root cause: ControlChannel used `localParticipant.sendText(for:)` (LiveKit DataStream protocol, v2.16). Agent listens on legacy data channel (`ctx.room.on("data_received")`), which never receives DataStream text → "ignoring text stream with topic 'voice-control', no callback attached" in agent logs; set_voice/set_model silently dropped, no model_changed ack. +Fix: `ControlChannel.publish` now uses `localParticipant.publish(data:options:)` with `DataPublishOptions(topic:, reliable: true)` — same path as web UI's `publishData`. +Verified (sim + agent docker logs, room voice-shane-7bf126cf): +- iOS: sync-on-join set_voice sent: en-US-JennyNeural; set_model sent: gemma-4-e4b → model_changed ack received and applied; models list count=30. +- Agent: "Switching TTS voice to en-US-JennyNeural" / "Voice updated" / "LLM model change: qwen2.5-vl-3b -> gemma-4-e4b". +- Persistence: relaunch → sync-on-join sent previously selected Jenny + gemma-4-e4b (survived UserDefaults). +Note: `sendText lk.chat` in VoiceSession is also DataStream-based and the agent has no lk.chat handler — text chat path is unverified server-side, out of scope here. + ## Subagent notes - Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md - Watch for loops (repeated identical commands) — kill and redirect if seen diff --git a/Sources/App/SettingsStore.swift b/Sources/App/SettingsStore.swift index 7770868..4cfef97 100644 --- a/Sources/App/SettingsStore.swift +++ b/Sources/App/SettingsStore.swift @@ -29,7 +29,8 @@ final class SettingsStore: ObservableObject { init() { let d = UserDefaults.standard serverURL = d.string(forKey: Keys.serverURL) ?? "https://192.168.86.2:9178" - voiceID = d.string(forKey: Keys.voiceID) ?? "" + let storedVoice = d.string(forKey: Keys.voiceID) + voiceID = Voices.all.contains { $0.id == storedVoice } ? (storedVoice ?? "") : Voices.defaultID modelID = d.string(forKey: Keys.modelID) textMode = d.bool(forKey: Keys.textMode) volume = d.object(forKey: Keys.volume) as? Double ?? 0.8 diff --git a/Sources/App/Voices.swift b/Sources/App/Voices.swift new file mode 100644 index 0000000..c37dd91 --- /dev/null +++ b/Sources/App/Voices.swift @@ -0,0 +1,34 @@ +import Foundation + +struct VoiceOption: Equatable { + let id: String + let name: String + let tag: String? + + var displayName: String { + tag.map { "\(name) (\($0))" } ?? name + } +} + +enum Voices { + static let all: [VoiceOption] = [ + VoiceOption(id: "en-US-Ava:DragonHDOmniLatestNeural", name: "Ava", tag: "Dragon HD Omni"), + VoiceOption(id: "en-US-JennyNeural", name: "Jenny", tag: nil), + VoiceOption(id: "en-US-GuyNeural", name: "Guy", tag: nil), + VoiceOption(id: "en-US-AndrewNeural", name: "Andrew", tag: nil), + VoiceOption(id: "en-US-AriaNeural", name: "Aria", tag: nil), + VoiceOption(id: "en-US-EmmaNeural", name: "Emma", tag: nil), + VoiceOption(id: "en-US-EricNeural", name: "Eric", tag: nil), + VoiceOption(id: "en-US-BrianNeural", name: "Brian", tag: nil), + VoiceOption(id: "en-US-AshleyNeural", name: "Ashley", tag: nil), + VoiceOption(id: "en-US-RichardNeural", name: "Richard", tag: nil), + VoiceOption(id: "en-US-TinaNeural", name: "Tina", tag: nil), + VoiceOption(id: "en-US-SteffanNeural", name: "Steffan", tag: nil), + ] + + static var defaultID: String { all[0].id } + + static func display(for id: String) -> String? { + all.first { $0.id == id }?.displayName + } +} diff --git a/Sources/Features/Conversation/ConversationView.swift b/Sources/Features/Conversation/ConversationView.swift index f8dc83c..ad1b57a 100644 --- a/Sources/Features/Conversation/ConversationView.swift +++ b/Sources/Features/Conversation/ConversationView.swift @@ -12,14 +12,16 @@ struct ConversationView: View { if case .success(let value) = config { _session = StateObject(wrappedValue: VoiceSession( config: value, - tokenClient: TokenClient(config: value) + tokenClient: TokenClient(config: value), + settings: settings )) } else { let fallback = ServerConfig.parse("https://192.168.86.2:9178") if case .success(let value) = fallback { _session = StateObject(wrappedValue: VoiceSession( config: value, - tokenClient: TokenClient(config: value) + tokenClient: TokenClient(config: value), + settings: settings )) } else { fatalError("Unreachable: default server URL is valid") @@ -73,6 +75,11 @@ struct ConversationView: View { .font(.caption) .foregroundStyle(.secondary) } + if session.state == .ready { + Text("\(Voices.display(for: settings.voiceID) ?? settings.voiceID) · \(session.selectedModel ?? settings.modelID ?? "—")") + .font(.caption) + .foregroundStyle(.secondary) + } if session.state == .ready { HStack(spacing: 16) { Toggle(isOn: Binding( @@ -168,7 +175,7 @@ struct ConversationView: View { private var toolbarContent: some ToolbarContent { ToolbarItem(placement: .topBarTrailing) { NavigationLink { - SettingsView(settings: settings) + SettingsView(settings: settings, session: session) } label: { Image(systemName: "gearshape") } diff --git a/Sources/Features/Settings/SettingsView.swift b/Sources/Features/Settings/SettingsView.swift index a2ff8fb..9722904 100644 --- a/Sources/Features/Settings/SettingsView.swift +++ b/Sources/Features/Settings/SettingsView.swift @@ -2,7 +2,15 @@ import SwiftUI struct SettingsView: View { @ObservedObject var settings: SettingsStore + var session: VoiceSession? @State private var urlText: String = "" + @State private var showVoicePicker = false + @State private var showModelPicker = false + + init(settings: SettingsStore, session: VoiceSession? = nil) { + self.settings = settings + self.session = session + } var body: some View { Form { @@ -15,6 +23,26 @@ struct SettingsView: View { settings.serverURL = newValue } } + Section("Voice") { + Button { + showVoicePicker = true + } label: { + LabeledContent("Voice", value: Voices.display(for: settings.voiceID) ?? settings.voiceID) + } + } + Section("Model") { + if let session, !session.models.isEmpty { + Button { + showModelPicker = true + } label: { + LabeledContent("Model", value: session.selectedModel ?? settings.modelID ?? "—") + } + } else { + Text(session != nil ? "Loading available models…" : "Start a conversation to load the model list.") + .font(.footnote) + .foregroundStyle(.secondary) + } + } Section("Interaction") { Toggle("Text mode", isOn: $settings.textMode) } @@ -25,9 +53,97 @@ struct SettingsView: View { } } .navigationTitle("Settings") + .sheet(isPresented: $showVoicePicker) { + VoicePickerView(settings: settings, session: session) + } + .sheet(isPresented: $showModelPicker) { + ModelPickerView(settings: settings, session: session) + } + } +} + +struct VoicePickerView: View { + @ObservedObject var settings: SettingsStore + var session: VoiceSession? + @Environment(\.dismiss) private var dismiss + + var body: some View { + NavigationStack { + List { + ForEach(Voices.all, id: \.id) { voice in + Button { + settings.voiceID = voice.id + Task { await session?.selectVoice(voice.id) } + dismiss() + } label: { + HStack { + Text(voice.displayName) + if settings.voiceID == voice.id { + Spacer() + Image(systemName: "checkmark") + .foregroundStyle(.blue) + } + } + } + } + } + .navigationTitle("Voice") + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button("Cancel") { dismiss() } + } + } + } + } +} + +struct ModelPickerView: View { + @ObservedObject var settings: SettingsStore + var session: VoiceSession? + @Environment(\.dismiss) private var dismiss + + private var activeModelID: String? { + guard let session else { return settings.modelID } + if !session.models.isEmpty, + let selected = session.selectedModel ?? settings.modelID, + session.models.contains(where: { $0.id == selected }) { + return selected + } + return session.models.first?.id + } + + var body: some View { + NavigationStack { + List { + ForEach(session?.models ?? [], id: \.id) { model in + Button { + settings.modelID = model.id + Task { await session?.selectModel(model.id) } + dismiss() + } label: { + HStack { + Text(model.name ?? model.id) + if activeModelID == model.id { + Spacer() + Image(systemName: "checkmark") + .foregroundStyle(.blue) + } + } + } + } + } + .navigationTitle("Model") + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button("Cancel") { dismiss() } + } + } + } } } #Preview { - SettingsView(settings: SettingsStore()) + NavigationStack { + SettingsView(settings: SettingsStore()) + } } diff --git a/Sources/LiveKit/ControlChannel.swift b/Sources/LiveKit/ControlChannel.swift index 0f59215..30b4167 100644 --- a/Sources/LiveKit/ControlChannel.swift +++ b/Sources/LiveKit/ControlChannel.swift @@ -84,8 +84,8 @@ final class ControlChannel: NSObject, RoomDelegate { // MARK: Outbound (topic'd JSON over reliable data streams) private func publish(_ object: [String: Any], topic: String, room: Room) async throws { - let text = String(data: try JSONSerialization.data(withJSONObject: object), encoding: .utf8) ?? "{}" - _ = try await room.localParticipant.sendText(text, for: topic) + let data = try JSONSerialization.data(withJSONObject: object) + try await room.localParticipant.publish(data: data, options: DataPublishOptions(topic: topic, reliable: true)) } func setVoice(_ voiceID: String, room: Room) async throws { diff --git a/Sources/LiveKit/VoiceSession.swift b/Sources/LiveKit/VoiceSession.swift index fda283c..3c512c9 100644 --- a/Sources/LiveKit/VoiceSession.swift +++ b/Sources/LiveKit/VoiceSession.swift @@ -26,6 +26,8 @@ final class VoiceSession: ObservableObject { @Published private(set) var agent: AgentInfo? @Published private(set) var isMuted = false @Published private(set) var micError: String? + @Published private(set) var models: [ControlChannel.ModelInfo] = [] + @Published private(set) var selectedModel: String? private(set) var room = Room() let controlChannel = ControlChannel() @@ -33,18 +35,40 @@ final class VoiceSession: ObservableObject { let stateModel = ConversationStateModel() private let config: ServerConfig private let tokenClient: TokenClient + private let settings: SettingsStore private var reconnectAttempted = false private var connectAttempts = 0 private var agentWaitTask: Task? - init(config: ServerConfig, tokenClient: TokenClient) { + init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore) { self.config = config self.tokenClient = tokenClient + self.settings = settings room.add(delegate: self) room.add(delegate: controlChannel) room.add(delegate: transcriptHandler) - controlChannel.onEvent = { [weak self] _ in - guard self != nil else { return } + controlChannel.onEvent = { [weak self] event in + Task { @MainActor [weak self] in + self?.handleControlEvent(event) + } + } + } + + private func handleControlEvent(_ event: ControlChannel.InboundEvent) { + switch event { + case .models(let received, let selected): + models = received + if selectedModel == nil, let selected { + selectedModel = selected + settings.modelID = selected + } + logger.log("models received count=\(received.count) selected=\(selected ?? "nil", privacy: .public)") + case .modelChanged(let model): + selectedModel = model + settings.modelID = model + logger.log("model_changed applied: \(model, privacy: .public)") + default: + break } } @@ -144,6 +168,44 @@ final class VoiceSession: ObservableObject { stateModel.start() logger.log("state=ready, enabling mic") Task { await enableMicrophone() } + Task { await syncSettingsOnJoin() } + } + + private func syncSettingsOnJoin() async { + guard isReady else { return } + let voiceID = settings.voiceID + do { + try await controlChannel.setVoice(voiceID, room: room) + logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)") + if let model = selectedModel ?? settings.modelID { + try await controlChannel.setModel(model, room: room) + logger.log("sync-on-join set_model sent: \(model, privacy: .public)") + } + } catch { + logger.error("sync-on-join failed: \(String(describing: error), privacy: .public)") + } + } + + func selectVoice(_ id: String) async { + settings.voiceID = id + guard isReady else { return } + do { + try await controlChannel.setVoice(id, room: room) + logger.log("set_voice sent: \(id, privacy: .public)") + } catch { + logger.error("set_voice failed: \(String(describing: error), privacy: .public)") + } + } + + func selectModel(_ id: String) async { + settings.modelID = id + guard isReady else { return } + do { + try await controlChannel.setModel(id, room: room) + logger.log("set_model sent: \(id, privacy: .public)") + } catch { + logger.error("set_model failed: \(String(describing: error), privacy: .public)") + } } private func scanForAgent() { @@ -216,6 +278,7 @@ final class VoiceSession: ObservableObject { agent = nil isMuted = false micError = nil + models = [] reconnectAttempted = false stateModel.stop() UIApplication.shared.isIdleTimerDisabled = false diff --git a/Voice.xcodeproj/project.pbxproj b/Voice.xcodeproj/project.pbxproj index 71eac8c..a3c4208 100644 --- a/Voice.xcodeproj/project.pbxproj +++ b/Voice.xcodeproj/project.pbxproj @@ -23,11 +23,13 @@ D18AD347DFF72AB6361165BA /* TokenClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9681E7410B684D2D38752A70 /* TokenClient.swift */; }; D2E343C8B8657316A993698E /* TranscriptHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */; }; D9A0059E342955A06A021D6C /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = A6BAD55715F383C0DB5F306F /* LiveKit */; }; + EE0C5762EC97B123A1880FFB /* Voices.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E5DF207EDB787F63CB3A64D /* Voices.swift */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentMonitor.swift; sourceTree = ""; }; 2A4C3CE2CA6FF828452803D1 /* Voice.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = Voice.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 2E5DF207EDB787F63CB3A64D /* Voices.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Voices.swift; sourceTree = ""; }; 2FED8976B60FBC13BC980FCB /* VoiceApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceApp.swift; sourceTree = ""; }; 3F3FA9F420065D2127B0AEC0 /* ConversationView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConversationView.swift; sourceTree = ""; }; 48F76881B4AE51DEE74511CC /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; @@ -127,6 +129,7 @@ children = ( 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */, 2FED8976B60FBC13BC980FCB /* VoiceApp.swift */, + 2E5DF207EDB787F63CB3A64D /* Voices.swift */, ); path = App; sourceTree = ""; @@ -220,6 +223,7 @@ 89B47754E5C39418BFD839E6 /* TranscriptHistory.swift in Sources */, 1ECD63F10A8EAF33FD2A5B62 /* VoiceApp.swift in Sources */, CDE38FE9679472D770C601E6 /* VoiceSession.swift in Sources */, + EE0C5762EC97B123A1880FFB /* Voices.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; };