From 4986b6d7dc982c2717e54efa87569d880fbad125 Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sun, 30 Aug 2026 07:26:58 -0400 Subject: [PATCH] feat: LiveKit voice session, control channel, transcripts (Phase 2) - VoiceSession: token->connect with fresh-room retry on timeout, agent discovery, agent_ready wait, mic enable/mute, text send over lk.chat - ControlChannel: outbound set_voice/set_model/get_models/set_mode/ media_state; inbound agent_ready/models/model_changed/task_event - TranscriptHandler: transcription segments + transcript topic finals, partial/final dedupe per role - ConversationView/OrbView: status orb, mic/text-mode toggles, composer - ServerConfig: plain ws:// signaling for private IPs (self-signed TLS not injectable into SDK WebSocket); Info.plist local networking --- PLAN.md | 61 +++++ Resources/Info.plist | 13 +- Sources/App/VoiceApp.swift | 5 + .../Conversation/ConversationView.swift | 166 +++++++++++-- Sources/Features/Conversation/OrbView.swift | 101 ++++++++ Sources/LiveKit/AgentMonitor.swift | 38 +++ Sources/LiveKit/ControlChannel.swift | 149 +++++++++++ Sources/LiveKit/TranscriptHandler.swift | 68 ++++++ Sources/LiveKit/VoiceSession.swift | 231 ++++++++++++++++++ Sources/Networking/ServerConfig.swift | 19 +- Sources/Networking/TLSPinning.swift | 61 +++-- Sources/Networking/TokenClient.swift | 4 +- Voice.xcodeproj/project.pbxproj | 28 +++ project.yml | 6 +- 14 files changed, 900 insertions(+), 50 deletions(-) create mode 100644 PLAN.md create mode 100644 Sources/Features/Conversation/OrbView.swift create mode 100644 Sources/LiveKit/AgentMonitor.swift create mode 100644 Sources/LiveKit/ControlChannel.swift create mode 100644 Sources/LiveKit/TranscriptHandler.swift create mode 100644 Sources/LiveKit/VoiceSession.swift diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..0df2d9e --- /dev/null +++ b/PLAN.md @@ -0,0 +1,61 @@ +# Voice iOS — Plan & Progress + +Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via XcodeBuildMCP. + +## Setup (reference) +- Repo /Users/shane/dev/voice-ios, branch feat/ios-app +- xcodegen at /opt/homebrew/bin/xcodegen; regenerate after file changes: `xcodegen generate` +- Build: `xcodebuild -project Voice.xcodeproj -scheme Voice_iOS -destination 'generic/platform=iOS Simulator' build` (scheme is **Voice_iOS**, not Voice) +- Run/relaunch: xcodebuildmcp_build_run_sim or launch_app_sim (bundle dev.shane.voice, sim iPhone 17 Pro AFDA137E-1657-428E-BF01-E5CD7989772C) +- UI: snapshot_ui → tap elementRef. Read logs: `xcrun simctl spawn booted log show --last Nm --predicate 'process == "Voice"' --style compact | grep -E "io.livekit|dev.shane.voice"` +- Mic permission (if simulator wiped): `xcrun simctl privacy booted grant microphone dev.shane.voice` + +## Server facts (verified) +- Base https://192.168.86.2:9178 (nginx, self-signed CN=192.168.86.2; cert regenerates daily-ish → pins break on rotation) +- POST /token → {token, room: voice-shane-, user{id, display_name}} +- LiveKit host-network container `voice` on Unraid (ssh root@192.168.86.2): signaling 7880/tcp, RTC tcp 7881, udp 7882 — all reachable from Mac (nc verified) +- SDK cannot trust self-signed cert on its WebSocket → use plain ws://192.168.86.2:7880 (no /livekit path; SDK derives /rtc). Token+media only via nginx :9178 TLS (AVPlayer media needs local proxy in Phase 5) +- agent_ready on topic conversation-control; models on voice-control (plain string list); agent identity agent-*, kind .agent; lk.agent.session stream unhandled (harmless) +- TLSPinning.session() = custom URLSessionDelegate DER pin for our HTTP calls + +## Phases +- [x] Phase 0: scaffold (commit 297fbaa) +- [x] Phase 1: ServerConfig/TokenClient/TLSPinning/ConnectionTest (commit 378b8cb) +- [ ] Phase 2: LiveKit session spike — **ACTIVE** +- [ ] Phase 3: state machine, audio levels, wake lock, history +- [ ] Phase 4: media playback reconciliation +- [ ] Phase 5: AVPlayer + local reverse proxy for self-signed https media +- [ ] Phase 6: settings parity (voice/model pickers, text mode) +- [ ] Phase 7: polish/edge cases +- [ ] Phase 8: real-device qualification checklist +- [ ] Phase 9: docs/final + +## Phase 2 status +Done: VoiceSession (token→connect, agent wait, mic enable, mute, sendText lk.chat), ControlChannel, TranscriptHandler, AgentMonitor, ConversationView/OrbView. Verified once end-to-end yesterday: token→room connected→agent_ready→models received. +UNCOMMITTED: all Phase 2 files + ServerConfig ws:// derivation + Info.plist NSAllowsLocalNetworking. + +### CONNECT TIMEOUT BUG — FIXED (2026-08-30 ~07:15) +Root cause: stale `Room` instance after a timed-out connect left signaling/transport in a broken state; subsequent connects hung. Fix in VoiceSession.swift: +- `room` is now `private(set) var`, recreated fresh via `makeRoom()` (delegates re-attached) on each retry. +- `attemptConnect()` retries up to 2 extra times (3 total) on `.timedOut`: tears down the room (`teardownRoom()`) and reconnects with a new Room + fresh token. +- `makeConnectOptions()` sets explicit timeouts (socket 15s, primary/publisher transport 20s). +- Removed manual `AVAudioSession` config/deactivation — the SDK's `AudioSessionEngineObserver` (isAutomaticConfigurationEnabled=true by default) owns the session; our manual setCategory/setActive raced it. +Verified: token→room connected in <1s (retry fired once on a slow first attempt, then succeeded), agent_ready received, state=ready. 3/3 connect attempts now reach room-connected. + +### REMAINING: mic enable -4010 (simulator env limitation, NOT app bug) +After state=ready, `setMicrophone(enabled:true)` fails with CoreAudio `-4010` (kAudioUnitErr_InvalidElement) on all 3 retries. Investigated thoroughly: +- TCC: `xcrun simctl privacy booted grant microphone dev.shane.voice` applied; still -4010 → not a permission issue. +- CoreAudio logs: input node reports **0 channel descriptions** and the engine associates with audio session `(0x0)` (null) → the simulator is not providing a usable mic source to this app in this environment. +- NSMicrophoneUsageDescription present in Info.plist; SDK auto-configures the session correctly. +Conclusion: simulator microphone hardware/source unavailable here. App code path is correct (mic enable retries, proper error surfacing). Needs verification on a real device or a simulator with a working mic source. Do NOT treat -4010 as an app regression. + +### After connect bug fixed +- [x] Verify ready state reached (state=ready logged; UI transitions to conversation view) +- [ ] Verify transcript arrival + disconnect cleanup (blocked on mic/audio in sim; do on device) +- [x] Fix Swift 6 warnings: ControlChannel @MainActor, VoiceSession closure, duplicate TranscriptHandler removed from ConversationView (build now warning-free) +- [ ] Commit Phase 2, mark done + +## Subagent notes +- Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md +- Watch for loops (repeated identical commands) — kill and redirect if seen +- Don't let subagents loop reading SDK source; compile instead. SDK checkout: ~/Library/Developer/XcodeBuildMCP/workspaces/voice-ios-9eb036fdea81/DerivedData/Voice-38a9ac34e5c8/SourcePackages/checkouts/client-sdk-swift diff --git a/Resources/Info.plist b/Resources/Info.plist index f37f95b..9c4a6de 100644 --- a/Resources/Info.plist +++ b/Resources/Info.plist @@ -22,14 +22,21 @@ 1 NSAppTransportSecurity - NSExceptionDomains + NSAllowsLocalNetworking + + NSPinnedDomains 192.168.86.2 - NSExceptionAllowsInsecureHTTPLoads - NSIncludesSubdomains + NSPinnedCAIdentities + + + SPKI-SHA256-BASE64 + vmrfRuHv1oHBBiEmJP4WX3C7mtVy3C4+UJ1Yk/Q5hMU= + + diff --git a/Sources/App/VoiceApp.swift b/Sources/App/VoiceApp.swift index 7705623..6129f39 100644 --- a/Sources/App/VoiceApp.swift +++ b/Sources/App/VoiceApp.swift @@ -1,9 +1,14 @@ import SwiftUI +import LiveKit @main struct VoiceApp: App { @StateObject private var settings = SettingsStore() + init() { + LiveKitSDK.setLogLevel(.debug) + } + var body: some Scene { WindowGroup { ConversationView(settings: settings) diff --git a/Sources/Features/Conversation/ConversationView.swift b/Sources/Features/Conversation/ConversationView.swift index c36cdbc..b980965 100644 --- a/Sources/Features/Conversation/ConversationView.swift +++ b/Sources/Features/Conversation/ConversationView.swift @@ -2,33 +2,161 @@ import SwiftUI struct ConversationView: View { @ObservedObject var settings: SettingsStore + @StateObject private var session: VoiceSession + @State private var composerText = "" + @State private var connectError: String? + + init(settings: SettingsStore) { + self.settings = settings + let config = ServerConfig.parse(settings.trimmedServerURL) + if case .success(let value) = config { + _session = StateObject(wrappedValue: VoiceSession( + config: value, + tokenClient: TokenClient(config: value) + )) + } else { + let fallback = ServerConfig.parse("https://192.168.86.2:9178") + if case .success(let value) = fallback { + _session = StateObject(wrappedValue: VoiceSession( + config: value, + tokenClient: TokenClient(config: value) + )) + } else { + fatalError("Unreachable: default server URL is valid") + } + } + } var body: some View { NavigationStack { - VStack(spacing: 24) { - Spacer() - Circle() - .fill(.blue.gradient) - .frame(width: 96, height: 96) - Text("Hope Voice") - .font(.title.bold()) - Text(settings.trimmedServerURL) - .font(.footnote) - .foregroundStyle(.secondary) - Spacer() + VStack(spacing: 0) { + statusHeader + TranscriptListView(events: session.transcriptHandler.events) + composer } - .navigationTitle("Voice") - .toolbar { - ToolbarItem(placement: .topBarTrailing) { - NavigationLink { - SettingsView(settings: settings) - } label: { - Image(systemName: "gearshape") - } + .navigationTitle("Hope Voice") + .toolbar { toolbarContent } + .onAppear { + if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) { + connectError = error.localizedDescription } } } } + + private var statusHeader: some View { + VStack(spacing: 12) { + OrbView(state: orbState) + Text(statusText) + .font(.headline) + if let agent = session.agent { + Text("Agent: \(agent.name ?? agent.identity)") + .font(.caption) + .foregroundStyle(.secondary) + } + if session.state == .ready { + HStack(spacing: 16) { + Toggle(isOn: Binding( + get: { !session.isMuted }, + set: { newValue in Task { await session.setMuted(!newValue) } } + )) { + Label(session.isMuted ? "Muted" : "Mic on", systemImage: session.isMuted ? "mic.slash" : "mic") + } + .toggleStyle(.button) + Toggle("Text mode", isOn: $settings.textMode) + .toggleStyle(.button) + } + } + if let connectError { + Label(connectError, systemImage: "exclamationmark.triangle") + .font(.footnote) + .foregroundStyle(.red) + } + } + .padding() + } + + private var composer: some View { + HStack(spacing: 8) { + switch session.state { + case .idle, .failed: + Button(action: startSession) { + Label("Start", systemImage: "phone.fill") + } + .buttonStyle(.borderedProminent) + case .connecting, .waitingAgent: + ProgressView() + case .ready: + TextField(settings.textMode ? "Type a message…" : "Voice mode active", + text: $composerText, + axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(1...4) + .disabled(!settings.textMode) + Button(action: sendComposer) { + Image(systemName: "arrow.up.circle.fill") + .font(.title2) + } + .disabled(!settings.textMode || composerText.isEmpty) + Button(role: .destructive, action: endSession) { + Image(systemName: "phone.down.fill") + } + } + } + .padding() + } + + private var toolbarContent: some ToolbarContent { + ToolbarItem(placement: .topBarTrailing) { + NavigationLink { + SettingsView(settings: settings) + } label: { + Image(systemName: "gearshape") + } + } + } + + private var orbState: OrbView.OrbState { + switch session.state { + case .idle: .idle + case .connecting, .waitingAgent: .connecting + case .ready: settings.textMode ? .idle : .listening + case .failed: .failed + } + } + + private var statusText: String { + switch session.state { + case .idle: "Ready to connect" + case .connecting: "Connecting…" + case .waitingAgent: "Waiting for agent…" + case .ready: settings.textMode ? "Text mode" : "Listening" + case .failed(let message): "Failed: \(message)" + } + } + + private func startSession() { + connectError = nil + session.transcriptHandler.clear() + Task { await session.connect() } + } + + private func endSession() { + Task { await session.disconnect() } + } + + private func sendComposer() { + let text = composerText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + composerText = "" + Task { + do { + try await session.sendText(text) + } catch { + connectError = error.localizedDescription + } + } + } } #Preview { diff --git a/Sources/Features/Conversation/OrbView.swift b/Sources/Features/Conversation/OrbView.swift new file mode 100644 index 0000000..f3de1c3 --- /dev/null +++ b/Sources/Features/Conversation/OrbView.swift @@ -0,0 +1,101 @@ +import SwiftUI + +struct OrbView: View { + enum OrbState { + case idle + case connecting + case listening + case speaking + case thinking + case failed + } + + let state: OrbState + + @State private var animating = false + + private var baseColor: Color { + switch state { + case .idle, .connecting: .blue + case .listening: .green + case .speaking, .thinking: .orange + case .failed: .red + } + } + + private var scale: CGFloat { + switch state { + case .connecting, .listening, .speaking, .thinking: 1.1 + default: 1.0 + } + } + + var body: some View { + let shape = Circle() + .fill(baseColor.gradient) + .frame(width: 120, height: 120) + let animated = shape + .scaleEffect(animating ? scale : 0.95) + .opacity(state == .idle || state == .failed ? 0.7 : 1.0) + let anim: Animation? = state == .idle || state == .failed + ? nil + : .easeInOut(duration: 0.8).repeatForever(autoreverses: true) + animated + .animation(anim, value: animating) + .onAppear { animating = true } + .onChange(of: state) { _, _ in animating = true } + } +} + +struct TranscriptListView: View { + let events: [TranscriptEvent] + + var body: some View { + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 8) { + ForEach(events) { event in + transcriptRow(event) + } + } + .padding(.horizontal) + } + .onChange(of: events.count) { _, count in + if let last = events.last { + withAnimation { proxy.scrollTo(last.id, anchor: .bottom) } + } + } + } + } + + private func transcriptRow(_ event: TranscriptEvent) -> some View { + let roleName = event.role == .user ? "You" : "Hope" + let nameColor: Color = event.role == .user ? .secondary : .blue + let rowBackground: Color = event.role == .user + ? Color(.secondarySystemBackground) + : Color.blue.opacity(0.08) + let textOpacity: Double = event.isFinal ? 1.0 : 0.55 + return VStack(alignment: .leading, spacing: 2) { + Text(roleName) + .font(.caption2.bold()) + .foregroundStyle(nameColor) + Text(stripped(event.text)) + .font(.body) + .opacity(textOpacity) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(8) + .background(rowBackground) + .cornerRadius(10) + .id(event.id) + } + + private func stripped(_ text: String) -> String { + // Strip delivery-style markers like [appreciative], [pauses] etc. + text.replacingOccurrences( + of: #"\[[a-zA-Z ,'-]+\]"#, + with: "", + options: .regularExpression + ).trimmingCharacters(in: .whitespaces) + } +} diff --git a/Sources/LiveKit/AgentMonitor.swift b/Sources/LiveKit/AgentMonitor.swift new file mode 100644 index 0000000..61af58d --- /dev/null +++ b/Sources/LiveKit/AgentMonitor.swift @@ -0,0 +1,38 @@ +import Foundation +import LiveKit + +/// Observes the agent participant's audio level for UI state inference. +@MainActor +final class AgentMonitor: ObservableObject { + @Published private(set) var agentSpeaking = false + @Published private(set) var localSpeaking = false + + private let session: VoiceSession + private var pollTask: Task? + + init(session: VoiceSession) { + self.session = session + } + + func start() { + stop() + pollTask = Task { [weak self] in + while !Task.isCancelled { + guard let self else { return } + let remote = self.session.room.remoteParticipants.values.first(where: { + $0.kind == .agent || $0.identity?.stringValue == "voice-assistant" + }) + self.agentSpeaking = remote?.isSpeaking ?? false + self.localSpeaking = self.session.room.localParticipant.isSpeaking + try? await Task.sleep(for: .milliseconds(250)) + } + } + } + + func stop() { + pollTask?.cancel() + pollTask = nil + agentSpeaking = false + localSpeaking = false + } +} diff --git a/Sources/LiveKit/ControlChannel.swift b/Sources/LiveKit/ControlChannel.swift new file mode 100644 index 0000000..0f59215 --- /dev/null +++ b/Sources/LiveKit/ControlChannel.swift @@ -0,0 +1,149 @@ +import Foundation +import LiveKit + +/// Outbound helpers for agent control topics and inbound parsing of +/// voice-control / conversation-control / media-control data messages. +@MainActor +final class ControlChannel: NSObject, RoomDelegate { + + struct ModelInfo: Equatable { + let id: String + let name: String? + } + + enum InboundEvent { + case agentReady + case models([ModelInfo], selected: String?) + case modelChanged(String) + case taskEvent(TaskEventPayload) + } + + struct TaskEventPayload: Equatable { + let type: String + let taskId: String? + let description: String? + let role: String? + let content: String? + let result: String? + let error: String? + } + + var onEvent: ((InboundEvent) -> Void)? + + private let decoder = JSONDecoder() + + /// Inbound voice-control handling. Returns true when the message was consumed. + func handleVoiceControl(_ data: Data, session: VoiceSession) -> Bool { + guard let json = try? decoder.decode([String: JSONValue].self, from: data), + let type = json["type"]?.stringValue else { return false } + switch type { + case "agent_ready": + Task { @MainActor [weak session] in + session?.agentReadyReceived() + } + return true + case "models": + var models: [ModelInfo] = [] + for item in json["models"]?.arrayValue ?? [] { + if let obj = item.objectValue, let id = obj["id"]?.stringValue { + models.append(ModelInfo(id: id, name: obj["name"]?.stringValue)) + } else if let id = item.stringValue { + models.append(ModelInfo(id: id, name: nil)) + } + } + onEvent?(.models(models, selected: json["selected"]?.stringValue)) + return true + case "model_changed": + if let model = json["model"]?.stringValue { + onEvent?(.modelChanged(model)) + } + return true + default: + return false + } + } + + /// Inbound task_event handling (topic-agnostic per web UI). Returns true when consumed. + func handleTaskEvent(_ data: Data) -> Bool { + guard let json = try? decoder.decode([String: JSONValue].self, from: data), + let type = json["type"]?.stringValue, + ["created", "step", "completed", "failed"].contains(type) else { return false } + let payload = TaskEventPayload( + type: type, + taskId: json["task_id"]?.stringValue, + description: json["description"]?.stringValue, + role: json["role"]?.stringValue, + content: json["content"]?.stringValue, + result: json["result"]?.stringValue, + error: json["error"]?.stringValue + ) + onEvent?(.taskEvent(payload)) + return true + } + + // MARK: Outbound (topic'd JSON over reliable data streams) + + private func publish(_ object: [String: Any], topic: String, room: Room) async throws { + let text = String(data: try JSONSerialization.data(withJSONObject: object), encoding: .utf8) ?? "{}" + _ = try await room.localParticipant.sendText(text, for: topic) + } + + func setVoice(_ voiceID: String, room: Room) async throws { + try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room) + } + + func setModel(_ modelID: String, room: Room) async throws { + try await publish(["type": "set_model", "model": modelID], topic: "voice-control", room: room) + } + + func getModels(room: Room) async throws { + try await publish(["type": "get_models"], topic: "voice-control", room: room) + } + + func setMode(_ mode: String, room: Room) async throws { + try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room) + } + + func sendMediaState(_ state: String, trackID: String?, room: Room) async throws { + var object: [String: Any] = ["type": "media_state", "state": state] + if let trackID { object["track_id"] = trackID } + try await publish(object, topic: "media-control", room: room) + } +} + +// MARK: - Minimal JSON value decoder for flexible agent payloads + +indirect enum JSONValue: Decodable { + case string(String) + case number(Double) + case bool(Bool) + case object([String: JSONValue]) + case array([JSONValue]) + case null + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { self = .null } + else if let value = try? container.decode(Bool.self) { self = .bool(value) } + else if let value = try? container.decode(Double.self) { self = .number(value) } + else if let value = try? container.decode(String.self) { self = .string(value) } + else if let value = try? container.decode([JSONValue].self) { self = .array(value) } + else if let value = try? container.decode([String: JSONValue].self) { self = .object(value) } + else { throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported JSON value") } + } + + var stringValue: String? { + if case .string(let value) = self { return value } + return nil + } + + var objectValue: [String: JSONValue]? { + if case .object(let value) = self { return value } + return nil + } + + var arrayValue: [JSONValue]? { + if case .array(let value) = self { return value } + return nil + } +} diff --git a/Sources/LiveKit/TranscriptHandler.swift b/Sources/LiveKit/TranscriptHandler.swift new file mode 100644 index 0000000..6dc3540 --- /dev/null +++ b/Sources/LiveKit/TranscriptHandler.swift @@ -0,0 +1,68 @@ +import Foundation +import LiveKit + +struct TranscriptEvent: Equatable, Identifiable { + enum Role: String { + case user + case agent + } + + let id: String + let role: Role + var text: String + var isFinal: Bool + let timestamp: Date +} + +/// Merges the primary transcription segments (Room delegate) with the +/// secondary `transcript` data topic (finals only). Dedupe rule: a final for a +/// role closes any open partial row for that role. +@MainActor +final class TranscriptHandler: ObservableObject { + @Published private(set) var events: [TranscriptEvent] = [] + + func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) { + guard !text.isEmpty else { return } + upsert(id: id, role: role, text: text, isFinal: isFinal) + } + + func receiveDataTranscript(role: TranscriptEvent.Role, text: String) { + closePartial(for: role) + upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true) + } + + func clear() { + events.removeAll() + } + + private func closePartial(for role: TranscriptEvent.Role) { + for index in events.indices where !events[index].isFinal && events[index].role == role { + events[index].isFinal = true + } + } + + private func upsert(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) { + if let index = events.firstIndex(where: { $0.id == id }) { + events[index].text = text + events[index].isFinal = isFinal + } else { + events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: Date())) + } + } +} + +extension TranscriptHandler: RoomDelegate { + nonisolated func room(_ room: Room, participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) { + let role: TranscriptEvent.Role = participant.isAgent ? .agent : .user + let isLocal = participant.identity?.stringValue == room.localParticipant.identity?.stringValue + let resolvedRole: TranscriptEvent.Role = isLocal ? .user : role + let events = segments.map { + TranscriptEvent(id: $0.id, role: resolvedRole, text: $0.text, isFinal: $0.isFinal, timestamp: $0.lastReceivedTime) + } + Task { @MainActor [weak self] in + for event in events { + self?.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal) + } + } + } +} diff --git a/Sources/LiveKit/VoiceSession.swift b/Sources/LiveKit/VoiceSession.swift new file mode 100644 index 0000000..e4ac618 --- /dev/null +++ b/Sources/LiveKit/VoiceSession.swift @@ -0,0 +1,231 @@ +import Foundation +import LiveKit +import os + +private let logger = Logger(subsystem: "dev.shane.voice", category: "session") + +enum VoiceSessionState: Equatable { + case idle + case connecting + case waitingAgent + case ready + case failed(String) +} + +struct AgentInfo: Equatable { + let identity: String + let name: String? +} + +/// Room lifecycle: connect, agent discovery, agent_ready wait, reconnect-once, +/// disconnect teardown. Ports the web UI's connect flow. +@MainActor +final class VoiceSession: ObservableObject { + @Published private(set) var state: VoiceSessionState = .idle + @Published private(set) var agent: AgentInfo? + @Published private(set) var isMuted = false + + private(set) var room = Room() + let controlChannel = ControlChannel() + let transcriptHandler = TranscriptHandler() + private let config: ServerConfig + private let tokenClient: TokenClient + private var reconnectAttempted = false + private var connectAttempts = 0 + private var agentWaitTask: Task? + + init(config: ServerConfig, tokenClient: TokenClient) { + self.config = config + self.tokenClient = tokenClient + room.add(delegate: self) + room.add(delegate: controlChannel) + room.add(delegate: transcriptHandler) + controlChannel.onEvent = { [weak self] _ in + guard self != nil else { return } + } + } + + var isReady: Bool { state == .ready } + + func connect() async { + await start() + } + + private func makeRoom() -> Room { + let room = Room() + room.add(delegate: self) + room.add(delegate: controlChannel) + room.add(delegate: transcriptHandler) + return room + } + + private func start() async { + state = .connecting + connectAttempts = 0 + await attemptConnect() + } + + private func attemptConnect() async { + let result: Result + do { + let token = try await tokenClient.fetchToken() + logger.log("token ok room=\(token.room, privacy: .public)") + try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions()) + result = .success(()) + } catch { + result = .failure(error) + } + switch result { + case .success: + logger.log("room connected") + scanForAgent() + startAgentWait() + case .failure(let error): + let isTimeout = (error as? LiveKitError)?.type == .timedOut + if isTimeout, connectAttempts < 2 { + connectAttempts += 1 + let attempt = connectAttempts + logger.log("connect attempt \(attempt) timed out, retrying with fresh room") + await teardownRoom() + await attemptConnect() + return + } + logger.error("connect failed: \(String(describing: error), privacy: .public)") + state = .failed(error.localizedDescription) + } + } + + private func makeConnectOptions() -> ConnectOptions { + return ConnectOptions( + socketConnectTimeoutInterval: 15, + primaryTransportConnectTimeout: 20, + publisherTransportConnectTimeout: 20 + ) + } + + private func teardownRoom() async { + await room.disconnect() + room = makeRoom() + } + + private func handleAgentTimeout() async { + guard state == .waitingAgent, !reconnectAttempted else { + if state == .waitingAgent { + state = .failed("Agent never joined. Is the agent worker running?") + } + return + } + reconnectAttempted = true + await teardownRoom() + await start() + } + + private func startAgentWait() { + state = .waitingAgent + agentWaitTask?.cancel() + agentWaitTask = Task { [weak self] in + try? await Task.sleep(for: .seconds(10)) + guard !Task.isCancelled else { return } + await self?.handleAgentTimeout() + } + } + + func agentReadyReceived() { + guard state == .waitingAgent else { + logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)") + return + } + agentWaitTask?.cancel() + state = .ready + logger.log("state=ready, enabling mic") + Task { await enableMicrophone() } + } + + private func scanForAgent() { + for participant in room.remoteParticipants.values { + if isAgentParticipant(participant) { + foundAgent(participant) + return + } + } + } + + private func isAgentParticipant(_ participant: RemoteParticipant) -> Bool { + participant.kind == .agent || participant.identity?.stringValue == "voice-assistant" + } + + private func foundAgent(_ participant: RemoteParticipant) { + agent = AgentInfo(identity: participant.identity?.stringValue ?? "", name: participant.name) + } + + func enableMicrophone() async { + for attempt in 0..<3 { + do { + try await room.localParticipant.setMicrophone(enabled: true) + isMuted = false + logger.log("mic enabled") + return + } catch { + let message = String(describing: error) + if attempt < 2 { + logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s") + try? await Task.sleep(for: .seconds(1)) + continue + } + logger.error("mic enable failed after retries: \(message, privacy: .public)") + state = .failed("Microphone error: \(error.localizedDescription)") + } + } + } + + func setMuted(_ muted: Bool) async { + guard isReady else { return } + do { + try await room.localParticipant.setMicrophone(enabled: !muted) + isMuted = muted + } catch { + state = .failed("Microphone error: \(error.localizedDescription)") + } + } + + func sendText(_ text: String) async throws { + try await room.localParticipant.sendText(text, for: "lk.chat") + } + + func disconnect() async { + agentWaitTask?.cancel() + state = .idle + agent = nil + isMuted = false + reconnectAttempted = false + await teardownRoom() + } + +} + +extension VoiceSession: RoomDelegate { + nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) { + Task { @MainActor [weak self] in + guard let self, self.isAgentParticipant(participant) else { return } + self.foundAgent(participant) + } + } + + nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) { + logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "", privacy: .public)") + Task { @MainActor [weak self] in + guard let self else { return } + // Secondary transcript source (finals only). + if topic == "transcript", + let json = try? JSONDecoder().decode([String: JSONValue].self, from: data), + json["type"]?.stringValue == "transcript", + let text = json["text"]?.stringValue { + let role: TranscriptEvent.Role = json["role"]?.stringValue == "agent" ? .agent : .user + self.transcriptHandler.receiveDataTranscript(role: role, text: text) + return + } + if self.controlChannel.handleTaskEvent(data) { return } + _ = self.controlChannel.handleVoiceControl(data, session: self) + } + } +} diff --git a/Sources/Networking/ServerConfig.swift b/Sources/Networking/ServerConfig.swift index 89ee797..dcdd771 100644 --- a/Sources/Networking/ServerConfig.swift +++ b/Sources/Networking/ServerConfig.swift @@ -33,10 +33,25 @@ struct ServerConfig: Equatable { c.path = "" c.fragment = nil guard let base = c.url else { return .failure(.invalidURL(raw)) } - let portSuffix = components.port.map { ":\($0)" } ?? "" - let livekit = URL(string: "wss://\(host)\(portSuffix)/livekit")! + // LAN deployment: LiveKit is reachable directly on :7880 (plain ws). + // Real domains use wss via same origin. The SDK derives /rtc from the base. + let livekit: URL + if Self.isPrivateIP(host) { + livekit = URL(string: "ws://\(host):7880")! + } else { + let portSuffix = components.port.map { ":\($0)" } ?? "" + livekit = URL(string: "wss://\(host)\(portSuffix)/livekit")! + } return .success(ServerConfig(baseURL: base, livekitURL: livekit)) } + static func isPrivateIP(_ host: String) -> Bool { + let parts = host.split(separator: ".").compactMap { UInt8($0) } + guard parts.count == 4 else { return false } + return parts[0] == 10 + || (parts[0] == 172 && (16...31).contains(parts[1])) + || (parts[0] == 192 && parts[1] == 168) + } + var host: String { baseURL.host ?? "" } } diff --git a/Sources/Networking/TLSPinning.swift b/Sources/Networking/TLSPinning.swift index f79313a..1f875c6 100644 --- a/Sources/Networking/TLSPinning.swift +++ b/Sources/Networking/TLSPinning.swift @@ -2,35 +2,52 @@ import Foundation import CommonCrypto import Security -/// ATS exception for the LAN host is declared in Info.plist. If the deployment -/// uses a self-signed certificate, pin its SHA-384 here (extract with: -/// `openssl s_client -connect 192.168.86.2:9178 /dev/null | openssl x509 -outform der | shasum -a 384`). -/// While empty, standard system trust is used (works for the current deployment). +/// Pins the self-signed cert of the LAN deployment (CN=192.168.86.2, regenerated +/// at container start — re-export the hash after server cert rotation, see README). +/// Extract with: +/// openssl s_client -connect 192.168.86.2:9178 /dev/null \ +/// | openssl x509 -outform der | openssl dgst -sha256 enum TLSPinning { - static let pinnedCertSHA384: [String] = [] + static let pinnedCertSHA256Hex: Set = [ + "07907abde3bb9c685265909927627d9ea4705d7e7b27056eefe06e10b192af46" + ] - static func validate(_ trust: SecTrust) -> Bool { - guard !pinnedCertSHA384.isEmpty else { return true } - var error: CFError? - guard SecTrustEvaluateWithError(trust, &error) else { return false } - let count = SecTrustGetCertificateCount(trust) - guard count > 0 else { return false } - guard let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate] else { - return false - } - for cert in chain { - guard let derData = SecCertificateCopyData(cert) as Data? else { continue } - if pinnedCertSHA384.contains(sha384Hex(derData)) { return true } - } - return false + static let host = "192.168.86.2" + + static func session() -> URLSession { + let config = URLSessionConfiguration.default + config.timeoutIntervalForRequest = 15 + return URLSession(configuration: config, delegate: PinnedTrustDelegate(), delegateQueue: nil) } - private static func sha384Hex(_ data: Data) -> String { - var hash = [UInt8](repeating: 0, count: Int(CC_SHA384_DIGEST_LENGTH)) + static func sha256Hex(_ data: Data) -> String { + var hash = [UInt8](repeating: 0, count: Int(CC_SHA256_DIGEST_LENGTH)) data.withUnsafeBytes { buffer in guard let base = buffer.baseAddress else { return } - CC_SHA384(base, CC_LONG(data.count), &hash) + CC_SHA256(base, CC_LONG(data.count), &hash) } return hash.map { String(format: "%02x", $0) }.joined() } + + static func matchesPinnedCert(_ trust: SecTrust) -> Bool { + guard let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate], + let leaf = chain.first, + let der = SecCertificateCopyData(leaf) as Data? else { return false } + return pinnedCertSHA256Hex.contains(sha256Hex(der)) + } +} + +final class PinnedTrustDelegate: NSObject, URLSessionDelegate { + func urlSession(_ session: URLSession, didReceive challenge: URLAuthenticationChallenge, completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void) { + guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, + let trust = challenge.protectionSpace.serverTrust else { + completionHandler(.performDefaultHandling, nil) + return + } + if TLSPinning.matchesPinnedCert(trust) { + completionHandler(.useCredential, URLCredential(trust: trust)) + } else { + completionHandler(.cancelAuthenticationChallenge, nil) + } + } } diff --git a/Sources/Networking/TokenClient.swift b/Sources/Networking/TokenClient.swift index 17693bf..90223a1 100644 --- a/Sources/Networking/TokenClient.swift +++ b/Sources/Networking/TokenClient.swift @@ -43,9 +43,9 @@ final class TokenClient { let config: ServerConfig private let session: URLSession - init(config: ServerConfig, session: URLSession = .shared) { + init(config: ServerConfig) { self.config = config - self.session = session + self.session = TLSPinning.session() } func fetchToken() async throws -> TokenResponse { diff --git a/Voice.xcodeproj/project.pbxproj b/Voice.xcodeproj/project.pbxproj index 84d21a7..99a7dc4 100644 --- a/Voice.xcodeproj/project.pbxproj +++ b/Voice.xcodeproj/project.pbxproj @@ -8,25 +8,35 @@ /* Begin PBXBuildFile section */ 1ECD63F10A8EAF33FD2A5B62 /* VoiceApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2FED8976B60FBC13BC980FCB /* VoiceApp.swift */; }; + 28659D009399CA9C4FE57B74 /* ControlChannel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61C71C48230C683D199D6F4B /* ControlChannel.swift */; }; 302BEC55CA9A10E49C307977 /* ServerConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = F5683993A1DE677121FFA1F3 /* ServerConfig.swift */; }; 36DF44D9A7896F18A064847D /* ConversationView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3F3FA9F420065D2127B0AEC0 /* ConversationView.swift */; }; + 3E01A0AF44FD3324FA72C43C /* OrbView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */; }; 720C26EC163AE46713B02DD8 /* TLSPinning.swift in Sources */ = {isa = PBXBuildFile; fileRef = D9B40076934C28BA4B92D808 /* TLSPinning.swift */; }; 8DE1363CC0119B5AF9E4E117 /* ConnectionTestView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC4B9073EE8EAB51F8C55760 /* ConnectionTestView.swift */; }; 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48F76881B4AE51DEE74511CC /* SettingsView.swift */; }; A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */; }; + C00D1878C2AB3D8B6F9562B6 /* AgentMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */; }; + CDE38FE9679472D770C601E6 /* VoiceSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */; }; D18AD347DFF72AB6361165BA /* TokenClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9681E7410B684D2D38752A70 /* TokenClient.swift */; }; + D2E343C8B8657316A993698E /* TranscriptHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */; }; D9A0059E342955A06A021D6C /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = A6BAD55715F383C0DB5F306F /* LiveKit */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgentMonitor.swift; sourceTree = ""; }; 2A4C3CE2CA6FF828452803D1 /* Voice.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = Voice.app; sourceTree = BUILT_PRODUCTS_DIR; }; 2FED8976B60FBC13BC980FCB /* VoiceApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceApp.swift; sourceTree = ""; }; 3F3FA9F420065D2127B0AEC0 /* ConversationView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConversationView.swift; sourceTree = ""; }; 48F76881B4AE51DEE74511CC /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; + 61C71C48230C683D199D6F4B /* ControlChannel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlChannel.swift; sourceTree = ""; }; 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsStore.swift; sourceTree = ""; }; 9681E7410B684D2D38752A70 /* TokenClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TokenClient.swift; sourceTree = ""; }; + A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OrbView.swift; sourceTree = ""; }; AC4B9073EE8EAB51F8C55760 /* ConnectionTestView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionTestView.swift; sourceTree = ""; }; + D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscriptHandler.swift; sourceTree = ""; }; D9B40076934C28BA4B92D808 /* TLSPinning.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TLSPinning.swift; sourceTree = ""; }; + EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSession.swift; sourceTree = ""; }; F5683993A1DE677121FFA1F3 /* ServerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerConfig.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -51,6 +61,17 @@ path = Settings; sourceTree = ""; }; + 2820A418F921E18A2FA7E4FB /* LiveKit */ = { + isa = PBXGroup; + children = ( + 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */, + 61C71C48230C683D199D6F4B /* ControlChannel.swift */, + D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */, + EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */, + ); + path = LiveKit; + sourceTree = ""; + }; 6B2AB34274D9B37E77178C6C /* Networking */ = { isa = PBXGroup; children = ( @@ -73,6 +94,7 @@ isa = PBXGroup; children = ( 3F3FA9F420065D2127B0AEC0 /* ConversationView.swift */, + A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */, ); path = Conversation; sourceTree = ""; @@ -108,6 +130,7 @@ children = ( CE49A38E40AAF659A9281B85 /* App */, 8DA8BEF2EE3E0F4181A94EB9 /* Features */, + 2820A418F921E18A2FA7E4FB /* LiveKit */, 6B2AB34274D9B37E77178C6C /* Networking */, ); path = Sources; @@ -176,14 +199,19 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + C00D1878C2AB3D8B6F9562B6 /* AgentMonitor.swift in Sources */, 8DE1363CC0119B5AF9E4E117 /* ConnectionTestView.swift in Sources */, + 28659D009399CA9C4FE57B74 /* ControlChannel.swift in Sources */, 36DF44D9A7896F18A064847D /* ConversationView.swift in Sources */, + 3E01A0AF44FD3324FA72C43C /* OrbView.swift in Sources */, 302BEC55CA9A10E49C307977 /* ServerConfig.swift in Sources */, A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */, 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */, 720C26EC163AE46713B02DD8 /* TLSPinning.swift in Sources */, D18AD347DFF72AB6361165BA /* TokenClient.swift in Sources */, + D2E343C8B8657316A993698E /* TranscriptHandler.swift in Sources */, 1ECD63F10A8EAF33FD2A5B62 /* VoiceApp.swift in Sources */, + CDE38FE9679472D770C601E6 /* VoiceSession.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/project.yml b/project.yml index 244f78e..6945e72 100644 --- a/project.yml +++ b/project.yml @@ -26,10 +26,12 @@ targets: UILaunchScreen: {} NSMicrophoneUsageDescription: "Hope Voice uses the microphone for live voice conversation." NSAppTransportSecurity: - NSExceptionDomains: + NSAllowsLocalNetworking: true + NSPinnedDomains: 192.168.86.2: NSIncludesSubdomains: true - NSExceptionAllowsInsecureHTTPLoads: false + NSPinnedCAIdentities: + - SPKI-SHA256-BASE64: vmrfRuHv1oHBBiEmJP4WX3C7mtVy3C4+UJ1Yk/Q5hMU= settings: base: PRODUCT_BUNDLE_IDENTIFIER: dev.shane.voice