feat: LiveKit voice session, control channel, transcripts (Phase 2)
- VoiceSession: token->connect with fresh-room retry on timeout, agent discovery, agent_ready wait, mic enable/mute, text send over lk.chat - ControlChannel: outbound set_voice/set_model/get_models/set_mode/ media_state; inbound agent_ready/models/model_changed/task_event - TranscriptHandler: transcription segments + transcript topic finals, partial/final dedupe per role - ConversationView/OrbView: status orb, mic/text-mode toggles, composer - ServerConfig: plain ws:// signaling for private IPs (self-signed TLS not injectable into SDK WebSocket); Info.plist local networking
This commit is contained in:
snowspeeder committed
1 parent
378b8cb36f
commit
4986b6d7dc
14 files changed
+900
-50
No files matched your search
@@ -1,9 +1,14 @@
|
||||
import SwiftUI
|
||||
import LiveKit
|
||||
|
||||
@main
|
||||
struct VoiceApp: App {
|
||||
@StateObject private var settings = SettingsStore()
|
||||
|
||||
init() {
|
||||
LiveKitSDK.setLogLevel(.debug)
|
||||
}
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ConversationView(settings: settings)
|
||||
|
||||
@@ -2,33 +2,161 @@ import SwiftUI
|
||||
|
||||
struct ConversationView: View {
|
||||
@ObservedObject var settings: SettingsStore
|
||||
@StateObject private var session: VoiceSession
|
||||
@State private var composerText = ""
|
||||
@State private var connectError: String?
|
||||
|
||||
init(settings: SettingsStore) {
|
||||
self.settings = settings
|
||||
let config = ServerConfig.parse(settings.trimmedServerURL)
|
||||
if case .success(let value) = config {
|
||||
_session = StateObject(wrappedValue: VoiceSession(
|
||||
config: value,
|
||||
tokenClient: TokenClient(config: value)
|
||||
))
|
||||
} else {
|
||||
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
|
||||
if case .success(let value) = fallback {
|
||||
_session = StateObject(wrappedValue: VoiceSession(
|
||||
config: value,
|
||||
tokenClient: TokenClient(config: value)
|
||||
))
|
||||
} else {
|
||||
fatalError("Unreachable: default server URL is valid")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
VStack(spacing: 24) {
|
||||
Spacer()
|
||||
Circle()
|
||||
.fill(.blue.gradient)
|
||||
.frame(width: 96, height: 96)
|
||||
Text("Hope Voice")
|
||||
.font(.title.bold())
|
||||
Text(settings.trimmedServerURL)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
Spacer()
|
||||
VStack(spacing: 0) {
|
||||
statusHeader
|
||||
TranscriptListView(events: session.transcriptHandler.events)
|
||||
composer
|
||||
}
|
||||
.navigationTitle("Voice")
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
NavigationLink {
|
||||
SettingsView(settings: settings)
|
||||
} label: {
|
||||
Image(systemName: "gearshape")
|
||||
}
|
||||
.navigationTitle("Hope Voice")
|
||||
.toolbar { toolbarContent }
|
||||
.onAppear {
|
||||
if case .failure(let error) = ServerConfig.parse(settings.trimmedServerURL) {
|
||||
connectError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var statusHeader: some View {
|
||||
VStack(spacing: 12) {
|
||||
OrbView(state: orbState)
|
||||
Text(statusText)
|
||||
.font(.headline)
|
||||
if let agent = session.agent {
|
||||
Text("Agent: \(agent.name ?? agent.identity)")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
if session.state == .ready {
|
||||
HStack(spacing: 16) {
|
||||
Toggle(isOn: Binding(
|
||||
get: { !session.isMuted },
|
||||
set: { newValue in Task { await session.setMuted(!newValue) } }
|
||||
)) {
|
||||
Label(session.isMuted ? "Muted" : "Mic on", systemImage: session.isMuted ? "mic.slash" : "mic")
|
||||
}
|
||||
.toggleStyle(.button)
|
||||
Toggle("Text mode", isOn: $settings.textMode)
|
||||
.toggleStyle(.button)
|
||||
}
|
||||
}
|
||||
if let connectError {
|
||||
Label(connectError, systemImage: "exclamationmark.triangle")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.red)
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
}
|
||||
|
||||
private var composer: some View {
|
||||
HStack(spacing: 8) {
|
||||
switch session.state {
|
||||
case .idle, .failed:
|
||||
Button(action: startSession) {
|
||||
Label("Start", systemImage: "phone.fill")
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
case .connecting, .waitingAgent:
|
||||
ProgressView()
|
||||
case .ready:
|
||||
TextField(settings.textMode ? "Type a message…" : "Voice mode active",
|
||||
text: $composerText,
|
||||
axis: .vertical)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.lineLimit(1...4)
|
||||
.disabled(!settings.textMode)
|
||||
Button(action: sendComposer) {
|
||||
Image(systemName: "arrow.up.circle.fill")
|
||||
.font(.title2)
|
||||
}
|
||||
.disabled(!settings.textMode || composerText.isEmpty)
|
||||
Button(role: .destructive, action: endSession) {
|
||||
Image(systemName: "phone.down.fill")
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
}
|
||||
|
||||
private var toolbarContent: some ToolbarContent {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
NavigationLink {
|
||||
SettingsView(settings: settings)
|
||||
} label: {
|
||||
Image(systemName: "gearshape")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var orbState: OrbView.OrbState {
|
||||
switch session.state {
|
||||
case .idle: .idle
|
||||
case .connecting, .waitingAgent: .connecting
|
||||
case .ready: settings.textMode ? .idle : .listening
|
||||
case .failed: .failed
|
||||
}
|
||||
}
|
||||
|
||||
private var statusText: String {
|
||||
switch session.state {
|
||||
case .idle: "Ready to connect"
|
||||
case .connecting: "Connecting…"
|
||||
case .waitingAgent: "Waiting for agent…"
|
||||
case .ready: settings.textMode ? "Text mode" : "Listening"
|
||||
case .failed(let message): "Failed: \(message)"
|
||||
}
|
||||
}
|
||||
|
||||
private func startSession() {
|
||||
connectError = nil
|
||||
session.transcriptHandler.clear()
|
||||
Task { await session.connect() }
|
||||
}
|
||||
|
||||
private func endSession() {
|
||||
Task { await session.disconnect() }
|
||||
}
|
||||
|
||||
private func sendComposer() {
|
||||
let text = composerText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty else { return }
|
||||
composerText = ""
|
||||
Task {
|
||||
do {
|
||||
try await session.sendText(text)
|
||||
} catch {
|
||||
connectError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#Preview {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import SwiftUI
|
||||
|
||||
struct OrbView: View {
|
||||
enum OrbState {
|
||||
case idle
|
||||
case connecting
|
||||
case listening
|
||||
case speaking
|
||||
case thinking
|
||||
case failed
|
||||
}
|
||||
|
||||
let state: OrbState
|
||||
|
||||
@State private var animating = false
|
||||
|
||||
private var baseColor: Color {
|
||||
switch state {
|
||||
case .idle, .connecting: .blue
|
||||
case .listening: .green
|
||||
case .speaking, .thinking: .orange
|
||||
case .failed: .red
|
||||
}
|
||||
}
|
||||
|
||||
private var scale: CGFloat {
|
||||
switch state {
|
||||
case .connecting, .listening, .speaking, .thinking: 1.1
|
||||
default: 1.0
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
let shape = Circle()
|
||||
.fill(baseColor.gradient)
|
||||
.frame(width: 120, height: 120)
|
||||
let animated = shape
|
||||
.scaleEffect(animating ? scale : 0.95)
|
||||
.opacity(state == .idle || state == .failed ? 0.7 : 1.0)
|
||||
let anim: Animation? = state == .idle || state == .failed
|
||||
? nil
|
||||
: .easeInOut(duration: 0.8).repeatForever(autoreverses: true)
|
||||
animated
|
||||
.animation(anim, value: animating)
|
||||
.onAppear { animating = true }
|
||||
.onChange(of: state) { _, _ in animating = true }
|
||||
}
|
||||
}
|
||||
|
||||
struct TranscriptListView: View {
|
||||
let events: [TranscriptEvent]
|
||||
|
||||
var body: some View {
|
||||
ScrollViewReader { proxy in
|
||||
ScrollView {
|
||||
LazyVStack(alignment: .leading, spacing: 8) {
|
||||
ForEach(events) { event in
|
||||
transcriptRow(event)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
.onChange(of: events.count) { _, count in
|
||||
if let last = events.last {
|
||||
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func transcriptRow(_ event: TranscriptEvent) -> some View {
|
||||
let roleName = event.role == .user ? "You" : "Hope"
|
||||
let nameColor: Color = event.role == .user ? .secondary : .blue
|
||||
let rowBackground: Color = event.role == .user
|
||||
? Color(.secondarySystemBackground)
|
||||
: Color.blue.opacity(0.08)
|
||||
let textOpacity: Double = event.isFinal ? 1.0 : 0.55
|
||||
return VStack(alignment: .leading, spacing: 2) {
|
||||
Text(roleName)
|
||||
.font(.caption2.bold())
|
||||
.foregroundStyle(nameColor)
|
||||
Text(stripped(event.text))
|
||||
.font(.body)
|
||||
.opacity(textOpacity)
|
||||
}
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.padding(8)
|
||||
.background(rowBackground)
|
||||
.cornerRadius(10)
|
||||
.id(event.id)
|
||||
}
|
||||
|
||||
private func stripped(_ text: String) -> String {
|
||||
// Strip delivery-style markers like [appreciative], [pauses] etc.
|
||||
text.replacingOccurrences(
|
||||
of: #"\[[a-zA-Z ,'-]+\]"#,
|
||||
with: "",
|
||||
options: .regularExpression
|
||||
).trimmingCharacters(in: .whitespaces)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Observes the agent participant's audio level for UI state inference.
|
||||
@MainActor
|
||||
final class AgentMonitor: ObservableObject {
|
||||
@Published private(set) var agentSpeaking = false
|
||||
@Published private(set) var localSpeaking = false
|
||||
|
||||
private let session: VoiceSession
|
||||
private var pollTask: Task<Void, Never>?
|
||||
|
||||
init(session: VoiceSession) {
|
||||
self.session = session
|
||||
}
|
||||
|
||||
func start() {
|
||||
stop()
|
||||
pollTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self else { return }
|
||||
let remote = self.session.room.remoteParticipants.values.first(where: {
|
||||
$0.kind == .agent || $0.identity?.stringValue == "voice-assistant"
|
||||
})
|
||||
self.agentSpeaking = remote?.isSpeaking ?? false
|
||||
self.localSpeaking = self.session.room.localParticipant.isSpeaking
|
||||
try? await Task.sleep(for: .milliseconds(250))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
pollTask?.cancel()
|
||||
pollTask = nil
|
||||
agentSpeaking = false
|
||||
localSpeaking = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Outbound helpers for agent control topics and inbound parsing of
|
||||
/// voice-control / conversation-control / media-control data messages.
|
||||
@MainActor
|
||||
final class ControlChannel: NSObject, RoomDelegate {
|
||||
|
||||
struct ModelInfo: Equatable {
|
||||
let id: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
enum InboundEvent {
|
||||
case agentReady
|
||||
case models([ModelInfo], selected: String?)
|
||||
case modelChanged(String)
|
||||
case taskEvent(TaskEventPayload)
|
||||
}
|
||||
|
||||
struct TaskEventPayload: Equatable {
|
||||
let type: String
|
||||
let taskId: String?
|
||||
let description: String?
|
||||
let role: String?
|
||||
let content: String?
|
||||
let result: String?
|
||||
let error: String?
|
||||
}
|
||||
|
||||
var onEvent: ((InboundEvent) -> Void)?
|
||||
|
||||
private let decoder = JSONDecoder()
|
||||
|
||||
/// Inbound voice-control handling. Returns true when the message was consumed.
|
||||
func handleVoiceControl(_ data: Data, session: VoiceSession) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
let type = json["type"]?.stringValue else { return false }
|
||||
switch type {
|
||||
case "agent_ready":
|
||||
Task { @MainActor [weak session] in
|
||||
session?.agentReadyReceived()
|
||||
}
|
||||
return true
|
||||
case "models":
|
||||
var models: [ModelInfo] = []
|
||||
for item in json["models"]?.arrayValue ?? [] {
|
||||
if let obj = item.objectValue, let id = obj["id"]?.stringValue {
|
||||
models.append(ModelInfo(id: id, name: obj["name"]?.stringValue))
|
||||
} else if let id = item.stringValue {
|
||||
models.append(ModelInfo(id: id, name: nil))
|
||||
}
|
||||
}
|
||||
onEvent?(.models(models, selected: json["selected"]?.stringValue))
|
||||
return true
|
||||
case "model_changed":
|
||||
if let model = json["model"]?.stringValue {
|
||||
onEvent?(.modelChanged(model))
|
||||
}
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// Inbound task_event handling (topic-agnostic per web UI). Returns true when consumed.
|
||||
func handleTaskEvent(_ data: Data) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
let type = json["type"]?.stringValue,
|
||||
["created", "step", "completed", "failed"].contains(type) else { return false }
|
||||
let payload = TaskEventPayload(
|
||||
type: type,
|
||||
taskId: json["task_id"]?.stringValue,
|
||||
description: json["description"]?.stringValue,
|
||||
role: json["role"]?.stringValue,
|
||||
content: json["content"]?.stringValue,
|
||||
result: json["result"]?.stringValue,
|
||||
error: json["error"]?.stringValue
|
||||
)
|
||||
onEvent?(.taskEvent(payload))
|
||||
return true
|
||||
}
|
||||
|
||||
// MARK: Outbound (topic'd JSON over reliable data streams)
|
||||
|
||||
private func publish(_ object: [String: Any], topic: String, room: Room) async throws {
|
||||
let text = String(data: try JSONSerialization.data(withJSONObject: object), encoding: .utf8) ?? "{}"
|
||||
_ = try await room.localParticipant.sendText(text, for: topic)
|
||||
}
|
||||
|
||||
func setVoice(_ voiceID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setModel(_ modelID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_model", "model": modelID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func getModels(room: Room) async throws {
|
||||
try await publish(["type": "get_models"], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setMode(_ mode: String, room: Room) async throws {
|
||||
try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room)
|
||||
}
|
||||
|
||||
func sendMediaState(_ state: String, trackID: String?, room: Room) async throws {
|
||||
var object: [String: Any] = ["type": "media_state", "state": state]
|
||||
if let trackID { object["track_id"] = trackID }
|
||||
try await publish(object, topic: "media-control", room: room)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Minimal JSON value decoder for flexible agent payloads
|
||||
|
||||
indirect enum JSONValue: Decodable {
|
||||
case string(String)
|
||||
case number(Double)
|
||||
case bool(Bool)
|
||||
case object([String: JSONValue])
|
||||
case array([JSONValue])
|
||||
case null
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.singleValueContainer()
|
||||
if container.decodeNil() { self = .null }
|
||||
else if let value = try? container.decode(Bool.self) { self = .bool(value) }
|
||||
else if let value = try? container.decode(Double.self) { self = .number(value) }
|
||||
else if let value = try? container.decode(String.self) { self = .string(value) }
|
||||
else if let value = try? container.decode([JSONValue].self) { self = .array(value) }
|
||||
else if let value = try? container.decode([String: JSONValue].self) { self = .object(value) }
|
||||
else { throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported JSON value") }
|
||||
}
|
||||
|
||||
var stringValue: String? {
|
||||
if case .string(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var objectValue: [String: JSONValue]? {
|
||||
if case .object(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var arrayValue: [JSONValue]? {
|
||||
if case .array(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
struct TranscriptEvent: Equatable, Identifiable {
|
||||
enum Role: String {
|
||||
case user
|
||||
case agent
|
||||
}
|
||||
|
||||
let id: String
|
||||
let role: Role
|
||||
var text: String
|
||||
var isFinal: Bool
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Merges the primary transcription segments (Room delegate) with the
|
||||
/// secondary `transcript` data topic (finals only). Dedupe rule: a final for a
|
||||
/// role closes any open partial row for that role.
|
||||
@MainActor
|
||||
final class TranscriptHandler: ObservableObject {
|
||||
@Published private(set) var events: [TranscriptEvent] = []
|
||||
|
||||
func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
guard !text.isEmpty else { return }
|
||||
upsert(id: id, role: role, text: text, isFinal: isFinal)
|
||||
}
|
||||
|
||||
func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
|
||||
closePartial(for: role)
|
||||
upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
|
||||
}
|
||||
|
||||
func clear() {
|
||||
events.removeAll()
|
||||
}
|
||||
|
||||
private func closePartial(for role: TranscriptEvent.Role) {
|
||||
for index in events.indices where !events[index].isFinal && events[index].role == role {
|
||||
events[index].isFinal = true
|
||||
}
|
||||
}
|
||||
|
||||
private func upsert(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
if let index = events.firstIndex(where: { $0.id == id }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = isFinal
|
||||
} else {
|
||||
events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: Date()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension TranscriptHandler: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) {
|
||||
let role: TranscriptEvent.Role = participant.isAgent ? .agent : .user
|
||||
let isLocal = participant.identity?.stringValue == room.localParticipant.identity?.stringValue
|
||||
let resolvedRole: TranscriptEvent.Role = isLocal ? .user : role
|
||||
let events = segments.map {
|
||||
TranscriptEvent(id: $0.id, role: resolvedRole, text: $0.text, isFinal: $0.isFinal, timestamp: $0.lastReceivedTime)
|
||||
}
|
||||
Task { @MainActor [weak self] in
|
||||
for event in events {
|
||||
self?.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
import os
|
||||
|
||||
private let logger = Logger(subsystem: "dev.shane.voice", category: "session")
|
||||
|
||||
enum VoiceSessionState: Equatable {
|
||||
case idle
|
||||
case connecting
|
||||
case waitingAgent
|
||||
case ready
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct AgentInfo: Equatable {
|
||||
let identity: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
/// Room lifecycle: connect, agent discovery, agent_ready wait, reconnect-once,
|
||||
/// disconnect teardown. Ports the web UI's connect flow.
|
||||
@MainActor
|
||||
final class VoiceSession: ObservableObject {
|
||||
@Published private(set) var state: VoiceSessionState = .idle
|
||||
@Published private(set) var agent: AgentInfo?
|
||||
@Published private(set) var isMuted = false
|
||||
|
||||
private(set) var room = Room()
|
||||
let controlChannel = ControlChannel()
|
||||
let transcriptHandler = TranscriptHandler()
|
||||
private let config: ServerConfig
|
||||
private let tokenClient: TokenClient
|
||||
private var reconnectAttempted = false
|
||||
private var connectAttempts = 0
|
||||
private var agentWaitTask: Task<Void, Never>?
|
||||
|
||||
init(config: ServerConfig, tokenClient: TokenClient) {
|
||||
self.config = config
|
||||
self.tokenClient = tokenClient
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
controlChannel.onEvent = { [weak self] _ in
|
||||
guard self != nil else { return }
|
||||
}
|
||||
}
|
||||
|
||||
var isReady: Bool { state == .ready }
|
||||
|
||||
func connect() async {
|
||||
await start()
|
||||
}
|
||||
|
||||
private func makeRoom() -> Room {
|
||||
let room = Room()
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
return room
|
||||
}
|
||||
|
||||
private func start() async {
|
||||
state = .connecting
|
||||
connectAttempts = 0
|
||||
await attemptConnect()
|
||||
}
|
||||
|
||||
private func attemptConnect() async {
|
||||
let result: Result<Void, Error>
|
||||
do {
|
||||
let token = try await tokenClient.fetchToken()
|
||||
logger.log("token ok room=\(token.room, privacy: .public)")
|
||||
try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions())
|
||||
result = .success(())
|
||||
} catch {
|
||||
result = .failure(error)
|
||||
}
|
||||
switch result {
|
||||
case .success:
|
||||
logger.log("room connected")
|
||||
scanForAgent()
|
||||
startAgentWait()
|
||||
case .failure(let error):
|
||||
let isTimeout = (error as? LiveKitError)?.type == .timedOut
|
||||
if isTimeout, connectAttempts < 2 {
|
||||
connectAttempts += 1
|
||||
let attempt = connectAttempts
|
||||
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
||||
await teardownRoom()
|
||||
await attemptConnect()
|
||||
return
|
||||
}
|
||||
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
||||
state = .failed(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func makeConnectOptions() -> ConnectOptions {
|
||||
return ConnectOptions(
|
||||
socketConnectTimeoutInterval: 15,
|
||||
primaryTransportConnectTimeout: 20,
|
||||
publisherTransportConnectTimeout: 20
|
||||
)
|
||||
}
|
||||
|
||||
private func teardownRoom() async {
|
||||
await room.disconnect()
|
||||
room = makeRoom()
|
||||
}
|
||||
|
||||
private func handleAgentTimeout() async {
|
||||
guard state == .waitingAgent, !reconnectAttempted else {
|
||||
if state == .waitingAgent {
|
||||
state = .failed("Agent never joined. Is the agent worker running?")
|
||||
}
|
||||
return
|
||||
}
|
||||
reconnectAttempted = true
|
||||
await teardownRoom()
|
||||
await start()
|
||||
}
|
||||
|
||||
private func startAgentWait() {
|
||||
state = .waitingAgent
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = Task { [weak self] in
|
||||
try? await Task.sleep(for: .seconds(10))
|
||||
guard !Task.isCancelled else { return }
|
||||
await self?.handleAgentTimeout()
|
||||
}
|
||||
}
|
||||
|
||||
func agentReadyReceived() {
|
||||
guard state == .waitingAgent else {
|
||||
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
||||
return
|
||||
}
|
||||
agentWaitTask?.cancel()
|
||||
state = .ready
|
||||
logger.log("state=ready, enabling mic")
|
||||
Task { await enableMicrophone() }
|
||||
}
|
||||
|
||||
private func scanForAgent() {
|
||||
for participant in room.remoteParticipants.values {
|
||||
if isAgentParticipant(participant) {
|
||||
foundAgent(participant)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func isAgentParticipant(_ participant: RemoteParticipant) -> Bool {
|
||||
participant.kind == .agent || participant.identity?.stringValue == "voice-assistant"
|
||||
}
|
||||
|
||||
private func foundAgent(_ participant: RemoteParticipant) {
|
||||
agent = AgentInfo(identity: participant.identity?.stringValue ?? "", name: participant.name)
|
||||
}
|
||||
|
||||
func enableMicrophone() async {
|
||||
for attempt in 0..<3 {
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: true)
|
||||
isMuted = false
|
||||
logger.log("mic enabled")
|
||||
return
|
||||
} catch {
|
||||
let message = String(describing: error)
|
||||
if attempt < 2 {
|
||||
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
||||
try? await Task.sleep(for: .seconds(1))
|
||||
continue
|
||||
}
|
||||
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
||||
state = .failed("Microphone error: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setMuted(_ muted: Bool) async {
|
||||
guard isReady else { return }
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||
isMuted = muted
|
||||
} catch {
|
||||
state = .failed("Microphone error: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
func sendText(_ text: String) async throws {
|
||||
try await room.localParticipant.sendText(text, for: "lk.chat")
|
||||
}
|
||||
|
||||
func disconnect() async {
|
||||
agentWaitTask?.cancel()
|
||||
state = .idle
|
||||
agent = nil
|
||||
isMuted = false
|
||||
reconnectAttempted = false
|
||||
await teardownRoom()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
extension VoiceSession: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.isAgentParticipant(participant) else { return }
|
||||
self.foundAgent(participant)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
||||
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
// Secondary transcript source (finals only).
|
||||
if topic == "transcript",
|
||||
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
||||
json["type"]?.stringValue == "transcript",
|
||||
let text = json["text"]?.stringValue {
|
||||
let role: TranscriptEvent.Role = json["role"]?.stringValue == "agent" ? .agent : .user
|
||||
self.transcriptHandler.receiveDataTranscript(role: role, text: text)
|
||||
return
|
||||
}
|
||||
if self.controlChannel.handleTaskEvent(data) { return }
|
||||
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,10 +33,25 @@ struct ServerConfig: Equatable {
|
||||
c.path = ""
|
||||
c.fragment = nil
|
||||
guard let base = c.url else { return .failure(.invalidURL(raw)) }
|
||||
let portSuffix = components.port.map { ":\($0)" } ?? ""
|
||||
let livekit = URL(string: "wss://\(host)\(portSuffix)/livekit")!
|
||||
// LAN deployment: LiveKit is reachable directly on :7880 (plain ws).
|
||||
// Real domains use wss via same origin. The SDK derives /rtc from the base.
|
||||
let livekit: URL
|
||||
if Self.isPrivateIP(host) {
|
||||
livekit = URL(string: "ws://\(host):7880")!
|
||||
} else {
|
||||
let portSuffix = components.port.map { ":\($0)" } ?? ""
|
||||
livekit = URL(string: "wss://\(host)\(portSuffix)/livekit")!
|
||||
}
|
||||
return .success(ServerConfig(baseURL: base, livekitURL: livekit))
|
||||
}
|
||||
|
||||
static func isPrivateIP(_ host: String) -> Bool {
|
||||
let parts = host.split(separator: ".").compactMap { UInt8($0) }
|
||||
guard parts.count == 4 else { return false }
|
||||
return parts[0] == 10
|
||||
|| (parts[0] == 172 && (16...31).contains(parts[1]))
|
||||
|| (parts[0] == 192 && parts[1] == 168)
|
||||
}
|
||||
|
||||
var host: String { baseURL.host ?? "" }
|
||||
}
|
||||
@@ -2,35 +2,52 @@ import Foundation
|
||||
import CommonCrypto
|
||||
import Security
|
||||
|
||||
/// ATS exception for the LAN host is declared in Info.plist. If the deployment
|
||||
/// uses a self-signed certificate, pin its SHA-384 here (extract with:
|
||||
/// `openssl s_client -connect 192.168.86.2:9178 </dev/null 2>/dev/null | openssl x509 -outform der | shasum -a 384`).
|
||||
/// While empty, standard system trust is used (works for the current deployment).
|
||||
/// Pins the self-signed cert of the LAN deployment (CN=192.168.86.2, regenerated
|
||||
/// at container start — re-export the hash after server cert rotation, see README).
|
||||
/// Extract with:
|
||||
/// openssl s_client -connect 192.168.86.2:9178 </dev/null 2>/dev/null \
|
||||
/// | openssl x509 -outform der | openssl dgst -sha256
|
||||
enum TLSPinning {
|
||||
static let pinnedCertSHA384: [String] = []
|
||||
static let pinnedCertSHA256Hex: Set<String> = [
|
||||
"07907abde3bb9c685265909927627d9ea4705d7e7b27056eefe06e10b192af46"
|
||||
]
|
||||
|
||||
static func validate(_ trust: SecTrust) -> Bool {
|
||||
guard !pinnedCertSHA384.isEmpty else { return true }
|
||||
var error: CFError?
|
||||
guard SecTrustEvaluateWithError(trust, &error) else { return false }
|
||||
let count = SecTrustGetCertificateCount(trust)
|
||||
guard count > 0 else { return false }
|
||||
guard let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate] else {
|
||||
return false
|
||||
}
|
||||
for cert in chain {
|
||||
guard let derData = SecCertificateCopyData(cert) as Data? else { continue }
|
||||
if pinnedCertSHA384.contains(sha384Hex(derData)) { return true }
|
||||
}
|
||||
return false
|
||||
static let host = "192.168.86.2"
|
||||
|
||||
static func session() -> URLSession {
|
||||
let config = URLSessionConfiguration.default
|
||||
config.timeoutIntervalForRequest = 15
|
||||
return URLSession(configuration: config, delegate: PinnedTrustDelegate(), delegateQueue: nil)
|
||||
}
|
||||
|
||||
private static func sha384Hex(_ data: Data) -> String {
|
||||
var hash = [UInt8](repeating: 0, count: Int(CC_SHA384_DIGEST_LENGTH))
|
||||
static func sha256Hex(_ data: Data) -> String {
|
||||
var hash = [UInt8](repeating: 0, count: Int(CC_SHA256_DIGEST_LENGTH))
|
||||
data.withUnsafeBytes { buffer in
|
||||
guard let base = buffer.baseAddress else { return }
|
||||
CC_SHA384(base, CC_LONG(data.count), &hash)
|
||||
CC_SHA256(base, CC_LONG(data.count), &hash)
|
||||
}
|
||||
return hash.map { String(format: "%02x", $0) }.joined()
|
||||
}
|
||||
|
||||
static func matchesPinnedCert(_ trust: SecTrust) -> Bool {
|
||||
guard let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate],
|
||||
let leaf = chain.first,
|
||||
let der = SecCertificateCopyData(leaf) as Data? else { return false }
|
||||
return pinnedCertSHA256Hex.contains(sha256Hex(der))
|
||||
}
|
||||
}
|
||||
|
||||
final class PinnedTrustDelegate: NSObject, URLSessionDelegate {
|
||||
func urlSession(_ session: URLSession, didReceive challenge: URLAuthenticationChallenge, completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void) {
|
||||
guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust,
|
||||
let trust = challenge.protectionSpace.serverTrust else {
|
||||
completionHandler(.performDefaultHandling, nil)
|
||||
return
|
||||
}
|
||||
if TLSPinning.matchesPinnedCert(trust) {
|
||||
completionHandler(.useCredential, URLCredential(trust: trust))
|
||||
} else {
|
||||
completionHandler(.cancelAuthenticationChallenge, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,9 +43,9 @@ final class TokenClient {
|
||||
let config: ServerConfig
|
||||
private let session: URLSession
|
||||
|
||||
init(config: ServerConfig, session: URLSession = .shared) {
|
||||
init(config: ServerConfig) {
|
||||
self.config = config
|
||||
self.session = session
|
||||
self.session = TLSPinning.session()
|
||||
}
|
||||
|
||||
func fetchToken() async throws -> TokenResponse {
|
||||
|
||||
Reference in new issue
Block a user