feat: LiveKit voice session, control channel, transcripts (Phase 2)

- VoiceSession: token->connect with fresh-room retry on timeout, agent
  discovery, agent_ready wait, mic enable/mute, text send over lk.chat
- ControlChannel: outbound set_voice/set_model/get_models/set_mode/
  media_state; inbound agent_ready/models/model_changed/task_event
- TranscriptHandler: transcription segments + transcript topic finals,
  partial/final dedupe per role
- ConversationView/OrbView: status orb, mic/text-mode toggles, composer
- ServerConfig: plain ws:// signaling for private IPs (self-signed TLS
  not injectable into SDK WebSocket); Info.plist local networking
This commit is contained in:
snowspeeder committed 2026-08-30 07:26:58 -04:00
1 parent 378b8cb36f
commit 4986b6d7dc
14 files changed
+900 -50

No files matched your search

+149
View File
@@ -0,0 +1,149 @@
import Foundation
import LiveKit
/// Outbound helpers for agent control topics and inbound parsing of
/// voice-control / conversation-control / media-control data messages.
@MainActor
final class ControlChannel: NSObject, RoomDelegate {
struct ModelInfo: Equatable {
let id: String
let name: String?
}
enum InboundEvent {
case agentReady
case models([ModelInfo], selected: String?)
case modelChanged(String)
case taskEvent(TaskEventPayload)
}
struct TaskEventPayload: Equatable {
let type: String
let taskId: String?
let description: String?
let role: String?
let content: String?
let result: String?
let error: String?
}
var onEvent: ((InboundEvent) -> Void)?
private let decoder = JSONDecoder()
/// Inbound voice-control handling. Returns true when the message was consumed.
func handleVoiceControl(_ data: Data, session: VoiceSession) -> Bool {
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
let type = json["type"]?.stringValue else { return false }
switch type {
case "agent_ready":
Task { @MainActor [weak session] in
session?.agentReadyReceived()
}
return true
case "models":
var models: [ModelInfo] = []
for item in json["models"]?.arrayValue ?? [] {
if let obj = item.objectValue, let id = obj["id"]?.stringValue {
models.append(ModelInfo(id: id, name: obj["name"]?.stringValue))
} else if let id = item.stringValue {
models.append(ModelInfo(id: id, name: nil))
}
}
onEvent?(.models(models, selected: json["selected"]?.stringValue))
return true
case "model_changed":
if let model = json["model"]?.stringValue {
onEvent?(.modelChanged(model))
}
return true
default:
return false
}
}
/// Inbound task_event handling (topic-agnostic per web UI). Returns true when consumed.
func handleTaskEvent(_ data: Data) -> Bool {
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
let type = json["type"]?.stringValue,
["created", "step", "completed", "failed"].contains(type) else { return false }
let payload = TaskEventPayload(
type: type,
taskId: json["task_id"]?.stringValue,
description: json["description"]?.stringValue,
role: json["role"]?.stringValue,
content: json["content"]?.stringValue,
result: json["result"]?.stringValue,
error: json["error"]?.stringValue
)
onEvent?(.taskEvent(payload))
return true
}
// MARK: Outbound (topic'd JSON over reliable data streams)
private func publish(_ object: [String: Any], topic: String, room: Room) async throws {
let text = String(data: try JSONSerialization.data(withJSONObject: object), encoding: .utf8) ?? "{}"
_ = try await room.localParticipant.sendText(text, for: topic)
}
func setVoice(_ voiceID: String, room: Room) async throws {
try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room)
}
func setModel(_ modelID: String, room: Room) async throws {
try await publish(["type": "set_model", "model": modelID], topic: "voice-control", room: room)
}
func getModels(room: Room) async throws {
try await publish(["type": "get_models"], topic: "voice-control", room: room)
}
func setMode(_ mode: String, room: Room) async throws {
try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room)
}
func sendMediaState(_ state: String, trackID: String?, room: Room) async throws {
var object: [String: Any] = ["type": "media_state", "state": state]
if let trackID { object["track_id"] = trackID }
try await publish(object, topic: "media-control", room: room)
}
}
// MARK: - Minimal JSON value decoder for flexible agent payloads
indirect enum JSONValue: Decodable {
case string(String)
case number(Double)
case bool(Bool)
case object([String: JSONValue])
case array([JSONValue])
case null
init(from decoder: Decoder) throws {
let container = try decoder.singleValueContainer()
if container.decodeNil() { self = .null }
else if let value = try? container.decode(Bool.self) { self = .bool(value) }
else if let value = try? container.decode(Double.self) { self = .number(value) }
else if let value = try? container.decode(String.self) { self = .string(value) }
else if let value = try? container.decode([JSONValue].self) { self = .array(value) }
else if let value = try? container.decode([String: JSONValue].self) { self = .object(value) }
else { throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported JSON value") }
}
var stringValue: String? {
if case .string(let value) = self { return value }
return nil
}
var objectValue: [String: JSONValue]? {
if case .object(let value) = self { return value }
return nil
}
var arrayValue: [JSONValue]? {
if case .array(let value) = self { return value }
return nil
}
}