feat: LiveKit voice session, control channel, transcripts (Phase 2)
- VoiceSession: token->connect with fresh-room retry on timeout, agent discovery, agent_ready wait, mic enable/mute, text send over lk.chat - ControlChannel: outbound set_voice/set_model/get_models/set_mode/ media_state; inbound agent_ready/models/model_changed/task_event - TranscriptHandler: transcription segments + transcript topic finals, partial/final dedupe per role - ConversationView/OrbView: status orb, mic/text-mode toggles, composer - ServerConfig: plain ws:// signaling for private IPs (self-signed TLS not injectable into SDK WebSocket); Info.plist local networking
This commit is contained in:
snowspeeder committed
1 parent
378b8cb36f
commit
4986b6d7dc
14 files changed
+900
-50
No files matched your search
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Observes the agent participant's audio level for UI state inference.
|
||||
@MainActor
|
||||
final class AgentMonitor: ObservableObject {
|
||||
@Published private(set) var agentSpeaking = false
|
||||
@Published private(set) var localSpeaking = false
|
||||
|
||||
private let session: VoiceSession
|
||||
private var pollTask: Task<Void, Never>?
|
||||
|
||||
init(session: VoiceSession) {
|
||||
self.session = session
|
||||
}
|
||||
|
||||
func start() {
|
||||
stop()
|
||||
pollTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self else { return }
|
||||
let remote = self.session.room.remoteParticipants.values.first(where: {
|
||||
$0.kind == .agent || $0.identity?.stringValue == "voice-assistant"
|
||||
})
|
||||
self.agentSpeaking = remote?.isSpeaking ?? false
|
||||
self.localSpeaking = self.session.room.localParticipant.isSpeaking
|
||||
try? await Task.sleep(for: .milliseconds(250))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
pollTask?.cancel()
|
||||
pollTask = nil
|
||||
agentSpeaking = false
|
||||
localSpeaking = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Outbound helpers for agent control topics and inbound parsing of
|
||||
/// voice-control / conversation-control / media-control data messages.
|
||||
@MainActor
|
||||
final class ControlChannel: NSObject, RoomDelegate {
|
||||
|
||||
struct ModelInfo: Equatable {
|
||||
let id: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
enum InboundEvent {
|
||||
case agentReady
|
||||
case models([ModelInfo], selected: String?)
|
||||
case modelChanged(String)
|
||||
case taskEvent(TaskEventPayload)
|
||||
}
|
||||
|
||||
struct TaskEventPayload: Equatable {
|
||||
let type: String
|
||||
let taskId: String?
|
||||
let description: String?
|
||||
let role: String?
|
||||
let content: String?
|
||||
let result: String?
|
||||
let error: String?
|
||||
}
|
||||
|
||||
var onEvent: ((InboundEvent) -> Void)?
|
||||
|
||||
private let decoder = JSONDecoder()
|
||||
|
||||
/// Inbound voice-control handling. Returns true when the message was consumed.
|
||||
func handleVoiceControl(_ data: Data, session: VoiceSession) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
let type = json["type"]?.stringValue else { return false }
|
||||
switch type {
|
||||
case "agent_ready":
|
||||
Task { @MainActor [weak session] in
|
||||
session?.agentReadyReceived()
|
||||
}
|
||||
return true
|
||||
case "models":
|
||||
var models: [ModelInfo] = []
|
||||
for item in json["models"]?.arrayValue ?? [] {
|
||||
if let obj = item.objectValue, let id = obj["id"]?.stringValue {
|
||||
models.append(ModelInfo(id: id, name: obj["name"]?.stringValue))
|
||||
} else if let id = item.stringValue {
|
||||
models.append(ModelInfo(id: id, name: nil))
|
||||
}
|
||||
}
|
||||
onEvent?(.models(models, selected: json["selected"]?.stringValue))
|
||||
return true
|
||||
case "model_changed":
|
||||
if let model = json["model"]?.stringValue {
|
||||
onEvent?(.modelChanged(model))
|
||||
}
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// Inbound task_event handling (topic-agnostic per web UI). Returns true when consumed.
|
||||
func handleTaskEvent(_ data: Data) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
let type = json["type"]?.stringValue,
|
||||
["created", "step", "completed", "failed"].contains(type) else { return false }
|
||||
let payload = TaskEventPayload(
|
||||
type: type,
|
||||
taskId: json["task_id"]?.stringValue,
|
||||
description: json["description"]?.stringValue,
|
||||
role: json["role"]?.stringValue,
|
||||
content: json["content"]?.stringValue,
|
||||
result: json["result"]?.stringValue,
|
||||
error: json["error"]?.stringValue
|
||||
)
|
||||
onEvent?(.taskEvent(payload))
|
||||
return true
|
||||
}
|
||||
|
||||
// MARK: Outbound (topic'd JSON over reliable data streams)
|
||||
|
||||
private func publish(_ object: [String: Any], topic: String, room: Room) async throws {
|
||||
let text = String(data: try JSONSerialization.data(withJSONObject: object), encoding: .utf8) ?? "{}"
|
||||
_ = try await room.localParticipant.sendText(text, for: topic)
|
||||
}
|
||||
|
||||
func setVoice(_ voiceID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setModel(_ modelID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_model", "model": modelID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func getModels(room: Room) async throws {
|
||||
try await publish(["type": "get_models"], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setMode(_ mode: String, room: Room) async throws {
|
||||
try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room)
|
||||
}
|
||||
|
||||
func sendMediaState(_ state: String, trackID: String?, room: Room) async throws {
|
||||
var object: [String: Any] = ["type": "media_state", "state": state]
|
||||
if let trackID { object["track_id"] = trackID }
|
||||
try await publish(object, topic: "media-control", room: room)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Minimal JSON value decoder for flexible agent payloads
|
||||
|
||||
indirect enum JSONValue: Decodable {
|
||||
case string(String)
|
||||
case number(Double)
|
||||
case bool(Bool)
|
||||
case object([String: JSONValue])
|
||||
case array([JSONValue])
|
||||
case null
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.singleValueContainer()
|
||||
if container.decodeNil() { self = .null }
|
||||
else if let value = try? container.decode(Bool.self) { self = .bool(value) }
|
||||
else if let value = try? container.decode(Double.self) { self = .number(value) }
|
||||
else if let value = try? container.decode(String.self) { self = .string(value) }
|
||||
else if let value = try? container.decode([JSONValue].self) { self = .array(value) }
|
||||
else if let value = try? container.decode([String: JSONValue].self) { self = .object(value) }
|
||||
else { throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported JSON value") }
|
||||
}
|
||||
|
||||
var stringValue: String? {
|
||||
if case .string(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var objectValue: [String: JSONValue]? {
|
||||
if case .object(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var arrayValue: [JSONValue]? {
|
||||
if case .array(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
struct TranscriptEvent: Equatable, Identifiable {
|
||||
enum Role: String {
|
||||
case user
|
||||
case agent
|
||||
}
|
||||
|
||||
let id: String
|
||||
let role: Role
|
||||
var text: String
|
||||
var isFinal: Bool
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Merges the primary transcription segments (Room delegate) with the
|
||||
/// secondary `transcript` data topic (finals only). Dedupe rule: a final for a
|
||||
/// role closes any open partial row for that role.
|
||||
@MainActor
|
||||
final class TranscriptHandler: ObservableObject {
|
||||
@Published private(set) var events: [TranscriptEvent] = []
|
||||
|
||||
func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
guard !text.isEmpty else { return }
|
||||
upsert(id: id, role: role, text: text, isFinal: isFinal)
|
||||
}
|
||||
|
||||
func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
|
||||
closePartial(for: role)
|
||||
upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
|
||||
}
|
||||
|
||||
func clear() {
|
||||
events.removeAll()
|
||||
}
|
||||
|
||||
private func closePartial(for role: TranscriptEvent.Role) {
|
||||
for index in events.indices where !events[index].isFinal && events[index].role == role {
|
||||
events[index].isFinal = true
|
||||
}
|
||||
}
|
||||
|
||||
private func upsert(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
if let index = events.firstIndex(where: { $0.id == id }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = isFinal
|
||||
} else {
|
||||
events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: Date()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension TranscriptHandler: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) {
|
||||
let role: TranscriptEvent.Role = participant.isAgent ? .agent : .user
|
||||
let isLocal = participant.identity?.stringValue == room.localParticipant.identity?.stringValue
|
||||
let resolvedRole: TranscriptEvent.Role = isLocal ? .user : role
|
||||
let events = segments.map {
|
||||
TranscriptEvent(id: $0.id, role: resolvedRole, text: $0.text, isFinal: $0.isFinal, timestamp: $0.lastReceivedTime)
|
||||
}
|
||||
Task { @MainActor [weak self] in
|
||||
for event in events {
|
||||
self?.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
import os
|
||||
|
||||
private let logger = Logger(subsystem: "dev.shane.voice", category: "session")
|
||||
|
||||
enum VoiceSessionState: Equatable {
|
||||
case idle
|
||||
case connecting
|
||||
case waitingAgent
|
||||
case ready
|
||||
case failed(String)
|
||||
}
|
||||
|
||||
struct AgentInfo: Equatable {
|
||||
let identity: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
/// Room lifecycle: connect, agent discovery, agent_ready wait, reconnect-once,
|
||||
/// disconnect teardown. Ports the web UI's connect flow.
|
||||
@MainActor
|
||||
final class VoiceSession: ObservableObject {
|
||||
@Published private(set) var state: VoiceSessionState = .idle
|
||||
@Published private(set) var agent: AgentInfo?
|
||||
@Published private(set) var isMuted = false
|
||||
|
||||
private(set) var room = Room()
|
||||
let controlChannel = ControlChannel()
|
||||
let transcriptHandler = TranscriptHandler()
|
||||
private let config: ServerConfig
|
||||
private let tokenClient: TokenClient
|
||||
private var reconnectAttempted = false
|
||||
private var connectAttempts = 0
|
||||
private var agentWaitTask: Task<Void, Never>?
|
||||
|
||||
init(config: ServerConfig, tokenClient: TokenClient) {
|
||||
self.config = config
|
||||
self.tokenClient = tokenClient
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
controlChannel.onEvent = { [weak self] _ in
|
||||
guard self != nil else { return }
|
||||
}
|
||||
}
|
||||
|
||||
var isReady: Bool { state == .ready }
|
||||
|
||||
func connect() async {
|
||||
await start()
|
||||
}
|
||||
|
||||
private func makeRoom() -> Room {
|
||||
let room = Room()
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
return room
|
||||
}
|
||||
|
||||
private func start() async {
|
||||
state = .connecting
|
||||
connectAttempts = 0
|
||||
await attemptConnect()
|
||||
}
|
||||
|
||||
private func attemptConnect() async {
|
||||
let result: Result<Void, Error>
|
||||
do {
|
||||
let token = try await tokenClient.fetchToken()
|
||||
logger.log("token ok room=\(token.room, privacy: .public)")
|
||||
try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions())
|
||||
result = .success(())
|
||||
} catch {
|
||||
result = .failure(error)
|
||||
}
|
||||
switch result {
|
||||
case .success:
|
||||
logger.log("room connected")
|
||||
scanForAgent()
|
||||
startAgentWait()
|
||||
case .failure(let error):
|
||||
let isTimeout = (error as? LiveKitError)?.type == .timedOut
|
||||
if isTimeout, connectAttempts < 2 {
|
||||
connectAttempts += 1
|
||||
let attempt = connectAttempts
|
||||
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
||||
await teardownRoom()
|
||||
await attemptConnect()
|
||||
return
|
||||
}
|
||||
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
||||
state = .failed(error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func makeConnectOptions() -> ConnectOptions {
|
||||
return ConnectOptions(
|
||||
socketConnectTimeoutInterval: 15,
|
||||
primaryTransportConnectTimeout: 20,
|
||||
publisherTransportConnectTimeout: 20
|
||||
)
|
||||
}
|
||||
|
||||
private func teardownRoom() async {
|
||||
await room.disconnect()
|
||||
room = makeRoom()
|
||||
}
|
||||
|
||||
private func handleAgentTimeout() async {
|
||||
guard state == .waitingAgent, !reconnectAttempted else {
|
||||
if state == .waitingAgent {
|
||||
state = .failed("Agent never joined. Is the agent worker running?")
|
||||
}
|
||||
return
|
||||
}
|
||||
reconnectAttempted = true
|
||||
await teardownRoom()
|
||||
await start()
|
||||
}
|
||||
|
||||
private func startAgentWait() {
|
||||
state = .waitingAgent
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = Task { [weak self] in
|
||||
try? await Task.sleep(for: .seconds(10))
|
||||
guard !Task.isCancelled else { return }
|
||||
await self?.handleAgentTimeout()
|
||||
}
|
||||
}
|
||||
|
||||
func agentReadyReceived() {
|
||||
guard state == .waitingAgent else {
|
||||
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
||||
return
|
||||
}
|
||||
agentWaitTask?.cancel()
|
||||
state = .ready
|
||||
logger.log("state=ready, enabling mic")
|
||||
Task { await enableMicrophone() }
|
||||
}
|
||||
|
||||
private func scanForAgent() {
|
||||
for participant in room.remoteParticipants.values {
|
||||
if isAgentParticipant(participant) {
|
||||
foundAgent(participant)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func isAgentParticipant(_ participant: RemoteParticipant) -> Bool {
|
||||
participant.kind == .agent || participant.identity?.stringValue == "voice-assistant"
|
||||
}
|
||||
|
||||
private func foundAgent(_ participant: RemoteParticipant) {
|
||||
agent = AgentInfo(identity: participant.identity?.stringValue ?? "", name: participant.name)
|
||||
}
|
||||
|
||||
func enableMicrophone() async {
|
||||
for attempt in 0..<3 {
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: true)
|
||||
isMuted = false
|
||||
logger.log("mic enabled")
|
||||
return
|
||||
} catch {
|
||||
let message = String(describing: error)
|
||||
if attempt < 2 {
|
||||
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
||||
try? await Task.sleep(for: .seconds(1))
|
||||
continue
|
||||
}
|
||||
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
||||
state = .failed("Microphone error: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setMuted(_ muted: Bool) async {
|
||||
guard isReady else { return }
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||
isMuted = muted
|
||||
} catch {
|
||||
state = .failed("Microphone error: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
func sendText(_ text: String) async throws {
|
||||
try await room.localParticipant.sendText(text, for: "lk.chat")
|
||||
}
|
||||
|
||||
func disconnect() async {
|
||||
agentWaitTask?.cancel()
|
||||
state = .idle
|
||||
agent = nil
|
||||
isMuted = false
|
||||
reconnectAttempted = false
|
||||
await teardownRoom()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
extension VoiceSession: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.isAgentParticipant(participant) else { return }
|
||||
self.foundAgent(participant)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
||||
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
// Secondary transcript source (finals only).
|
||||
if topic == "transcript",
|
||||
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
||||
json["type"]?.stringValue == "transcript",
|
||||
let text = json["text"]?.stringValue {
|
||||
let role: TranscriptEvent.Role = json["role"]?.stringValue == "agent" ? .agent : .user
|
||||
self.transcriptHandler.receiveDataTranscript(role: role, text: text)
|
||||
return
|
||||
}
|
||||
if self.controlChannel.handleTaskEvent(data) { return }
|
||||
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user