feat: conversation state machine, mic meter, history, wake lock (Phase 3)

Sources/LiveKit/ConversationState.swift

@@ -0,0 +1,92 @@
+import Foundation
+import os
+
+enum ConversationPhase: String {
+    case idle
+    case connecting
+    case listening
+    case thinking
+    case speaking
+}
+
+/// Ports the web UI's `inferState` + EMA smoothing (app.js): mic/agent levels
+/// are smoothed with `prev * 0.6 + raw * 0.4`, thresholds are user > 0.035 and
+/// agent > 0.02, and speaking → thinking when both go silent.
+@MainActor
+final class ConversationStateModel: ObservableObject {
+    @Published private(set) var phase: ConversationPhase = .idle
+    @Published private(set) var micLevel: Double = 0
+    @Published private(set) var agentLevel: Double = 0
+    @Published private(set) var isThinkingChipVisible = false
+
+    static let userThreshold = 0.035
+    static let agentThreshold = 0.02
+    static let thinkingDelay: Duration = .seconds(2)
+
+    private var smoothedMic: Double = 0
+    private var smoothedAgent: Double = 0
+    private var lastTransition = Date()
+    private var chipTask: Task<Void, Never>?
+
+    func start() {
+        stop()
+        setPhase(.connecting)
+    }
+
+    func stop() {
+        chipTask?.cancel()
+        chipTask = nil
+        smoothedMic = 0
+        smoothedAgent = 0
+        micLevel = 0
+        agentLevel = 0
+        isThinkingChipVisible = false
+        setPhase(.idle)
+    }
+
+    func tick(localAudioLevel: Float, remoteAudioLevel: Float) {
+        guard phase != .idle else { return }
+        smoothedMic = smoothedMic * 0.6 + Double(localAudioLevel) * 0.4
+        smoothedAgent = smoothedAgent * 0.6 + Double(remoteAudioLevel) * 0.4
+        micLevel = smoothedMic
+        agentLevel = smoothedAgent
+
+        let userSpeaking = smoothedMic > Self.userThreshold
+        let next: ConversationPhase
+        if smoothedAgent > Self.agentThreshold {
+            next = .speaking
+        } else if userSpeaking {
+            next = .listening
+        } else {
+            switch phase {
+            case .speaking: next = .thinking
+            default: next = .listening
+            }
+        }
+        setPhase(next)
+    }
+
+    private func setPhase(_ next: ConversationPhase) {
+        guard next != phase else { return }
+        let old = phase
+        logger.log("state \(old.rawValue, privacy: .public) -> \(next.rawValue, privacy: .public)")
+        phase = next
+        lastTransition = Date()
+        if next == .thinking {
+            chipTask?.cancel()
+            chipTask = Task { [weak self] in
+                try? await Task.sleep(for: Self.thinkingDelay)
+                guard !Task.isCancelled, let self else { return }
+                if self.phase == .thinking {
+                    self.isThinkingChipVisible = true
+                }
+            }
+        } else {
+            chipTask?.cancel()
+            chipTask = nil
+            isThinkingChipVisible = false
+        }
+    }
+}
+
+private let logger = Logger(subsystem: "dev.shane.voice", category: "state")

Sources/LiveKit/TranscriptHandler.swift

@@ -21,20 +21,30 @@ struct TranscriptEvent: Equatable, Identifiable {
 final class TranscriptHandler: ObservableObject {
     @Published private(set) var events: [TranscriptEvent] = []
 
+    let history = TranscriptHistory()
+
     func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
         guard !text.isEmpty else { return }
         upsert(id: id, role: role, text: text, isFinal: isFinal)
+        if isFinal {
+            history.add(role: role.rawValue, text: text)
+        }
     }
 
     func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
         closePartial(for: role)
         upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
+        history.add(role: role.rawValue, text: text)
     }
 
     func clear() {
         events.removeAll()
     }
 
+    func clearHistory() {
+        history.clear()
+    }
+
     private func closePartial(for role: TranscriptEvent.Role) {
         for index in events.indices where !events[index].isFinal && events[index].role == role {
             events[index].isFinal = true

Sources/LiveKit/TranscriptHistory.swift

@@ -0,0 +1,43 @@
+import Foundation
+
+struct HistoryEntry: Codable, Equatable {
+    let role: String
+    let text: String
+    let timestamp: Date
+}
+
+/// Last 50 final transcript entries persisted as JSON in UserDefaults.
+@MainActor
+final class TranscriptHistory: ObservableObject {
+    @Published private(set) var entries: [HistoryEntry] = []
+
+    private static let key = "voice_history"
+    private static let maxEntries = 50
+
+    init() {
+        if let raw = UserDefaults.standard.string(forKey: Self.key),
+           let data = raw.data(using: .utf8),
+           let decoded = try? JSONDecoder().decode([HistoryEntry].self, from: data) {
+            entries = Array(decoded.suffix(Self.maxEntries))
+        }
+    }
+
+    func add(role: String, text: String) {
+        guard !text.isEmpty else { return }
+        entries.append(HistoryEntry(role: role, text: text, timestamp: Date()))
+        if entries.count > Self.maxEntries {
+            entries.removeFirst(entries.count - Self.maxEntries)
+        }
+        persist()
+    }
+
+    func clear() {
+        entries.removeAll()
+        persist()
+    }
+
+    private func persist() {
+        guard let data = try? JSONEncoder().encode(entries) else { return }
+        UserDefaults.standard.set(String(decoding: data, as: UTF8.self), forKey: Self.key)
+    }
+}

Sources/LiveKit/VoiceSession.swift

@@ -1,3 +1,4 @@
+import UIKit
 import Foundation
 import LiveKit
 import os
@@ -24,10 +25,12 @@ final class VoiceSession: ObservableObject {
     @Published private(set) var state: VoiceSessionState = .idle
     @Published private(set) var agent: AgentInfo?
     @Published private(set) var isMuted = false
+    @Published private(set) var micError: String?
 
     private(set) var room = Room()
     let controlChannel = ControlChannel()
     let transcriptHandler = TranscriptHandler()
+    let stateModel = ConversationStateModel()
     private let config: ServerConfig
     private let tokenClient: TokenClient
     private var reconnectAttempted = false
@@ -78,6 +81,7 @@ final class VoiceSession: ObservableObject {
         switch result {
         case .success:
             logger.log("room connected")
+            UIApplication.shared.isIdleTimerDisabled = true
             scanForAgent()
             startAgentWait()
         case .failure(let error):
@@ -137,6 +141,7 @@ final class VoiceSession: ObservableObject {
         }
         agentWaitTask?.cancel()
         state = .ready
+        stateModel.start()
         logger.log("state=ready, enabling mic")
         Task { await enableMicrophone() }
     }
@@ -163,6 +168,7 @@ final class VoiceSession: ObservableObject {
             do {
                 try await room.localParticipant.setMicrophone(enabled: true)
                 isMuted = false
+                micError = nil
                 logger.log("mic enabled")
                 return
             } catch {
@@ -172,8 +178,9 @@ final class VoiceSession: ObservableObject {
                     try? await Task.sleep(for: .seconds(1))
                     continue
                 }
+                // Non-fatal: keep the session usable (text mode still works).
                 logger.error("mic enable failed after retries: \(message, privacy: .public)")
-                state = .failed("Microphone error: \(error.localizedDescription)")
+                micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
             }
         }
     }
@@ -184,11 +191,22 @@ final class VoiceSession: ObservableObject {
             try await room.localParticipant.setMicrophone(enabled: !muted)
             isMuted = muted
         } catch {
-            state = .failed("Microphone error: \(error.localizedDescription)")
+            micError = "Microphone error: \(error.localizedDescription)"
+        }
+    }
+
+    func setMode(_ mode: String) async {
+        guard isReady else { return }
+        do {
+            try await controlChannel.setMode(mode, room: room)
+            logger.log("set_mode sent: \(mode, privacy: .public)")
+        } catch {
+            logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
         }
     }
 
     func sendText(_ text: String) async throws {
+        logger.log("sendText lk.chat chars=\(text.count)")
         try await room.localParticipant.sendText(text, for: "lk.chat")
     }
 
@@ -197,10 +215,17 @@ final class VoiceSession: ObservableObject {
         state = .idle
         agent = nil
         isMuted = false
+        micError = nil
         reconnectAttempted = false
+        stateModel.stop()
+        UIApplication.shared.isIdleTimerDisabled = false
         await teardownRoom()
     }
 
+    func tickAudioLevels(local: Float, remote: Float) {
+        stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
+    }
+
 }
 
 extension VoiceSession: RoomDelegate {
@@ -211,6 +236,22 @@ extension VoiceSession: RoomDelegate {
         }
     }
 
+    nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
+        Task { @MainActor [weak self] in
+            guard let self, self.state == .ready else { return }
+            var local = Float(0)
+            var remote = Float(0)
+            for participant in participants {
+                if participant.identity?.stringValue == room.localParticipant.identity?.stringValue {
+                    local = max(local, participant.audioLevel)
+                } else {
+                    remote = max(remote, participant.audioLevel)
+                }
+            }
+            self.tickAudioLevels(local: local, remote: remote)
+        }
+    }
+
     nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
         logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
         Task { @MainActor [weak self] in