snowspeeder/hope-voice-api
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: conversation state machine, mic meter, history, wake lock (Phase 3)

Browse Source
This commit is contained in:
snowspeeder committed 2026-08-30 07:36:13 -04:00
1 parent 4986b6d7dc
commit 5483e6ed48
8 files changed
+348 -29
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Sources/Features/Conversation/ConversationView.swift
+85 -11
View File
@@ -31,7 +31,11 @@ struct ConversationView: View {
NavigationStack {
VStack(spacing: 0) {
statusHeader
TranscriptListView(events: session.transcriptHandler.events)
TranscriptListView(
events: session.transcriptHandler.events,
history: session.transcriptHandler.history,
onClear: { session.transcriptHandler.clearHistory() }
)
composer
}
.navigationTitle("Hope Voice")
@@ -49,6 +53,21 @@ struct ConversationView: View {
OrbView(state: orbState)
Text(statusText)
.font(.headline)
if session.state == .ready, session.stateModel.isThinkingChipVisible {
Label("Thinking…", systemImage: "ellipsis")
.font(.caption)
.padding(.horizontal, 10)
.padding(.vertical, 4)
.background(.orange.opacity(0.2), in: Capsule())
}
if session.state == .ready, !settings.textMode {
micMeter
}
if session.state == .ready, settings.textMode {
Text("Barge in anytime — type to reply")
.font(.caption)
.foregroundStyle(.secondary)
}
if let agent = session.agent {
Text("Agent: \(agent.name ?? agent.identity)")
.font(.caption)
@@ -63,10 +82,16 @@ struct ConversationView: View {
Label(session.isMuted ? "Muted" : "Mic on", systemImage: session.isMuted ? "mic.slash" : "mic")
}
.toggleStyle(.button)
Toggle("Text mode", isOn: $settings.textMode)
Toggle("Text mode", isOn: textModeBinding)
.toggleStyle(.button)
}
}
if let micError = session.micError {
Label(micError, systemImage: "exclamationmark.triangle")
.font(.footnote)
.foregroundStyle(.orange)
.frame(maxWidth: .infinity, alignment: .leading)
}
if let connectError {
Label(connectError, systemImage: "exclamationmark.triangle")
.font(.footnote)
@@ -76,6 +101,40 @@ struct ConversationView: View {
.padding()
}
private var micMeter: some View {
GeometryReader { geo in
ZStack(alignment: .leading) {
Capsule()
.fill(Color(.systemFill))
Capsule()
.fill(Color.green.gradient)
.frame(width: min(geo.size.width, geo.size.width * session.stateModel.micLevel * 2.5))
}
}
.frame(height: 4)
}
private var textModeBinding: Binding<Bool> {
Binding(
get: { settings.textMode },
set: { newValue in
settings.textMode = newValue
Task { await applyTextMode(newValue) }
}
)
}
private func applyTextMode(_ enabled: Bool) async {
guard session.isReady else { return }
if enabled {
await session.setMuted(true)
await session.setMode("text")
} else {
await session.setMode("voice")
await session.enableMicrophone()
}
}
private var composer: some View {
HStack(spacing: 8) {
switch session.state {
@@ -118,20 +177,35 @@ struct ConversationView: View {
private var orbState: OrbView.OrbState {
switch session.state {
case .idle: .idle
case .connecting, .waitingAgent: .connecting
case .ready: settings.textMode ? .idle : .listening
case .failed: .failed
case .idle: return .idle
case .connecting, .waitingAgent: return .connecting
case .ready:
guard !settings.textMode else { return .idle }
switch session.stateModel.phase {
case .idle: return .idle
case .connecting: return .connecting
case .listening: return .listening
case .thinking: return .thinking
case .speaking: return .speaking
}
case .failed: return .failed
}
}
private var statusText: String {
switch session.state {
case .idle: "Ready to connect"
case .connecting: "Connecting…"
case .waitingAgent: "Waiting for agent…"
case .ready: settings.textMode ? "Text mode" : "Listening"
case .failed(let message): "Failed: \(message)"
case .idle: return "Ready to connect"
case .connecting: return "Connecting…"
case .waitingAgent: return "Waiting for agent…"
case .ready:
if settings.textMode { return "Text mode" }
switch session.stateModel.phase {
case .listening: return "Listening"
case .thinking: return "Thinking it through"
case .speaking: return "Hope is speaking"
default: return "Ready when you are"
}
case .failed(let message): return "Failed: \(message)"
}
}
Sources/Features/Conversation/OrbView.swift
+44 -13
View File
@@ -49,20 +49,46 @@ struct OrbView: View {
struct TranscriptListView: View {
let events: [TranscriptEvent]
@ObservedObject var history: TranscriptHistory
var onClear: () -> Void
private static let deliveryMarkers = #"\[(?:appreciative|calm|concerned|curious|encouraging|excited|happy|joking|quiet|reassuring|reflective|sad|serious|surprised|sympathetic|urgent|whispering|laughter|sighing|breathing|neutral)\]\s*"#
private static let timestampFormatter: DateFormatter = {
let formatter = DateFormatter()
formatter.dateStyle = .none
formatter.timeStyle = .short
return formatter
}()
var body: some View {
ScrollViewReader { proxy in
ScrollView {
LazyVStack(alignment: .leading, spacing: 8) {
ForEach(events) { event in
transcriptRow(event)
VStack(alignment: .leading, spacing: 0) {
HStack {
Text("Transcript")
.font(.caption.bold())
.foregroundStyle(.secondary)
Spacer()
Button(role: .destructive, action: onClear) {
Label("Clear", systemImage: "trash")
.font(.caption)
}
}
.padding(.horizontal)
}
.onChange(of: events.count) { _, count in
if let last = events.last {
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
.padding(.bottom, 4)
ScrollView {
LazyVStack(alignment: .leading, spacing: 8) {
ForEach(events) { event in
transcriptRow(event)
}
}
.padding(.horizontal)
}
.onChange(of: events.count) { _, count in
if let last = events.last {
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
}
}
}
}
@@ -76,9 +102,15 @@ struct TranscriptListView: View {
: Color.blue.opacity(0.08)
let textOpacity: Double = event.isFinal ? 1.0 : 0.55
return VStack(alignment: .leading, spacing: 2) {
Text(roleName)
.font(.caption2.bold())
.foregroundStyle(nameColor)
HStack {
Text(roleName)
.font(.caption2.bold())
.foregroundStyle(nameColor)
Spacer()
Text(Self.timestampFormatter.string(from: event.timestamp))
.font(.caption2)
.foregroundStyle(.tertiary)
}
Text(stripped(event.text))
.font(.body)
.opacity(textOpacity)
@@ -91,9 +123,8 @@ struct TranscriptListView: View {
}
private func stripped(_ text: String) -> String {
// Strip delivery-style markers like [appreciative], [pauses] etc.
text.replacingOccurrences(
of: #"\[[a-zA-Z ,'-]+\]"#,
of: Self.deliveryMarkers,
with: "",
options: .regularExpression
).trimmingCharacters(in: .whitespaces)
Sources/LiveKit/ConversationState.swift
+92
View File
@@ -0,0 +1,92 @@
import Foundation
import os
enum ConversationPhase: String {
case idle
case connecting
case listening
case thinking
case speaking
}
/// Ports the web UI's `inferState` + EMA smoothing (app.js): mic/agent levels
/// are smoothed with `prev * 0.6 + raw * 0.4`, thresholds are user > 0.035 and
/// agent > 0.02, and speaking → thinking when both go silent.
@MainActor
final class ConversationStateModel: ObservableObject {
@Published private(set) var phase: ConversationPhase = .idle
@Published private(set) var micLevel: Double = 0
@Published private(set) var agentLevel: Double = 0
@Published private(set) var isThinkingChipVisible = false
static let userThreshold = 0.035
static let agentThreshold = 0.02
static let thinkingDelay: Duration = .seconds(2)
private var smoothedMic: Double = 0
private var smoothedAgent: Double = 0
private var lastTransition = Date()
private var chipTask: Task<Void, Never>?
func start() {
stop()
setPhase(.connecting)
}
func stop() {
chipTask?.cancel()
chipTask = nil
smoothedMic = 0
smoothedAgent = 0
micLevel = 0
agentLevel = 0
isThinkingChipVisible = false
setPhase(.idle)
}
func tick(localAudioLevel: Float, remoteAudioLevel: Float) {
guard phase != .idle else { return }
smoothedMic = smoothedMic * 0.6 + Double(localAudioLevel) * 0.4
smoothedAgent = smoothedAgent * 0.6 + Double(remoteAudioLevel) * 0.4
micLevel = smoothedMic
agentLevel = smoothedAgent
let userSpeaking = smoothedMic > Self.userThreshold
let next: ConversationPhase
if smoothedAgent > Self.agentThreshold {
next = .speaking
} else if userSpeaking {
next = .listening
} else {
switch phase {
case .speaking: next = .thinking
default: next = .listening
}
}
setPhase(next)
}
private func setPhase(_ next: ConversationPhase) {
guard next != phase else { return }
let old = phase
logger.log("state \(old.rawValue, privacy: .public) -> \(next.rawValue, privacy: .public)")
phase = next
lastTransition = Date()
if next == .thinking {
chipTask?.cancel()
chipTask = Task { [weak self] in
try? await Task.sleep(for: Self.thinkingDelay)
guard !Task.isCancelled, let self else { return }
if self.phase == .thinking {
self.isThinkingChipVisible = true
}
}
} else {
chipTask?.cancel()
chipTask = nil
isThinkingChipVisible = false
}
}
}
private let logger = Logger(subsystem: "dev.shane.voice", category: "state")
Sources/LiveKit/TranscriptHandler.swift
+10
View File
@@ -21,20 +21,30 @@ struct TranscriptEvent: Equatable, Identifiable {
final class TranscriptHandler: ObservableObject {
@Published private(set) var events: [TranscriptEvent] = []
let history = TranscriptHistory()
func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
guard !text.isEmpty else { return }
upsert(id: id, role: role, text: text, isFinal: isFinal)
if isFinal {
history.add(role: role.rawValue, text: text)
}
}
func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
closePartial(for: role)
upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
history.add(role: role.rawValue, text: text)
}
func clear() {
events.removeAll()
}
func clearHistory() {
history.clear()
}
private func closePartial(for role: TranscriptEvent.Role) {
for index in events.indices where !events[index].isFinal && events[index].role == role {
events[index].isFinal = true
Sources/LiveKit/TranscriptHistory.swift
+43
View File
@@ -0,0 +1,43 @@
import Foundation
struct HistoryEntry: Codable, Equatable {
let role: String
let text: String
let timestamp: Date
}
/// Last 50 final transcript entries persisted as JSON in UserDefaults.
@MainActor
final class TranscriptHistory: ObservableObject {
@Published private(set) var entries: [HistoryEntry] = []
private static let key = "voice_history"
private static let maxEntries = 50
init() {
if let raw = UserDefaults.standard.string(forKey: Self.key),
let data = raw.data(using: .utf8),
let decoded = try? JSONDecoder().decode([HistoryEntry].self, from: data) {
entries = Array(decoded.suffix(Self.maxEntries))
}
}
func add(role: String, text: String) {
guard !text.isEmpty else { return }
entries.append(HistoryEntry(role: role, text: text, timestamp: Date()))
if entries.count > Self.maxEntries {
entries.removeFirst(entries.count - Self.maxEntries)
}
persist()
}
func clear() {
entries.removeAll()
persist()
}
private func persist() {
guard let data = try? JSONEncoder().encode(entries) else { return }
UserDefaults.standard.set(String(decoding: data, as: UTF8.self), forKey: Self.key)
}
}
Sources/LiveKit/VoiceSession.swift
+43 -2
View File
@@ -1,3 +1,4 @@
import UIKit
import Foundation
import LiveKit
import os
@@ -24,10 +25,12 @@ final class VoiceSession: ObservableObject {
@Published private(set) var state: VoiceSessionState = .idle
@Published private(set) var agent: AgentInfo?
@Published private(set) var isMuted = false
@Published private(set) var micError: String?
private(set) var room = Room()
let controlChannel = ControlChannel()
let transcriptHandler = TranscriptHandler()
let stateModel = ConversationStateModel()
private let config: ServerConfig
private let tokenClient: TokenClient
private var reconnectAttempted = false
@@ -78,6 +81,7 @@ final class VoiceSession: ObservableObject {
switch result {
case .success:
logger.log("room connected")
UIApplication.shared.isIdleTimerDisabled = true
scanForAgent()
startAgentWait()
case .failure(let error):
@@ -137,6 +141,7 @@ final class VoiceSession: ObservableObject {
}
agentWaitTask?.cancel()
state = .ready
stateModel.start()
logger.log("state=ready, enabling mic")
Task { await enableMicrophone() }
}
@@ -163,6 +168,7 @@ final class VoiceSession: ObservableObject {
do {
try await room.localParticipant.setMicrophone(enabled: true)
isMuted = false
micError = nil
logger.log("mic enabled")
return
} catch {
@@ -172,8 +178,9 @@ final class VoiceSession: ObservableObject {
try? await Task.sleep(for: .seconds(1))
continue
}
// Non-fatal: keep the session usable (text mode still works).
logger.error("mic enable failed after retries: \(message, privacy: .public)")
state = .failed("Microphone error: \(error.localizedDescription)")
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
}
}
}
@@ -184,11 +191,22 @@ final class VoiceSession: ObservableObject {
try await room.localParticipant.setMicrophone(enabled: !muted)
isMuted = muted
} catch {
state = .failed("Microphone error: \(error.localizedDescription)")
micError = "Microphone error: \(error.localizedDescription)"
}
}
func setMode(_ mode: String) async {
guard isReady else { return }
do {
try await controlChannel.setMode(mode, room: room)
logger.log("set_mode sent: \(mode, privacy: .public)")
} catch {
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
}
}
func sendText(_ text: String) async throws {
logger.log("sendText lk.chat chars=\(text.count)")
try await room.localParticipant.sendText(text, for: "lk.chat")
}
@@ -197,10 +215,17 @@ final class VoiceSession: ObservableObject {
state = .idle
agent = nil
isMuted = false
micError = nil
reconnectAttempted = false
stateModel.stop()
UIApplication.shared.isIdleTimerDisabled = false
await teardownRoom()
}
func tickAudioLevels(local: Float, remote: Float) {
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
}
}
extension VoiceSession: RoomDelegate {
@@ -211,6 +236,22 @@ extension VoiceSession: RoomDelegate {
}
}
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
Task { @MainActor [weak self] in
guard let self, self.state == .ready else { return }
var local = Float(0)
var remote = Float(0)
for participant in participants {
if participant.identity?.stringValue == room.localParticipant.identity?.stringValue {
local = max(local, participant.audioLevel)
} else {
remote = max(remote, participant.audioLevel)
}
}
self.tickAudioLevels(local: local, remote: remote)
}
}
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
Task { @MainActor [weak self] in