feat: consolidate iOS with API and web preserving history
Rebuild & Deploy / rebuild (push) Failing after 23s
Rebuild & Deploy / rebuild (push) Failing after 23s
This commit is contained in:
commit
d735ee72fa
102 files changed
+13077
-258
No files matched your search
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Observes the agent participant's audio level for UI state inference.
|
||||
@MainActor
|
||||
final class AgentMonitor: ObservableObject {
|
||||
@Published private(set) var agentSpeaking = false
|
||||
@Published private(set) var localSpeaking = false
|
||||
|
||||
private let session: VoiceSession
|
||||
private var pollTask: Task<Void, Never>?
|
||||
|
||||
init(session: VoiceSession) {
|
||||
self.session = session
|
||||
}
|
||||
|
||||
func start() {
|
||||
stop()
|
||||
pollTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self else { return }
|
||||
let remote = self.session.room.remoteParticipants.values.first(where: {
|
||||
$0.kind == .agent || $0.identity?.stringValue == "voice-assistant"
|
||||
})
|
||||
self.agentSpeaking = remote?.isSpeaking ?? false
|
||||
self.localSpeaking = self.session.room.localParticipant.isSpeaking
|
||||
try? await Task.sleep(for: .milliseconds(250))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
pollTask?.cancel()
|
||||
pollTask = nil
|
||||
agentSpeaking = false
|
||||
localSpeaking = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
/// Outbound helpers for agent control topics and inbound parsing of
|
||||
/// voice-control / conversation-control data messages.
|
||||
@MainActor
|
||||
final class ControlChannel: NSObject, RoomDelegate {
|
||||
|
||||
enum InboundEvent {
|
||||
case agentReady
|
||||
case taskEvent(TaskEventPayload)
|
||||
}
|
||||
|
||||
struct TaskEventPayload: Equatable {
|
||||
let type: String
|
||||
let taskId: String?
|
||||
let description: String?
|
||||
let role: String?
|
||||
let content: String?
|
||||
let result: String?
|
||||
let error: String?
|
||||
let createdAt: Double?
|
||||
}
|
||||
|
||||
var onEvent: ((InboundEvent) -> Void)?
|
||||
|
||||
private let decoder = JSONDecoder()
|
||||
|
||||
/// Inbound voice-control handling. Returns true when the message was consumed.
|
||||
func handleVoiceControl(_ data: Data, session: VoiceSession) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
let type = json["type"]?.stringValue else { return false }
|
||||
switch type {
|
||||
case "agent_ready":
|
||||
Task { @MainActor [weak session] in
|
||||
session?.agentReadyReceived()
|
||||
}
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// Inbound task_event handling (topic-agnostic per web UI). The agent publishes
|
||||
/// {type:"task_event", task_id, event, ...} — the discriminator is `event`, not `type`.
|
||||
func handleTaskEvent(_ data: Data) -> Bool {
|
||||
guard let json = try? decoder.decode([String: JSONValue].self, from: data),
|
||||
json["type"]?.stringValue == "task_event",
|
||||
let type = json["event"]?.stringValue,
|
||||
["created", "step", "completed", "failed"].contains(type) else { return false }
|
||||
let taskObj = json["task"]?.objectValue ?? [:]
|
||||
let payload = TaskEventPayload(
|
||||
type: type,
|
||||
taskId: json["task_id"]?.stringValue,
|
||||
description: json["description"]?.stringValue ?? taskObj["description"]?.stringValue,
|
||||
role: json["role"]?.stringValue,
|
||||
content: json["content"]?.stringValue,
|
||||
result: json["result"]?.stringValue,
|
||||
error: json["error"]?.stringValue,
|
||||
createdAt: json["created_at"]?.doubleValue ?? taskObj["created_at"]?.doubleValue
|
||||
)
|
||||
onEvent?(.taskEvent(payload))
|
||||
return true
|
||||
}
|
||||
|
||||
// MARK: Outbound (topic'd JSON over reliable data streams)
|
||||
|
||||
private func publish(_ object: [String: Any], topic: String, room: Room) async throws {
|
||||
let data = try JSONSerialization.data(withJSONObject: object)
|
||||
try await room.localParticipant.publish(data: data, options: DataPublishOptions(topic: topic, reliable: true))
|
||||
}
|
||||
|
||||
func setVoice(_ voiceID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setMode(_ mode: String, room: Room) async throws {
|
||||
try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// MARK: - Minimal JSON value decoder for flexible agent payloads
|
||||
|
||||
indirect enum JSONValue: Decodable {
|
||||
case string(String)
|
||||
case number(Double)
|
||||
case bool(Bool)
|
||||
case object([String: JSONValue])
|
||||
case array([JSONValue])
|
||||
case null
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.singleValueContainer()
|
||||
if container.decodeNil() { self = .null }
|
||||
else if let value = try? container.decode(Bool.self) { self = .bool(value) }
|
||||
else if let value = try? container.decode(Double.self) { self = .number(value) }
|
||||
else if let value = try? container.decode(String.self) { self = .string(value) }
|
||||
else if let value = try? container.decode([JSONValue].self) { self = .array(value) }
|
||||
else if let value = try? container.decode([String: JSONValue].self) { self = .object(value) }
|
||||
else { throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported JSON value") }
|
||||
}
|
||||
|
||||
var stringValue: String? {
|
||||
if case .string(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var doubleValue: Double? {
|
||||
if case .number(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var objectValue: [String: JSONValue]? {
|
||||
if case .object(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
|
||||
var arrayValue: [JSONValue]? {
|
||||
if case .array(let value) = self { return value }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import Foundation
|
||||
import os
|
||||
|
||||
enum ConversationPhase: String {
|
||||
case idle
|
||||
case connecting
|
||||
case listening
|
||||
case thinking
|
||||
case speaking
|
||||
}
|
||||
|
||||
/// Ports the web UI's `inferState` + EMA smoothing (app.js): mic/agent levels
|
||||
/// are smoothed with `prev * 0.6 + raw * 0.4`, thresholds are user > 0.035 and
|
||||
/// agent > 0.02. When remote audio ends, the state returns to listening
|
||||
/// (matching the current web reference at app.js:414-426) rather than becoming
|
||||
/// "thinking" forever.
|
||||
@MainActor
|
||||
final class ConversationStateModel: ObservableObject {
|
||||
@Published private(set) var phase: ConversationPhase = .idle
|
||||
@Published private(set) var micLevel: Double = 0
|
||||
@Published private(set) var agentLevel: Double = 0
|
||||
@Published private(set) var isThinkingChipVisible = false
|
||||
|
||||
static let userThreshold = 0.035
|
||||
static let agentThreshold = 0.02
|
||||
static let thinkingDelay: Duration = .seconds(2)
|
||||
|
||||
private var smoothedMic: Double = 0
|
||||
private var smoothedAgent: Double = 0
|
||||
private var lastTransition = Date()
|
||||
private var chipTask: Task<Void, Never>?
|
||||
|
||||
func start() {
|
||||
stop()
|
||||
setPhase(.connecting)
|
||||
}
|
||||
|
||||
func stop() {
|
||||
chipTask?.cancel()
|
||||
chipTask = nil
|
||||
smoothedMic = 0
|
||||
smoothedAgent = 0
|
||||
micLevel = 0
|
||||
agentLevel = 0
|
||||
isThinkingChipVisible = false
|
||||
setPhase(.idle)
|
||||
}
|
||||
|
||||
func tick(localAudioLevel: Float, remoteAudioLevel: Float) {
|
||||
guard phase != .idle else { return }
|
||||
smoothedMic = smoothedMic * 0.6 + Double(localAudioLevel) * 0.4
|
||||
smoothedAgent = smoothedAgent * 0.6 + Double(remoteAudioLevel) * 0.4
|
||||
micLevel = smoothedMic
|
||||
agentLevel = smoothedAgent
|
||||
|
||||
let userSpeaking = smoothedMic > Self.userThreshold
|
||||
let next: ConversationPhase
|
||||
if smoothedAgent > Self.agentThreshold {
|
||||
next = .speaking
|
||||
} else if userSpeaking {
|
||||
next = .listening
|
||||
} else {
|
||||
// Both silent: return to listening. The previous implementation
|
||||
// converted speaking → thinking forever, leaving the UI apparently
|
||||
// frozen after an agent response (matches current web reference).
|
||||
next = .listening
|
||||
}
|
||||
setPhase(next)
|
||||
}
|
||||
|
||||
private func setPhase(_ next: ConversationPhase) {
|
||||
guard next != phase else { return }
|
||||
let old = phase
|
||||
logger.log("state \(old.rawValue, privacy: .public) -> \(next.rawValue, privacy: .public)")
|
||||
phase = next
|
||||
lastTransition = Date()
|
||||
if next == .thinking {
|
||||
chipTask?.cancel()
|
||||
chipTask = Task { [weak self] in
|
||||
try? await Task.sleep(for: Self.thinkingDelay)
|
||||
guard !Task.isCancelled, let self else { return }
|
||||
if self.phase == .thinking {
|
||||
self.isThinkingChipVisible = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
chipTask?.cancel()
|
||||
chipTask = nil
|
||||
isThinkingChipVisible = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private let logger = Logger(subsystem: "dev.shane.voice", category: "state")
|
||||
@@ -0,0 +1,130 @@
|
||||
import Foundation
|
||||
import os
|
||||
|
||||
private let logger = Logger(subsystem: "dev.shane.voice", category: "tasks")
|
||||
|
||||
struct TaskStep: Identifiable, Equatable {
|
||||
let id = UUID()
|
||||
let role: String?
|
||||
let content: String?
|
||||
}
|
||||
|
||||
struct TaskItem: Identifiable, Equatable {
|
||||
enum Status: Equatable {
|
||||
case pending
|
||||
case running
|
||||
case completed
|
||||
case failed
|
||||
}
|
||||
|
||||
let id: String
|
||||
var description: String
|
||||
var status: Status
|
||||
var steps: [TaskStep]
|
||||
var result: String?
|
||||
var error: String?
|
||||
var createdAt: Date
|
||||
var completedAt: Date?
|
||||
}
|
||||
|
||||
/// Folds task_event messages into task items, mirroring the web UI's tasks Map.
|
||||
@MainActor
|
||||
final class TaskStore: ObservableObject {
|
||||
@Published private(set) var tasks: [String: TaskItem] = [:]
|
||||
|
||||
var sortedTasks: [TaskItem] {
|
||||
tasks.values.sorted { $0.createdAt > $1.createdAt }
|
||||
}
|
||||
|
||||
var activeCount: Int {
|
||||
tasks.values.filter { $0.status == .pending || $0.status == .running }.count
|
||||
}
|
||||
|
||||
func apply(_ payload: ControlChannel.TaskEventPayload) {
|
||||
guard let taskId = payload.taskId, !taskId.isEmpty else { return }
|
||||
switch payload.type {
|
||||
case "created":
|
||||
tasks[taskId] = TaskItem(
|
||||
id: taskId,
|
||||
description: payload.description ?? "Background task",
|
||||
status: .pending,
|
||||
steps: [],
|
||||
result: nil,
|
||||
error: nil,
|
||||
createdAt: date(from: payload.createdAt),
|
||||
completedAt: nil
|
||||
)
|
||||
case "step":
|
||||
var task = tasks[taskId] ?? TaskItem(
|
||||
id: taskId,
|
||||
description: payload.description ?? "Background task",
|
||||
status: .pending,
|
||||
steps: [],
|
||||
result: nil,
|
||||
error: nil,
|
||||
createdAt: date(from: payload.createdAt),
|
||||
completedAt: nil
|
||||
)
|
||||
task.steps.append(TaskStep(role: payload.role, content: payload.content))
|
||||
if task.status == .pending { task.status = .running }
|
||||
tasks[taskId] = task
|
||||
case "completed":
|
||||
var task = tasks[taskId] ?? TaskItem(
|
||||
id: taskId,
|
||||
description: payload.description ?? "Background task",
|
||||
status: .pending,
|
||||
steps: [],
|
||||
result: nil,
|
||||
error: nil,
|
||||
createdAt: date(from: payload.createdAt),
|
||||
completedAt: nil
|
||||
)
|
||||
task.status = .completed
|
||||
task.result = payload.result
|
||||
task.completedAt = Date()
|
||||
tasks[taskId] = task
|
||||
case "failed":
|
||||
var task = tasks[taskId] ?? TaskItem(
|
||||
id: taskId,
|
||||
description: payload.description ?? "Background task",
|
||||
status: .pending,
|
||||
steps: [],
|
||||
result: nil,
|
||||
error: nil,
|
||||
createdAt: date(from: payload.createdAt),
|
||||
completedAt: nil
|
||||
)
|
||||
task.status = .failed
|
||||
task.error = payload.error
|
||||
task.completedAt = Date()
|
||||
tasks[taskId] = task
|
||||
default:
|
||||
return
|
||||
}
|
||||
logger.log("task \(payload.type, privacy: .public) id=\(taskId, privacy: .public) steps=\(self.tasks[taskId]?.steps.count ?? 0, privacy: .public)")
|
||||
}
|
||||
|
||||
func clear() {
|
||||
tasks.removeAll()
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
func injectSampleEvents() {
|
||||
let now = Date().timeIntervalSince1970
|
||||
apply(ControlChannel.TaskEventPayload(type: "created", taskId: "sample-a", description: "Research the best coffee roasters in Portland", role: nil, content: nil, result: nil, error: nil, createdAt: now - 45))
|
||||
apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-a", description: nil, role: "thinking", content: "Searching for Portland roasteries…", result: nil, error: nil, createdAt: nil))
|
||||
apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-a", description: nil, role: "tool_result", content: "Found 12 results across 3 review sites.", result: nil, error: nil, createdAt: nil))
|
||||
apply(ControlChannel.TaskEventPayload(type: "completed", taskId: "sample-a", description: nil, role: nil, content: nil, result: "Top three: Coava, Stumptown, and Heart Roasters — all scored 4.5+ across reviews.", error: nil, createdAt: nil))
|
||||
apply(ControlChannel.TaskEventPayload(type: "created", taskId: "sample-b", description: "Compare iPhone 17 vs Pixel 10 battery life", role: nil, content: nil, result: nil, error: nil, createdAt: now - 20))
|
||||
apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-b", description: nil, role: "thinking", content: "Fetching benchmark articles…", result: nil, error: nil, createdAt: nil))
|
||||
apply(ControlChannel.TaskEventPayload(type: "failed", taskId: "sample-b", description: nil, role: nil, content: nil, result: nil, error: "Search backend timed out after 30s", createdAt: nil))
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
private extension TaskStore {
|
||||
func date(from interval: Double?) -> Date {
|
||||
guard let interval, interval > 0 else { return Date() }
|
||||
return Date(timeIntervalSince1970: interval)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import Foundation
|
||||
import LiveKit
|
||||
|
||||
struct TranscriptEvent: Equatable, Identifiable {
|
||||
enum Role: String {
|
||||
case user
|
||||
case agent
|
||||
}
|
||||
|
||||
let id: String
|
||||
let role: Role
|
||||
var text: String
|
||||
var isFinal: Bool
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Merges the primary transcription segments (Room delegate) with the
|
||||
/// secondary `transcript` data topic (finals only).
|
||||
///
|
||||
/// **Single canonical collection:** `events` is the one visible transcript.
|
||||
/// It is initialized from the persisted history snapshot at init time and
|
||||
/// updated in place by live events. The view renders only `events`.
|
||||
///
|
||||
/// **Cross-source dedupe:** Records `(role, normalized text, source, timestamp)`
|
||||
/// and suppresses only the *opposite-source* copy within `dedupeWindow` seconds.
|
||||
/// Same-source repeated utterances are never suppressed, even inside the window.
|
||||
@MainActor
|
||||
final class TranscriptHandler: ObservableObject {
|
||||
/// The single canonical visible transcript. Initialized from persisted
|
||||
/// history; live events append/update here. Trimmed to 50 finals.
|
||||
@Published private(set) var events: [TranscriptEvent] = []
|
||||
|
||||
var acceptsRoom: ((Room) -> Bool)?
|
||||
let history: TranscriptHistory
|
||||
|
||||
// MARK: Cross-source dedupe state
|
||||
|
||||
enum Source {
|
||||
case primary // LiveKit transcription segments
|
||||
case data // `transcript` data topic
|
||||
}
|
||||
|
||||
struct RecentFinal {
|
||||
let source: Source
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Keyed by normalized (role, text). Records the most recent final for
|
||||
/// each key and which source produced it. Only the opposite-source copy
|
||||
/// is suppressed within the window; same-source repeats pass through.
|
||||
private var recentFinals: [String: RecentFinal] = [:]
|
||||
static let dedupeWindow: TimeInterval = 5
|
||||
static let maxEvents = 50
|
||||
|
||||
/// Injectable clock for deterministic tests. Defaults to `Date()`.
|
||||
var now: () -> Date = { Date() }
|
||||
|
||||
convenience init() {
|
||||
self.init(history: TranscriptHistory())
|
||||
}
|
||||
|
||||
init(history: TranscriptHistory) {
|
||||
self.history = history
|
||||
// Initialize the canonical events from the persisted snapshot so the
|
||||
// UI shows prior-session transcripts without a separate "restored" store.
|
||||
events = history.entries.map { entry in
|
||||
TranscriptEvent(
|
||||
id: "hist-\(entry.timestamp.timeIntervalSince1970)-\(entry.role)",
|
||||
role: entry.role == "agent" ? .agent : .user,
|
||||
text: entry.text,
|
||||
isFinal: true,
|
||||
timestamp: entry.timestamp
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: Ingest
|
||||
|
||||
func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
guard !text.isEmpty else { return }
|
||||
if isFinal {
|
||||
// Cross-source dedupe: suppress only if the *data* source already
|
||||
// recorded this exact final within the window. Same-source (primary)
|
||||
// repeats are always allowed. When suppressing, consume (remove) the
|
||||
// matched record so a second same-source repeat from primary is not
|
||||
// also suppressed — the next arrival establishes a new pair.
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
if let recent = recentFinals[key],
|
||||
recent.source == .data,
|
||||
now().timeIntervalSince(recent.timestamp) < Self.dedupeWindow {
|
||||
// Consume: remove the matched opposite-source record.
|
||||
recentFinals[key] = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
upsert(id: id, role: role, text: text, isFinal: isFinal)
|
||||
if isFinal {
|
||||
recordFinal(role: role, text: text, source: .primary)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
}
|
||||
}
|
||||
|
||||
func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
|
||||
guard !text.isEmpty else { return }
|
||||
// When a data final arrives for a role that has an open partial, replace
|
||||
// the partial's text with the final and mark it final in place. The data
|
||||
// topic is the authoritative final; the user sees one row either way.
|
||||
if let index = events.lastIndex(where: { !$0.isFinal && $0.role == role }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = true
|
||||
recordFinal(role: role, text: text, source: .data)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
return
|
||||
}
|
||||
// No open partial: check whether the *primary* source already recorded
|
||||
// this exact final within the window (primary→data order). Same-source
|
||||
// (data) repeats are always allowed. When suppressing, consume (remove)
|
||||
// the matched record so a second same-source repeat from data is not
|
||||
// also suppressed — the next arrival establishes a new pair.
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
if let recent = recentFinals[key],
|
||||
recent.source == .primary,
|
||||
now().timeIntervalSince(recent.timestamp) < Self.dedupeWindow {
|
||||
// Consume: remove the matched opposite-source record.
|
||||
recentFinals[key] = nil
|
||||
return
|
||||
}
|
||||
upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
|
||||
recordFinal(role: role, text: text, source: .data)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
}
|
||||
|
||||
/// Mirrors the canonical finals into the persisted snapshot so the UI and
|
||||
/// storage never diverge. Same-source repeats are preserved verbatim.
|
||||
private func syncPersistence() {
|
||||
let finals = events.filter(\.isFinal).map { (role: $0.role.rawValue, text: $0.text, timestamp: $0.timestamp) }
|
||||
history.sync(finals: finals)
|
||||
}
|
||||
|
||||
// MARK: Clear
|
||||
|
||||
/// Clears visible rows, dedupe state, and persistence atomically.
|
||||
func clearAll() {
|
||||
events.removeAll()
|
||||
recentFinals.removeAll()
|
||||
history.clear()
|
||||
}
|
||||
|
||||
// MARK: Private
|
||||
|
||||
private func upsert(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
if let index = events.firstIndex(where: { $0.id == id }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = isFinal
|
||||
} else {
|
||||
events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: now()))
|
||||
}
|
||||
}
|
||||
|
||||
private func recordFinal(role: TranscriptEvent.Role, text: String, source: Source) {
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
recentFinals[key] = RecentFinal(source: source, timestamp: now())
|
||||
// Prune expired entries to bound the dictionary.
|
||||
let current = now()
|
||||
for (k, recent) in recentFinals where current.timeIntervalSince(recent.timestamp) >= Self.dedupeWindow {
|
||||
recentFinals[k] = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps at most `maxEvents` finals in the canonical collection. Partial
|
||||
/// (non-final) rows are never trimmed; only surplus old finals are dropped.
|
||||
private func trimEvents() {
|
||||
let finalCount = events.filter(\.isFinal).count
|
||||
guard finalCount > Self.maxEvents else { return }
|
||||
// Drop oldest finals until we're back to maxEvents.
|
||||
var toRemove = finalCount - Self.maxEvents
|
||||
var writeIndex = 0
|
||||
for event in events {
|
||||
if event.isFinal && toRemove > 0 {
|
||||
toRemove -= 1
|
||||
continue
|
||||
}
|
||||
events[writeIndex] = event
|
||||
writeIndex += 1
|
||||
}
|
||||
events.removeSubrange(writeIndex...)
|
||||
}
|
||||
|
||||
private static func dedupeKey(role: TranscriptEvent.Role, text: String) -> String {
|
||||
let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression)
|
||||
return "\(role.rawValue):\(normalized)"
|
||||
}
|
||||
}
|
||||
|
||||
extension TranscriptHandler: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) {
|
||||
let role: TranscriptEvent.Role = participant.isAgent ? .agent : .user
|
||||
let isLocal = participant.identity?.stringValue == room.localParticipant.identity?.stringValue
|
||||
let resolvedRole: TranscriptEvent.Role = isLocal ? .user : role
|
||||
let events = segments.map {
|
||||
TranscriptEvent(id: $0.id, role: resolvedRole, text: $0.text, isFinal: $0.isFinal, timestamp: $0.lastReceivedTime)
|
||||
}
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.acceptsRoom?(room) ?? true else { return }
|
||||
for event in events {
|
||||
self.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import Foundation
|
||||
|
||||
struct HistoryEntry: Codable, Equatable {
|
||||
let role: String
|
||||
let text: String
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Last 50 final transcript entries persisted as JSON in UserDefaults.
|
||||
///
|
||||
/// Persistence-only store. The visible transcript is the single canonical
|
||||
/// collection owned by `TranscriptHandler.events`; this class provides the
|
||||
/// durable snapshot it initializes from and syncs to. It performs no dedupe —
|
||||
/// all suppression is source-aware and time-bounded in the handler.
|
||||
@MainActor
|
||||
final class TranscriptHistory: ObservableObject {
|
||||
@Published private(set) var entries: [HistoryEntry] = []
|
||||
|
||||
static let key = "voice_history_isolated_tests"
|
||||
private let storageKey: String
|
||||
static let maxEntries = 50
|
||||
|
||||
init(origin: String? = nil, userID: String? = nil) {
|
||||
// Length-prefixed parts prevent ambiguous origin/account combinations.
|
||||
storageKey = origin.flatMap { origin in userID.map { "voice_history_v2:\(origin.utf8.count):\(origin):\($0)" } } ?? Self.key
|
||||
if let raw = AppPreferences.store.string(forKey: storageKey),
|
||||
let data = raw.data(using: .utf8),
|
||||
let decoded = try? JSONDecoder().decode([HistoryEntry].self, from: data) {
|
||||
entries = Array(decoded.suffix(Self.maxEntries))
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces the persisted snapshot with exactly the finals currently in the
|
||||
/// canonical visible collection. Called by `TranscriptHandler` after every
|
||||
/// change so persistence and the UI never diverge. No dedupe is applied here:
|
||||
/// same-source repeats are legitimate entries.
|
||||
func sync(finals: [(role: String, text: String, timestamp: Date)]) {
|
||||
entries = Array(finals.suffix(Self.maxEntries).map {
|
||||
HistoryEntry(role: $0.role, text: $0.text, timestamp: $0.timestamp)
|
||||
})
|
||||
persist()
|
||||
}
|
||||
|
||||
func clear() {
|
||||
entries.removeAll()
|
||||
persist()
|
||||
}
|
||||
|
||||
private func persist() {
|
||||
guard let data = try? JSONEncoder().encode(entries) else { return }
|
||||
AppPreferences.store.set(String(decoding: data, as: UTF8.self), forKey: storageKey)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
import UIKit
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import LiveKit
|
||||
import os
|
||||
|
||||
private let logger = Logger(subsystem: "dev.shane.voice", category: "session")
|
||||
|
||||
enum VoiceSessionState: Equatable {
|
||||
case idle
|
||||
case connecting
|
||||
case waitingAgent
|
||||
case ready
|
||||
/// Failure teardown in progress. Start is unavailable; the session is not
|
||||
/// yet safe for a fresh connect. Transitions to `.failed` when cleanup completes.
|
||||
case tearingDown
|
||||
case failed(String)
|
||||
|
||||
var isFailed: Bool {
|
||||
if case .failed = self { return true }
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
struct AgentInfo: Equatable {
|
||||
let identity: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
/// Room lifecycle: connect, agent discovery, agent_ready wait, reconnect-once,
|
||||
/// disconnect teardown. Ports the web UI's connect flow.
|
||||
@MainActor
|
||||
final class VoiceSession: ObservableObject {
|
||||
@Published private(set) var state: VoiceSessionState = .idle
|
||||
@Published private(set) var agent: AgentInfo?
|
||||
@Published private(set) var isMuted = false
|
||||
@Published private(set) var micError: String?
|
||||
@Published private(set) var isReconnecting = false
|
||||
@Published private(set) var assistantActivity: String?
|
||||
private var workingRequests: [String: String] = [:]
|
||||
|
||||
/// Identity of the current room. Every async callback that could outlive a
|
||||
/// disconnect/reconnect compares its captured generation against this before
|
||||
/// mutating state (port of the web UI's `=== room` checks).
|
||||
private(set) var sessionGeneration = 0
|
||||
private(set) var room = Room()
|
||||
let controlChannel = ControlChannel()
|
||||
let transcriptHandler: TranscriptHandler
|
||||
let stateModel = ConversationStateModel()
|
||||
let taskStore = TaskStore()
|
||||
private let config: ServerConfig
|
||||
private let tokenClient: TokenClient
|
||||
private let settings: SettingsStore
|
||||
private var reconnectAttempted = false
|
||||
private var connectAttempts = 0
|
||||
private var agentWaitTask: Task<Void, Never>?
|
||||
/// Whether the user (or session mode) intentionally wants the microphone
|
||||
/// muted, independent of interruption-owned hardware state. Set by public
|
||||
/// mute actions (`setMuted(true)`, text-mode enable) after the operation
|
||||
/// succeeds; cleared by a successful normal mic enable (`enableMicrophone`).
|
||||
/// Interruption begin operations use this *after* awaiting the predecessor
|
||||
/// to decide whether to mute, rather than snapshotting transient `isMuted`
|
||||
/// before the predecessor completes (Iteration 08 Item 1).
|
||||
private(set) var userWantsMicMuted = false
|
||||
/// Whether the current audio interruption actually muted the mic. Set only
|
||||
/// *after* the mute operation completes successfully; cleared on `.ended`
|
||||
/// and `disconnect()`.
|
||||
private var micMutedByInterruption = false
|
||||
/// Sequence number for interruption-owned operations. Incremented on every
|
||||
/// `.began`; a `.ended` handler captures the sequence at `.began` time and
|
||||
/// only resumes if its sequence is still current (no newer `.began` has
|
||||
/// superseded it). This prevents out-of-order mute/unmute when interruptions
|
||||
/// overlap or arrive in rapid succession.
|
||||
private var interruptionSequence = 0
|
||||
/// The retained operation chain for microphone interruption operations.
|
||||
/// Both `.began` (mute) and `.ended` (unmute) are assigned to this chain so
|
||||
/// every new operation awaits its predecessor. This guarantees:
|
||||
/// - A newer `.began` orders after an in-flight old end/unmute and finishes
|
||||
/// by muting again (the old unmute cannot leave hardware enabled).
|
||||
/// - An `.ended` never launches unretained; it is part of the chain.
|
||||
/// Cancelled on disconnect/failure.
|
||||
private var micOpTask: Task<Void, Never>?
|
||||
/// Set before awaiting `room.disconnect()` during intentional teardown so
|
||||
/// the resulting `.disconnected` callback is recognized as expected (not a
|
||||
/// terminal failure) and ignored. Cleared when the new room connects.
|
||||
private var isTearingDown = false
|
||||
|
||||
init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore, userID: String? = nil) {
|
||||
transcriptHandler = TranscriptHandler(history: TranscriptHistory(origin: config.baseURL.absoluteString, userID: userID))
|
||||
self.config = config
|
||||
self.tokenClient = tokenClient
|
||||
self.settings = settings
|
||||
transcriptHandler.acceptsRoom = { [weak self] source in
|
||||
guard let self else { return false }
|
||||
return self.room === source && !self.isTearingDown && self.state == .ready
|
||||
}
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
controlChannel.onEvent = { [weak self] event in
|
||||
Task { @MainActor [weak self] in
|
||||
self?.handleControlEvent(event)
|
||||
}
|
||||
}
|
||||
NotificationCenter.default.addObserver(
|
||||
self, selector: #selector(handleAudioInterruption),
|
||||
name: AVAudioSession.interruptionNotification, object: nil
|
||||
)
|
||||
}
|
||||
|
||||
deinit {
|
||||
NotificationCenter.default.removeObserver(self)
|
||||
}
|
||||
|
||||
/// Explicit chat-to-Talk navigation tears down the prior room first.
|
||||
func bindConversation(_ id: String?) async {
|
||||
await disconnect()
|
||||
tokenClient.conversationID = id
|
||||
}
|
||||
|
||||
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
|
||||
// Restrict inbound control-event application to active lifecycle states
|
||||
// so late events from a failed/tearing-down room cannot repopulate stale
|
||||
// fields (agent, models, mic state) into a fresh retry (Iteration 08 Item 3).
|
||||
guard !isTearingDown, state == .connecting || state == .waitingAgent || state == .ready else { return }
|
||||
switch event {
|
||||
case .taskEvent(let payload):
|
||||
taskStore.apply(payload)
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var onUnauthorized: (() async -> Void)? {
|
||||
get { tokenClient.onUnauthorized }
|
||||
set { tokenClient.onUnauthorized = newValue }
|
||||
}
|
||||
|
||||
var bootstrapBearerToken: String { tokenClient.bearerToken }
|
||||
|
||||
var isReady: Bool { state == .ready }
|
||||
|
||||
/// Clear all per-room/session ephemeral published fields so the failed or
|
||||
/// idle UI and any subsequent retry do not display stale identity, error,
|
||||
/// mute, model, or reconnect data from a torn-down room.
|
||||
///
|
||||
/// Deliberately preserved across reset: `settings` (durable user
|
||||
/// preferences), `transcriptHandler` events (conversation history), and
|
||||
/// `taskStore` (task history). These are not per-room ephemeral state.
|
||||
///
|
||||
/// Reused from `teardownForFailure` (both the initial pass and the final
|
||||
/// pass immediately before `.failed` publication) and `disconnect()`.
|
||||
private func resetEphemeralFields() {
|
||||
agent = nil
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
workingRequests.removeAll()
|
||||
assistantActivity = nil
|
||||
isReconnecting = false
|
||||
reconnectAttempted = false
|
||||
micMutedByInterruption = false
|
||||
}
|
||||
|
||||
/// Pure predicate: whether a direct `RoomDelegate` callback that mutates
|
||||
/// published ephemeral fields (agent discovery, connection-state changes)
|
||||
/// should be accepted given the current lifecycle state and teardown flag.
|
||||
///
|
||||
/// Callbacks are accepted only for active lifecycle states (`.connecting`,
|
||||
/// `.waitingAgent`, `.ready`) AND only when `isTearingDown` is false. This
|
||||
/// prevents old-room callbacks delivered during failure teardown from
|
||||
/// repopulating ephemeral fields that were just cleared (Iteration 09 Item 1).
|
||||
func shouldAcceptRoomCallback(state: VoiceSessionState, isTearingDown: Bool) -> Bool {
|
||||
!isTearingDown && (state == .connecting || state == .waitingAgent || state == .ready)
|
||||
}
|
||||
|
||||
/// Start (or restart) a session. Accepts `.idle` and `.failed` so the
|
||||
/// Start button can retry after a failure without first resetting to idle.
|
||||
/// Rejects `.tearingDown`: failure cleanup must complete before a fresh
|
||||
/// connect is allowed, preventing the old failure task from tearing down
|
||||
/// the new room.
|
||||
func connect() async {
|
||||
guard state == .idle || state.isFailed else { return }
|
||||
reconnectAttempted = false
|
||||
await start()
|
||||
}
|
||||
|
||||
/// Whether a failure teardown is currently in progress. Exposed for UI and
|
||||
/// test purposes; `connect()` rejects while this is true.
|
||||
var isFailureTearingDown: Bool { state == .tearingDown }
|
||||
|
||||
private func makeRoom() -> Room {
|
||||
let room = Room()
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
return room
|
||||
}
|
||||
|
||||
private func start() async {
|
||||
state = .connecting
|
||||
isReconnecting = false
|
||||
sessionGeneration += 1
|
||||
let generation = sessionGeneration
|
||||
connectAttempts = 0
|
||||
await attemptConnect(generation: generation)
|
||||
}
|
||||
|
||||
private func attemptConnect(generation: Int) async {
|
||||
guard generation == sessionGeneration else { return }
|
||||
let result: Result<Void, Error>
|
||||
do {
|
||||
let token = try await tokenClient.fetchToken()
|
||||
guard generation == sessionGeneration, !Task.isCancelled, !isTearingDown else { return }
|
||||
try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions())
|
||||
result = .success(())
|
||||
} catch {
|
||||
result = .failure(error)
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
switch result {
|
||||
case .success:
|
||||
logger.log("room connected")
|
||||
UIApplication.shared.isIdleTimerDisabled = true
|
||||
scanForAgent()
|
||||
startAgentWait()
|
||||
case .failure(let error):
|
||||
let isTimeout = (error as? LiveKitError)?.type == .timedOut
|
||||
if isTimeout, connectAttempts < 2 {
|
||||
connectAttempts += 1
|
||||
let attempt = connectAttempts
|
||||
logger.log("connect attempt \(attempt) timed out, retrying with fresh room")
|
||||
await teardownRoom()
|
||||
guard generation == sessionGeneration else { return }
|
||||
await attemptConnect(generation: generation)
|
||||
return
|
||||
}
|
||||
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
||||
await teardownForFailure(message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
private func makeConnectOptions() -> ConnectOptions {
|
||||
return ConnectOptions(
|
||||
socketConnectTimeoutInterval: 15,
|
||||
primaryTransportConnectTimeout: 20,
|
||||
publisherTransportConnectTimeout: 20
|
||||
)
|
||||
}
|
||||
|
||||
/// Intentional, idempotent room teardown. Sets the teardown flag *before*
|
||||
/// awaiting `room.disconnect()` so the resulting `.disconnected` callback is
|
||||
/// recognized as expected and ignored (not treated as a terminal failure).
|
||||
/// Safe to call when the room is already disconnected: LiveKit's
|
||||
/// `disconnect()` is a no-op in that case, and the flag is cleared either way.
|
||||
private func teardownRoom() async {
|
||||
isTearingDown = true
|
||||
await room.disconnect()
|
||||
room = makeRoom()
|
||||
isTearingDown = false
|
||||
}
|
||||
|
||||
private func handleAgentTimeout(generation: Int) async {
|
||||
guard generation == sessionGeneration, state == .waitingAgent else { return }
|
||||
guard !reconnectAttempted else {
|
||||
// Final failure: tear down the live room and restore the idle timer.
|
||||
logger.log("agent timeout final, tearing down")
|
||||
await teardownForFailure(message: "Agent never joined. Is the agent worker running?")
|
||||
return
|
||||
}
|
||||
reconnectAttempted = true
|
||||
logger.log("agent did not join in time, retrying with fresh token")
|
||||
await teardownRoom()
|
||||
guard generation == sessionGeneration else { return }
|
||||
await start()
|
||||
}
|
||||
|
||||
/// Centralized failure teardown: mark teardown active before any await so
|
||||
/// `connect()` is rejected during cleanup, cancel wait work, invalidate the
|
||||
/// current generation (so stale callbacks become no-ops), clear all
|
||||
/// per-room/session ephemeral published fields (agent, mute state, model
|
||||
/// list, error state, interruption ownership), disconnect and recreate the
|
||||
/// room, stop conversation state, restore the idle timer, and publish
|
||||
/// `.failed(message)`, making retry safe: the old room is fully torn down
|
||||
/// before Start becomes available again.
|
||||
///
|
||||
/// Deliberately preserved across failure: `settings` (durable user
|
||||
/// preferences), `transcriptHandler` events (conversation history), and
|
||||
/// `taskStore` (task history). These are not per-room ephemeral state.
|
||||
///
|
||||
/// Idempotent: if already tearing down or failed, keeps the first message.
|
||||
private func teardownForFailure(message: String) async {
|
||||
// Mark teardown active *before* any await so connect() rejects during
|
||||
// cleanup and the UI does not expose Start prematurely.
|
||||
if state == .tearingDown || state.isFailed { return }
|
||||
state = .tearingDown
|
||||
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = nil
|
||||
sessionGeneration += 1
|
||||
resetEphemeralFields()
|
||||
// Cancel any in-flight interruption operation and invalidate the
|
||||
// sequence so a pending .ended cannot resume the mic after teardown.
|
||||
micOpTask?.cancel()
|
||||
micOpTask = nil
|
||||
interruptionSequence += 1
|
||||
stateModel.stop()
|
||||
UIApplication.shared.isIdleTimerDisabled = false
|
||||
await teardownRoom()
|
||||
|
||||
// Final reset immediately before publishing .failed: any old-room
|
||||
// callback delivered during the teardownRoom() await (participantDidConnect,
|
||||
// connection-state changes) may have repopulated ephemeral fields. This
|
||||
// second pass erases them so the published .failed state is clean
|
||||
// (Iteration 09 Item 1).
|
||||
resetEphemeralFields()
|
||||
|
||||
// Publish .failed only after all cleanup has completed. This makes the
|
||||
// state a truthful signal: retry is safe because the old room is gone.
|
||||
state = .failed(message)
|
||||
}
|
||||
|
||||
private func startAgentWait() {
|
||||
state = .waitingAgent
|
||||
let generation = sessionGeneration
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = Task { [weak self] in
|
||||
try? await Task.sleep(for: .seconds(10))
|
||||
guard !Task.isCancelled else { return }
|
||||
await self?.handleAgentTimeout(generation: generation)
|
||||
}
|
||||
}
|
||||
|
||||
func agentReadyReceived() {
|
||||
guard !isTearingDown, state == .waitingAgent, sessionGeneration > 0 else {
|
||||
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
||||
return
|
||||
}
|
||||
agentWaitTask?.cancel()
|
||||
state = .ready
|
||||
stateModel.start()
|
||||
let generation = sessionGeneration
|
||||
// Honor the persisted interaction mode: text mode joins with the mic
|
||||
// off and the agent in text mode; voice mode enables the mic.
|
||||
if settings.textMode {
|
||||
isMuted = true
|
||||
userWantsMicMuted = true
|
||||
logger.log("state=ready (text mode), mic stays off")
|
||||
} else {
|
||||
logger.log("state=ready, enabling mic")
|
||||
Task { await enableMicrophone() }
|
||||
}
|
||||
Task { await syncSettingsOnJoin(generation: generation) }
|
||||
}
|
||||
|
||||
private func syncSettingsOnJoin(generation: Int) async {
|
||||
guard generation == sessionGeneration, isReady else { return }
|
||||
// Each sync operation is independent and best-effort: a failed set_voice
|
||||
// must not prevent set_mode from being sent. Hermes is server-managed.
|
||||
let voiceID = settings.voiceID
|
||||
do {
|
||||
try await controlChannel.setVoice(voiceID, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("sync-on-join set_voice failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
// Sync the agent's conversation mode with the persisted setting.
|
||||
let mode = settings.textMode ? "text" : "voice"
|
||||
do {
|
||||
try await controlChannel.setMode(mode, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_mode sent: \(mode, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("sync-on-join set_mode failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func selectVoice(_ id: String) async -> Bool {
|
||||
let previous = settings.voiceID
|
||||
guard isReady else { settings.voiceID = id; return true }
|
||||
let generation = sessionGeneration
|
||||
do {
|
||||
try await controlChannel.setVoice(id, room: room)
|
||||
guard generation == sessionGeneration else { return false }
|
||||
settings.voiceID = id
|
||||
logger.log("set_voice sent: \(id, privacy: .public)")
|
||||
return true
|
||||
} catch {
|
||||
settings.voiceID = previous
|
||||
logger.error("set_voice failed: \(String(describing: error), privacy: .public)")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private func scanForAgent() {
|
||||
for participant in room.remoteParticipants.values {
|
||||
if isAgentParticipant(participant) {
|
||||
foundAgent(participant)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func isAgentParticipant(_ participant: RemoteParticipant) -> Bool {
|
||||
participant.kind == .agent || participant.identity?.stringValue == "voice-assistant"
|
||||
}
|
||||
|
||||
private func foundAgent(_ participant: RemoteParticipant) {
|
||||
agent = AgentInfo(identity: participant.identity?.stringValue ?? "", name: participant.name)
|
||||
}
|
||||
|
||||
func enableMicrophone() async {
|
||||
let predecessor = micOpTask
|
||||
let generation = sessionGeneration
|
||||
let task = Task { @MainActor in
|
||||
await predecessor?.value
|
||||
guard !Task.isCancelled, generation == sessionGeneration, isReady, !settings.textMode else { return }
|
||||
await enableMicrophoneSerialized()
|
||||
}
|
||||
micOpTask = task
|
||||
await task.value
|
||||
}
|
||||
|
||||
private func enableMicrophoneSerialized() async {
|
||||
// Text mode invariant: the microphone must never be enabled while text
|
||||
// mode is active. This enforces the privacy contract at the API level,
|
||||
// not just in the view layer (Finding 4).
|
||||
guard !settings.textMode else {
|
||||
logger.log("enableMicrophone blocked: text mode active")
|
||||
return
|
||||
}
|
||||
let generation = sessionGeneration
|
||||
for attempt in 0..<3 {
|
||||
guard generation == sessionGeneration, isReady, !settings.textMode, !Task.isCancelled else { return }
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: true)
|
||||
guard generation == sessionGeneration else { return }
|
||||
if settings.textMode || Task.isCancelled {
|
||||
try? await room.localParticipant.setMicrophone(enabled: false)
|
||||
isMuted = true
|
||||
return
|
||||
}
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
logger.log("mic enabled")
|
||||
return
|
||||
} catch {
|
||||
let message = String(describing: error)
|
||||
if attempt < 2 {
|
||||
logger.log("mic enable failed (\(message, privacy: .public)), retrying in 1s")
|
||||
try? await Task.sleep(for: .seconds(1))
|
||||
guard generation == sessionGeneration else { return }
|
||||
continue
|
||||
}
|
||||
// Non-fatal: keep the session usable (text mode still works).
|
||||
logger.error("mic enable failed after retries: \(message, privacy: .public)")
|
||||
guard generation == sessionGeneration else { return }
|
||||
micError = "Microphone unavailable (\(error.localizedDescription)). Text mode still works."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setMuted(_ muted: Bool) async {
|
||||
let predecessor = micOpTask
|
||||
let generation = sessionGeneration
|
||||
let task = Task { @MainActor in
|
||||
await predecessor?.value
|
||||
guard !Task.isCancelled, generation == sessionGeneration else { return }
|
||||
await setMutedSerialized(muted)
|
||||
}
|
||||
micOpTask = task
|
||||
await task.value
|
||||
}
|
||||
|
||||
private func setMutedSerialized(_ muted: Bool) async {
|
||||
guard isReady else { return }
|
||||
// Text mode invariant: unmuting (enabling the mic) is not permitted
|
||||
// while text mode is active. Muting is always allowed.
|
||||
if !muted && settings.textMode {
|
||||
logger.log("setMuted(false) blocked: text mode active")
|
||||
return
|
||||
}
|
||||
let generation = sessionGeneration
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||
guard generation == sessionGeneration else { return }
|
||||
if !muted && settings.textMode {
|
||||
try? await room.localParticipant.setMicrophone(enabled: false)
|
||||
isMuted = true
|
||||
return
|
||||
}
|
||||
isMuted = muted
|
||||
// Track explicit user mute intent separately from hardware state.
|
||||
// Interruption begin operations use this after awaiting the
|
||||
// predecessor to decide whether to mute (Iteration 08 Item 1).
|
||||
userWantsMicMuted = muted
|
||||
} catch {
|
||||
guard generation == sessionGeneration else { return }
|
||||
micError = "Microphone error: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
|
||||
func setMode(_ mode: String) async {
|
||||
guard isReady else { return }
|
||||
let generation = sessionGeneration
|
||||
do {
|
||||
try await controlChannel.setMode(mode, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("set_mode sent: \(mode, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("set_mode failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a persisted text-mode change to the live session: mute + set_mode
|
||||
/// when enabling, set_mode + mic on when disabling. Called from
|
||||
/// ConversationView's `.onChange(of: settings.textMode)` so changes from
|
||||
/// either the conversation toggle or the Settings sheet apply to the
|
||||
/// current session.
|
||||
func applyTextMode(_ enabled: Bool) async {
|
||||
guard isReady else { return }
|
||||
guard enabled == settings.textMode else { return }
|
||||
if enabled {
|
||||
await setMuted(true)
|
||||
guard settings.textMode else { return }
|
||||
await setMode("text")
|
||||
} else {
|
||||
await setMode("voice")
|
||||
guard !settings.textMode else { return }
|
||||
await enableMicrophone()
|
||||
}
|
||||
}
|
||||
|
||||
func sendText(_ text: String) async throws {
|
||||
guard isReady else { throw URLError(.notConnectedToInternet) }
|
||||
logger.log("sendText lk.chat chars=\(text.count)")
|
||||
try await room.localParticipant.sendText(text, for: "lk.chat")
|
||||
}
|
||||
|
||||
func disconnect() async {
|
||||
tokenClient.cancel()
|
||||
// Set the teardown guard first so any .disconnected callback from our
|
||||
// own room.disconnect() is ignored (not treated as a terminal failure).
|
||||
isTearingDown = true
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = nil
|
||||
sessionGeneration += 1
|
||||
resetEphemeralFields()
|
||||
// Cancel any in-flight interruption operation and invalidate the
|
||||
// sequence so a pending .ended cannot resume the mic after disconnect.
|
||||
micOpTask?.cancel()
|
||||
micOpTask = nil
|
||||
interruptionSequence += 1
|
||||
stateModel.stop()
|
||||
UIApplication.shared.isIdleTimerDisabled = false
|
||||
await teardownRoom()
|
||||
// Final reset before publishing .idle: any old-room callback delivered
|
||||
// during the teardownRoom() await may have repopulated ephemeral fields.
|
||||
// This ensures the idle state is clean for server replacement (Iteration 09).
|
||||
resetEphemeralFields()
|
||||
// Publish .idle only after room teardown has completed: the old session
|
||||
// is fully torn down before the UI shows "ready to start".
|
||||
state = .idle
|
||||
}
|
||||
|
||||
// MARK: Audio session interruptions (phone call, Siri)
|
||||
|
||||
/// Handles audio session interruptions with a single retained operation
|
||||
/// chain (`micOpTask`) for both begin and end operations.
|
||||
///
|
||||
/// **Serialization invariants (Iteration 07 Item 2):**
|
||||
/// - Every new operation captures the prior `micOpTask` and awaits it before
|
||||
/// executing. This guarantees strict ordering: a newer `.began` orders
|
||||
/// after an in-flight old end/unmute and finishes by muting again, so a
|
||||
/// stale unmute cannot leave hardware enabled during a newer interruption.
|
||||
/// - `.began` increments `interruptionSequence`, then after predecessor
|
||||
/// completion mutes and records ownership (`micMutedByInterruption`) only
|
||||
/// after the mute succeeds.
|
||||
/// - `.ended` is also assigned to the retained chain (never unretained),
|
||||
/// awaits begin, and un-mutes only for matching sequence/generation/
|
||||
/// ownership/shouldResume.
|
||||
/// - Cancel/invalidate on disconnect or failure teardown.
|
||||
@objc private func handleAudioInterruption(_ notification: Notification) {
|
||||
guard let info = notification.userInfo else { return }
|
||||
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
||||
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
||||
switch type {
|
||||
case .began:
|
||||
// Increment the sequence so any pending `.ended` from a prior
|
||||
// interruption sees a stale sequence and does not resume.
|
||||
interruptionSequence += 1
|
||||
let seq = interruptionSequence
|
||||
let generation = sessionGeneration
|
||||
let predecessor = micOpTask
|
||||
|
||||
// Assign the new begin operation to the retained chain. It awaits
|
||||
// the predecessor (which may be an in-flight end/unmute) *before*
|
||||
// deciding whether to mute. The mute decision uses the user's
|
||||
// explicit intent (`userWantsMicMuted`) and current text mode, NOT
|
||||
// a pre-await snapshot of transient `isMuted` which may still
|
||||
// reflect the predecessor's in-flight state (Iteration 08 Item 1).
|
||||
//
|
||||
// This guarantees: begin A → end A/unmute in flight → begin B.
|
||||
// B awaits A's unmute (which sets isMuted=false, userWantsMicMuted
|
||||
// already false from the successful enable). B then sees
|
||||
// userWantsMicMuted == false and textMode == false, so it mutes —
|
||||
// the final state is muted during interruption B.
|
||||
micOpTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
if let pred = predecessor { await pred.value }
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
// Reset ownership: a new interruption supersedes any prior claim.
|
||||
self.micMutedByInterruption = false
|
||||
// Decide whether to mute AFTER the predecessor completes, using
|
||||
// user intent (not a pre-await isMuted snapshot). The mic should
|
||||
// be muted during the interruption unless the user explicitly
|
||||
// wanted it muted already (in which case hardware is already off)
|
||||
// or text mode is active (mic must stay off by invariant).
|
||||
let shouldMute = !self.userWantsMicMuted && !self.settings.textMode
|
||||
&& self.state == .ready
|
||||
guard shouldMute else {
|
||||
logger.log("audio interruption began, no mute needed (userMuted=\(self.userWantsMicMuted), textMode=\(self.settings.textMode), state=\(String(describing: self.state)), seq=\(seq))")
|
||||
return
|
||||
}
|
||||
do {
|
||||
try await self.room.localParticipant.setMicrophone(enabled: false)
|
||||
// Mark ownership only after the mute actually completed and
|
||||
// the sequence is still current (no even newer .began).
|
||||
guard !Task.isCancelled,
|
||||
self.sessionGeneration == generation,
|
||||
seq == self.interruptionSequence else { return }
|
||||
self.isMuted = true
|
||||
self.micMutedByInterruption = true
|
||||
logger.log("interruption mute succeeded (seq=\(seq))")
|
||||
} catch {
|
||||
logger.log("interruption mute failed: \(error.localizedDescription) (seq=\(seq))")
|
||||
}
|
||||
}
|
||||
logger.log("audio interruption began, seq=\(seq)")
|
||||
case .ended:
|
||||
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
||||
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
||||
// Capture the current sequence and generation before any async hop.
|
||||
let currentSeq = interruptionSequence
|
||||
let generation = sessionGeneration
|
||||
let predecessor = micOpTask
|
||||
|
||||
// Assign the end operation to the retained chain (never unretained).
|
||||
// It awaits the predecessor (the begin's mute) before deciding.
|
||||
micOpTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
if let pred = predecessor { await pred.value }
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
// Re-check all conditions after the await:
|
||||
// 1. The mute succeeded (flag set only on success).
|
||||
// 2. No newer .began has superseded this sequence.
|
||||
// 3. The session generation still matches.
|
||||
// 4. Text mode is off.
|
||||
// 5. .shouldResume is present.
|
||||
let shouldUnmute = self.micMutedByInterruption
|
||||
&& currentSeq == self.interruptionSequence
|
||||
&& self.sessionGeneration == generation
|
||||
&& self.state == .ready
|
||||
&& !self.settings.textMode
|
||||
&& shouldResume
|
||||
|
||||
// Clear the flag regardless of whether we resume, so a stale
|
||||
// true cannot cause an incorrect future resume.
|
||||
self.micMutedByInterruption = false
|
||||
|
||||
if shouldUnmute {
|
||||
do {
|
||||
try await self.room.localParticipant.setMicrophone(enabled: true)
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
if self.settings.textMode {
|
||||
try? await self.room.localParticipant.setMicrophone(enabled: false)
|
||||
self.isMuted = true
|
||||
return
|
||||
}
|
||||
self.isMuted = false
|
||||
logger.log("audio interruption ended, resuming mic (seq=\(currentSeq))")
|
||||
} catch {
|
||||
logger.log("interruption resume failed: \(error.localizedDescription) (seq=\(currentSeq))")
|
||||
}
|
||||
} else {
|
||||
let description = String(describing: self.state)
|
||||
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public), shouldResume=\(shouldResume), seq=\(currentSeq))")
|
||||
}
|
||||
}
|
||||
@unknown default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func receiveAssistantProgress(_ data: Data) {
|
||||
struct Progress: Decodable { let type: String; let request_id: String; let status: String; let message: String }
|
||||
guard let progress = try? JSONDecoder().decode(Progress.self, from: data),
|
||||
progress.type == "assistant_progress", !progress.request_id.isEmpty else { return }
|
||||
switch progress.status {
|
||||
case "working": workingRequests[progress.request_id] = progress.message.isEmpty ? "Hope is working on your request…" : progress.message
|
||||
case "completed": workingRequests.removeValue(forKey: progress.request_id)
|
||||
case "failed": workingRequests.removeValue(forKey: progress.request_id)
|
||||
default: return
|
||||
}
|
||||
assistantActivity = workingRequests.values.sorted().first
|
||||
if progress.status == "failed" { assistantActivity = "Hope could not complete that request. Please try again." }
|
||||
}
|
||||
|
||||
func tickAudioLevels(local: Float, remote: Float) {
|
||||
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
||||
}
|
||||
|
||||
/// Pure decision seam for interruption-begin mute logic (Iteration 08 Item 1).
|
||||
/// Extracted from `handleAudioInterruption` so the begin A → end A/unmute →
|
||||
/// begin B transient-state case can be tested deterministically without a
|
||||
/// live microphone or AVAudioSession.
|
||||
///
|
||||
/// Returns `true` if the begin operation should mute the mic, given the
|
||||
/// user's explicit intent and current session state *after* the predecessor
|
||||
/// has completed (not a pre-await snapshot).
|
||||
func interruptionBeginShouldMute(
|
||||
userWantsMicMuted: Bool,
|
||||
textMode: Bool,
|
||||
state: VoiceSessionState
|
||||
) -> Bool {
|
||||
!userWantsMicMuted && !textMode && state == .ready
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
extension VoiceSession: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room else { return }
|
||||
// Gate on active lifecycle states and teardown flag so old-room
|
||||
// callbacks delivered during failure teardown cannot repopulate
|
||||
// the agent field after it was cleared (Iteration 09 Item 1).
|
||||
guard self.shouldAcceptRoomCallback(state: self.state, isTearingDown: self.isTearingDown) else { return }
|
||||
guard self.isAgentParticipant(participant) else { return }
|
||||
self.foundAgent(participant)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room, self.state == .ready else { return }
|
||||
var local = Float(0)
|
||||
var remote = Float(0)
|
||||
for participant in participants {
|
||||
if participant.identity?.stringValue == room.localParticipant.identity?.stringValue {
|
||||
local = max(local, participant.audioLevel)
|
||||
} else {
|
||||
remote = max(remote, participant.audioLevel)
|
||||
}
|
||||
}
|
||||
self.tickAudioLevels(local: local, remote: remote)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room, !self.isTearingDown, self.state == .ready || self.state == .waitingAgent else { return }
|
||||
if topic == "assistant-progress" {
|
||||
self.receiveAssistantProgress(data)
|
||||
return
|
||||
}
|
||||
// Secondary transcript source (finals only).
|
||||
if topic == "transcript",
|
||||
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
||||
json["type"]?.stringValue == "transcript",
|
||||
let text = json["text"]?.stringValue {
|
||||
let role: TranscriptEvent.Role = json["role"]?.stringValue == "agent" ? .agent : .user
|
||||
self.transcriptHandler.receiveDataTranscript(role: role, text: text)
|
||||
return
|
||||
}
|
||||
if self.controlChannel.handleTaskEvent(data) { return }
|
||||
_ = self.controlChannel.handleVoiceControl(data, session: self)
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room else { return }
|
||||
// Ignore all connection-state updates while intentional teardown is
|
||||
// in progress. The caller (teardownForFailure or disconnect) has
|
||||
// already set the correct terminal/transitional state; any callback
|
||||
// from the old room during the disconnect await must not mutate
|
||||
// published fields (Iteration 09 Item 1).
|
||||
if self.isTearingDown {
|
||||
logger.log("connection-state callback ignored during teardown: \(String(describing: connectionState))")
|
||||
return
|
||||
}
|
||||
switch (oldConnectionState, connectionState) {
|
||||
case (.connected, .reconnecting):
|
||||
// Only mutate published fields for active lifecycle states.
|
||||
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
||||
self.isReconnecting = true
|
||||
logger.log("connection lost, waiting for LiveKit reconnect")
|
||||
case (.reconnecting, .connected):
|
||||
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
||||
self.isReconnecting = false
|
||||
logger.log("connection restored")
|
||||
case (_, .disconnected):
|
||||
// Terminal disconnect (not a transient reconnect). If the
|
||||
// session was active, tear down cleanly so no stale room or
|
||||
// wake lock survives. The user can retry via Start.
|
||||
if self.state == .ready || self.state == .waitingAgent {
|
||||
logger.log("terminal disconnect, tearing down session")
|
||||
await self.teardownForFailure(message: "Connection lost.")
|
||||
} else {
|
||||
self.isReconnecting = false
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user