status
This commit is contained in:
1 parent
c9c25f86eb
commit
40aaf4fea7
56 files changed
+5683
-179
No files matched your search
@@ -26,9 +26,23 @@ final class SettingsStore: ObservableObject {
|
||||
static let volume = "volume"
|
||||
}
|
||||
|
||||
static let defaultServerURL = "https://192.168.86.2:9178"
|
||||
|
||||
init() {
|
||||
let d = UserDefaults.standard
|
||||
serverURL = d.string(forKey: Keys.serverURL) ?? "https://192.168.86.2:9178"
|
||||
// Sanitize legacy values: earlier versions wrote the field to storage on
|
||||
// every keystroke, so an invalid half-typed URL could be persisted and
|
||||
// shipped to the network layer. Fall back to the documented default and
|
||||
// repair the stored value once.
|
||||
let storedURL = d.string(forKey: Keys.serverURL) ?? Self.defaultServerURL
|
||||
switch ServerConfig.parse(storedURL) {
|
||||
case .success:
|
||||
serverURL = storedURL
|
||||
case .failure:
|
||||
// Legacy invalid value: repair to the documented default.
|
||||
serverURL = Self.defaultServerURL
|
||||
d.set(Self.defaultServerURL, forKey: Keys.serverURL)
|
||||
}
|
||||
let storedVoice = d.string(forKey: Keys.voiceID)
|
||||
voiceID = Voices.all.contains { $0.id == storedVoice } ? (storedVoice ?? "") : Voices.defaultID
|
||||
modelID = d.string(forKey: Keys.modelID)
|
||||
|
||||
@@ -5,25 +5,47 @@ struct ConversationView: View {
|
||||
@ObservedObject var settings: SettingsStore
|
||||
@State private var sessionID = UUID()
|
||||
@State private var session: VoiceSession
|
||||
/// The exact server URL the current session was constructed with. Used to
|
||||
/// avoid redundant session replacement when the applied URL hasn't actually
|
||||
/// changed (e.g. re-applying the same value from Settings).
|
||||
@State private var activeServerURL: String
|
||||
/// Identity guard: prevents a second `.onChange` from starting a concurrent
|
||||
/// session replacement while the first is still disconnecting the old one.
|
||||
@State private var isReplacingSession = false
|
||||
/// Latest accepted server URL that has not yet been applied to a session.
|
||||
/// Coalesced: rapid Applies during an active replacement retain the newest
|
||||
/// value rather than dropping it. A single retained worker loop drains
|
||||
/// arbitrary pending URLs until none remain (Iteration 08 Item 2).
|
||||
@State private var pendingServerURL: String?
|
||||
@State private var composerText = ""
|
||||
@State private var connectError: String?
|
||||
@State private var tick = 0
|
||||
|
||||
init(settings: SettingsStore) {
|
||||
self.settings = settings
|
||||
_session = State(initialValue: Self.makeSession(settings: settings))
|
||||
let initialSession = Self.makeSession(settings: settings)
|
||||
_session = State(initialValue: initialSession)
|
||||
_activeServerURL = State(initialValue: settings.trimmedServerURL)
|
||||
}
|
||||
|
||||
/// Creates a session from the configured server URL. `SettingsStore.init`
|
||||
/// guarantees the stored value is always valid (legacy invalid values are
|
||||
/// sanitized to the documented default), so the failure branch is a safety
|
||||
/// net only.
|
||||
private static func makeSession(settings: SettingsStore) -> VoiceSession {
|
||||
let config = ServerConfig.parse(settings.trimmedServerURL)
|
||||
if case .success(let value) = config {
|
||||
switch config {
|
||||
case .success(let value):
|
||||
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||
case .failure:
|
||||
// Defensive: should be unreachable because SettingsStore sanitizes.
|
||||
let fallback = ServerConfig.parse(SettingsStore.defaultServerURL)
|
||||
if case .success(let value) = fallback {
|
||||
settings.serverURL = SettingsStore.defaultServerURL
|
||||
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||
}
|
||||
fatalError("Unreachable: default server URL is valid")
|
||||
}
|
||||
let fallback = ServerConfig.parse("https://192.168.86.2:9178")
|
||||
if case .success(let value) = fallback {
|
||||
return VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||
}
|
||||
fatalError("Unreachable: default server URL is valid")
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
@@ -32,8 +54,11 @@ struct ConversationView: View {
|
||||
statusHeader
|
||||
TranscriptListView(
|
||||
events: session.transcriptHandler.events,
|
||||
history: session.transcriptHandler.history,
|
||||
onClear: { session.transcriptHandler.clearHistory() }
|
||||
onClear: {
|
||||
// Atomically clear the canonical visible collection,
|
||||
// dedupe state, and persistence so Clear is truthful.
|
||||
session.transcriptHandler.clearAll()
|
||||
}
|
||||
)
|
||||
composer
|
||||
}
|
||||
@@ -45,17 +70,56 @@ struct ConversationView: View {
|
||||
connectError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
.onChange(of: settings.textMode) { _, newValue in
|
||||
guard session.isReady else { return }
|
||||
Task { await session.applyTextMode(newValue) }
|
||||
}
|
||||
}
|
||||
.onReceive(session.objectWillChange.receive(on: RunLoop.main)) { _ in
|
||||
tick &+= 1
|
||||
}
|
||||
.onChange(of: settings.serverURL) { _, _ in
|
||||
guard session.state == .idle else { return }
|
||||
let oldSession = session
|
||||
.onChange(of: settings.serverURL) { _, newValue in
|
||||
// The URL field now uses explicit Apply semantics (SettingsView),
|
||||
// so this fires only when the user has applied a value that passed
|
||||
// validation. Coalesce rapid accepted values: every accepted Apply
|
||||
// writes `pendingServerURL` (latest wins). A single retained worker
|
||||
// loop drains arbitrary pending URLs until none remain, covering
|
||||
// A→B→C semantics where C arrives while B is disconnecting
|
||||
// (Iteration 08 Item 2).
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty, case .success = ServerConfig.parse(trimmed) else { return }
|
||||
|
||||
pendingServerURL = trimmed
|
||||
|
||||
// Start the worker if not already active. The loop consumes
|
||||
// pending values until empty, so a single Task handles any number
|
||||
// of rapid Applies without nested tasks.
|
||||
guard !isReplacingSession else { return }
|
||||
isReplacingSession = true
|
||||
Task { @MainActor [settings] in
|
||||
await oldSession.disconnect()
|
||||
self.sessionID = UUID()
|
||||
self.session = Self.makeSession(settings: settings)
|
||||
while let url = self.pendingServerURL {
|
||||
self.pendingServerURL = nil
|
||||
guard url != self.activeServerURL else { continue }
|
||||
let oldSession = self.session
|
||||
await oldSession.disconnect()
|
||||
self.activeServerURL = url
|
||||
self.sessionID = UUID()
|
||||
// Construct from the captured validated config, not by
|
||||
// rereading settings.serverURL (which may have changed
|
||||
// during disconnect).
|
||||
switch ServerConfig.parse(url) {
|
||||
case .success(let config):
|
||||
self.session = VoiceSession(config: config, tokenClient: TokenClient(config: config), settings: settings)
|
||||
case .failure:
|
||||
let fallback = ServerConfig.parse(SettingsStore.defaultServerURL)
|
||||
if case .success(let value) = fallback {
|
||||
self.session = VoiceSession(config: value, tokenClient: TokenClient(config: value), settings: settings)
|
||||
} else {
|
||||
fatalError("Unreachable: default server URL is valid")
|
||||
}
|
||||
}
|
||||
}
|
||||
self.isReplacingSession = false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -92,6 +156,8 @@ struct ConversationView: View {
|
||||
}
|
||||
if session.state == .ready {
|
||||
HStack(spacing: 16) {
|
||||
// The mic toggle is disabled in text mode to enforce the
|
||||
// privacy invariant at the UI level as well (Finding 4).
|
||||
Toggle(isOn: Binding(
|
||||
get: { !session.isMuted },
|
||||
set: { newValue in Task { await session.setMuted(!newValue) } }
|
||||
@@ -99,6 +165,7 @@ struct ConversationView: View {
|
||||
Label(session.isMuted ? "Muted" : "Mic on", systemImage: session.isMuted ? "mic.slash" : "mic")
|
||||
}
|
||||
.toggleStyle(.button)
|
||||
.disabled(settings.textMode)
|
||||
Toggle("Text mode", isOn: textModeBinding)
|
||||
.toggleStyle(.button)
|
||||
}
|
||||
@@ -138,24 +205,7 @@ struct ConversationView: View {
|
||||
}
|
||||
|
||||
private var textModeBinding: Binding<Bool> {
|
||||
Binding(
|
||||
get: { settings.textMode },
|
||||
set: { newValue in
|
||||
settings.textMode = newValue
|
||||
Task { await applyTextMode(newValue) }
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private func applyTextMode(_ enabled: Bool) async {
|
||||
guard session.isReady else { return }
|
||||
if enabled {
|
||||
await session.setMuted(true)
|
||||
await session.setMode("text")
|
||||
} else {
|
||||
await session.setMode("voice")
|
||||
await session.enableMicrophone()
|
||||
}
|
||||
$settings.textMode
|
||||
}
|
||||
|
||||
private var composer: some View {
|
||||
@@ -166,7 +216,7 @@ struct ConversationView: View {
|
||||
Label("Start", systemImage: "phone.fill")
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
case .connecting, .waitingAgent:
|
||||
case .connecting, .waitingAgent, .tearingDown:
|
||||
ProgressView()
|
||||
case .ready:
|
||||
TextField(settings.textMode ? "Type a message…" : "Voice mode active",
|
||||
@@ -231,7 +281,7 @@ struct ConversationView: View {
|
||||
private var orbState: OrbView.OrbState {
|
||||
switch session.state {
|
||||
case .idle: return .idle
|
||||
case .connecting, .waitingAgent: return .connecting
|
||||
case .connecting, .waitingAgent, .tearingDown: return .connecting
|
||||
case .ready:
|
||||
guard !settings.textMode else { return .idle }
|
||||
switch session.stateModel.phase {
|
||||
@@ -250,6 +300,7 @@ struct ConversationView: View {
|
||||
case .idle: return "Ready to connect"
|
||||
case .connecting: return "Connecting…"
|
||||
case .waitingAgent: return "Waiting for agent…"
|
||||
case .tearingDown: return "Cleaning up…"
|
||||
case .ready:
|
||||
if settings.textMode { return "Text mode" }
|
||||
switch session.stateModel.phase {
|
||||
@@ -264,7 +315,8 @@ struct ConversationView: View {
|
||||
|
||||
private func startSession() {
|
||||
connectError = nil
|
||||
session.transcriptHandler.clear()
|
||||
// Starting a conversation preserves the restored last-50 history; only
|
||||
// the explicit Clear button erases it.
|
||||
Task { await session.connect() }
|
||||
}
|
||||
|
||||
|
||||
@@ -62,8 +62,10 @@ struct OrbView: View {
|
||||
}
|
||||
|
||||
struct TranscriptListView: View {
|
||||
/// The single canonical visible transcript (restored persisted finals plus
|
||||
/// current live events). The view renders exactly this collection — there is
|
||||
/// no second "restored" store, so the UI and persistence can never diverge.
|
||||
let events: [TranscriptEvent]
|
||||
@ObservedObject var history: TranscriptHistory
|
||||
var onClear: () -> Void
|
||||
|
||||
private static let deliveryMarkers = #"\[(?:appreciative|calm|concerned|curious|encouraging|excited|happy|joking|quiet|reassuring|reflective|sad|serious|surprised|sympathetic|urgent|whispering|laughter|sighing|breathing|neutral)\]\s*"#
|
||||
@@ -75,6 +77,8 @@ struct TranscriptListView: View {
|
||||
return formatter
|
||||
}()
|
||||
|
||||
private var visibleRows: [TranscriptEvent] { events }
|
||||
|
||||
var body: some View {
|
||||
ScrollViewReader { proxy in
|
||||
VStack(alignment: .leading, spacing: 0) {
|
||||
@@ -93,14 +97,14 @@ struct TranscriptListView: View {
|
||||
|
||||
ScrollView {
|
||||
LazyVStack(alignment: .leading, spacing: 8) {
|
||||
ForEach(events) { event in
|
||||
ForEach(visibleRows) { event in
|
||||
transcriptRow(event)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal)
|
||||
}
|
||||
.onChange(of: events.count) { _, count in
|
||||
if let last = events.last {
|
||||
.onChange(of: visibleRows.count) { _, _ in
|
||||
if let last = visibleRows.last {
|
||||
withAnimation { proxy.scrollTo(last.id, anchor: .bottom) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -446,6 +446,9 @@ struct MediaPanelView: View {
|
||||
}
|
||||
|
||||
private var serverConfig: ServerConfig {
|
||||
// Use the configured URL directly. If it's invalid, fall back to the
|
||||
// documented default (media operations are less critical than the main
|
||||
// session, and the Settings view will show a validation error).
|
||||
if case .success(let config) = ServerConfig.parse(settings.trimmedServerURL) {
|
||||
return config
|
||||
}
|
||||
|
||||
@@ -2,26 +2,53 @@ import SwiftUI
|
||||
|
||||
struct SettingsView: View {
|
||||
@ObservedObject var settings: SettingsStore
|
||||
var session: VoiceSession?
|
||||
/// The current voice session, directly observed so the Apply button's
|
||||
/// disabled state and help text update reactively when the session
|
||||
/// transitions (e.g. `.tearingDown` → `.failed`).
|
||||
@ObservedObject var session: VoiceSession
|
||||
/// Local draft of the server URL. Only written to `settings` when the user
|
||||
/// explicitly applies a value that passes validation — keystrokes never
|
||||
/// reach storage or the network layer.
|
||||
@State private var urlText: String = ""
|
||||
@State private var urlError: String?
|
||||
@State private var showVoicePicker = false
|
||||
@State private var showModelPicker = false
|
||||
|
||||
init(settings: SettingsStore, session: VoiceSession? = nil) {
|
||||
init(settings: SettingsStore, session: VoiceSession) {
|
||||
self.settings = settings
|
||||
self.session = session
|
||||
}
|
||||
|
||||
/// Whether the current session is safe to replace. Apply is permitted for
|
||||
/// `.idle` (no active call) and `.failed` (fully torn down after failure —
|
||||
/// the user needs to correct an unreachable server without relaunching).
|
||||
/// Disabled during `.tearingDown`, `.connecting`, `.waitingAgent`, and
|
||||
/// `.ready` to avoid disconnecting an active call or racing cleanup.
|
||||
private var canApplyServerURL: Bool {
|
||||
session.state == .idle || session.state.isFailed
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
Form {
|
||||
Section("Server") {
|
||||
TextField("https://192.168.86.2:9178", text: $urlText)
|
||||
TextField(SettingsStore.defaultServerURL, text: $urlText)
|
||||
.keyboardType(.URL)
|
||||
.autocorrectionDisabled()
|
||||
.onAppear { urlText = settings.serverURL }
|
||||
.onChange(of: urlText) { _, newValue in
|
||||
settings.serverURL = newValue
|
||||
}
|
||||
if let urlError {
|
||||
Text(urlError)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.red)
|
||||
}
|
||||
Button("Apply server URL") { applyServerURL() }
|
||||
.disabled(!canApplyServerURL || urlText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
|
||||
if !canApplyServerURL {
|
||||
Text(session.state == .tearingDown
|
||||
? "Waiting for failure cleanup to finish…"
|
||||
: "End the conversation before changing servers.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
Section("Voice") {
|
||||
Button {
|
||||
@@ -31,14 +58,14 @@ struct SettingsView: View {
|
||||
}
|
||||
}
|
||||
Section("Model") {
|
||||
if let session, !session.models.isEmpty {
|
||||
if session.isReady, !session.models.isEmpty {
|
||||
Button {
|
||||
showModelPicker = true
|
||||
} label: {
|
||||
LabeledContent("Model", value: session.selectedModel ?? settings.modelID ?? "—")
|
||||
}
|
||||
} else {
|
||||
Text(session != nil ? "Loading available models…" : "Start a conversation to load the model list.")
|
||||
Text("Start a conversation to load the model list.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
@@ -60,11 +87,35 @@ struct SettingsView: View {
|
||||
ModelPickerView(settings: settings, session: session)
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the draft and persists it only on success. On failure the
|
||||
/// draft stays in the field with an inline error; `settings.serverURL` is
|
||||
/// left untouched so the active session keeps using its original URL.
|
||||
/// Guarded: only applies when the session is idle (no active call).
|
||||
private func applyServerURL() {
|
||||
guard canApplyServerURL else {
|
||||
urlError = "End the conversation before changing servers."
|
||||
return
|
||||
}
|
||||
let trimmed = urlText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else {
|
||||
urlError = "Enter a server URL."
|
||||
return
|
||||
}
|
||||
switch ServerConfig.parse(trimmed) {
|
||||
case .success:
|
||||
settings.serverURL = trimmed
|
||||
urlText = trimmed
|
||||
urlError = nil
|
||||
case .failure(let error):
|
||||
urlError = error.localizedDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct VoicePickerView: View {
|
||||
@ObservedObject var settings: SettingsStore
|
||||
var session: VoiceSession?
|
||||
var session: VoiceSession
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
|
||||
var body: some View {
|
||||
@@ -73,7 +124,7 @@ struct VoicePickerView: View {
|
||||
ForEach(Voices.all, id: \.id) { voice in
|
||||
Button {
|
||||
settings.voiceID = voice.id
|
||||
Task { await session?.selectVoice(voice.id) }
|
||||
Task { await session.selectVoice(voice.id) }
|
||||
dismiss()
|
||||
} label: {
|
||||
HStack {
|
||||
@@ -99,11 +150,10 @@ struct VoicePickerView: View {
|
||||
|
||||
struct ModelPickerView: View {
|
||||
@ObservedObject var settings: SettingsStore
|
||||
var session: VoiceSession?
|
||||
var session: VoiceSession
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
|
||||
private var activeModelID: String? {
|
||||
guard let session else { return settings.modelID }
|
||||
if !session.models.isEmpty,
|
||||
let selected = session.selectedModel ?? settings.modelID,
|
||||
session.models.contains(where: { $0.id == selected }) {
|
||||
@@ -115,10 +165,10 @@ struct ModelPickerView: View {
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
List {
|
||||
ForEach(session?.models ?? [], id: \.id) { model in
|
||||
ForEach(session.models, id: \.id) { model in
|
||||
Button {
|
||||
settings.modelID = model.id
|
||||
Task { await session?.selectModel(model.id) }
|
||||
Task { await session.selectModel(model.id) }
|
||||
dismiss()
|
||||
} label: {
|
||||
HStack {
|
||||
@@ -143,7 +193,12 @@ struct ModelPickerView: View {
|
||||
}
|
||||
|
||||
#Preview {
|
||||
NavigationStack {
|
||||
SettingsView(settings: SettingsStore())
|
||||
let previewSettings = SettingsStore()
|
||||
guard case .success(let previewConfig) = ServerConfig.parse(SettingsStore.defaultServerURL) else {
|
||||
return AnyView(Text("Invalid default server URL"))
|
||||
}
|
||||
let previewSession = VoiceSession(config: previewConfig, tokenClient: TokenClient(config: previewConfig), settings: previewSettings)
|
||||
return AnyView(NavigationStack {
|
||||
SettingsView(settings: previewSettings, session: previewSession)
|
||||
})
|
||||
}
|
||||
@@ -11,7 +11,9 @@ enum ConversationPhase: String {
|
||||
|
||||
/// Ports the web UI's `inferState` + EMA smoothing (app.js): mic/agent levels
|
||||
/// are smoothed with `prev * 0.6 + raw * 0.4`, thresholds are user > 0.035 and
|
||||
/// agent > 0.02, and speaking → thinking when both go silent.
|
||||
/// agent > 0.02. When remote audio ends, the state returns to listening
|
||||
/// (matching the current web reference at app.js:414-426) rather than becoming
|
||||
/// "thinking" forever.
|
||||
@MainActor
|
||||
final class ConversationStateModel: ObservableObject {
|
||||
@Published private(set) var phase: ConversationPhase = .idle
|
||||
@@ -58,10 +60,10 @@ final class ConversationStateModel: ObservableObject {
|
||||
} else if userSpeaking {
|
||||
next = .listening
|
||||
} else {
|
||||
switch phase {
|
||||
case .speaking: next = .thinking
|
||||
default: next = .listening
|
||||
}
|
||||
// Both silent: return to listening. The previous implementation
|
||||
// converted speaking → thinking forever, leaving the UI apparently
|
||||
// frozen after an agent response (matches current web reference).
|
||||
next = .listening
|
||||
}
|
||||
setPhase(next)
|
||||
}
|
||||
|
||||
@@ -15,50 +15,184 @@ struct TranscriptEvent: Equatable, Identifiable {
|
||||
}
|
||||
|
||||
/// Merges the primary transcription segments (Room delegate) with the
|
||||
/// secondary `transcript` data topic (finals only). Dedupe rule: a final for a
|
||||
/// role closes any open partial row for that role.
|
||||
/// secondary `transcript` data topic (finals only).
|
||||
///
|
||||
/// **Single canonical collection:** `events` is the one visible transcript.
|
||||
/// It is initialized from the persisted history snapshot at init time and
|
||||
/// updated in place by live events. The view renders only `events`.
|
||||
///
|
||||
/// **Cross-source dedupe:** Records `(role, normalized text, source, timestamp)`
|
||||
/// and suppresses only the *opposite-source* copy within `dedupeWindow` seconds.
|
||||
/// Same-source repeated utterances are never suppressed, even inside the window.
|
||||
@MainActor
|
||||
final class TranscriptHandler: ObservableObject {
|
||||
/// The single canonical visible transcript. Initialized from persisted
|
||||
/// history; live events append/update here. Trimmed to 50 finals.
|
||||
@Published private(set) var events: [TranscriptEvent] = []
|
||||
|
||||
let history = TranscriptHistory()
|
||||
let history: TranscriptHistory
|
||||
|
||||
// MARK: Cross-source dedupe state
|
||||
|
||||
enum Source {
|
||||
case primary // LiveKit transcription segments
|
||||
case data // `transcript` data topic
|
||||
}
|
||||
|
||||
struct RecentFinal {
|
||||
let source: Source
|
||||
let timestamp: Date
|
||||
}
|
||||
|
||||
/// Keyed by normalized (role, text). Records the most recent final for
|
||||
/// each key and which source produced it. Only the opposite-source copy
|
||||
/// is suppressed within the window; same-source repeats pass through.
|
||||
private var recentFinals: [String: RecentFinal] = [:]
|
||||
static let dedupeWindow: TimeInterval = 5
|
||||
static let maxEvents = 50
|
||||
|
||||
/// Injectable clock for deterministic tests. Defaults to `Date()`.
|
||||
var now: () -> Date = { Date() }
|
||||
|
||||
convenience init() {
|
||||
self.init(history: TranscriptHistory())
|
||||
}
|
||||
|
||||
init(history: TranscriptHistory) {
|
||||
self.history = history
|
||||
// Initialize the canonical events from the persisted snapshot so the
|
||||
// UI shows prior-session transcripts without a separate "restored" store.
|
||||
events = history.entries.map { entry in
|
||||
TranscriptEvent(
|
||||
id: "hist-\(entry.timestamp.timeIntervalSince1970)-\(entry.role)",
|
||||
role: entry.role == "agent" ? .agent : .user,
|
||||
text: entry.text,
|
||||
isFinal: true,
|
||||
timestamp: entry.timestamp
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: Ingest
|
||||
|
||||
func receiveSegment(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
guard !text.isEmpty else { return }
|
||||
if isFinal {
|
||||
// Cross-source dedupe: suppress only if the *data* source already
|
||||
// recorded this exact final within the window. Same-source (primary)
|
||||
// repeats are always allowed. When suppressing, consume (remove) the
|
||||
// matched record so a second same-source repeat from primary is not
|
||||
// also suppressed — the next arrival establishes a new pair.
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
if let recent = recentFinals[key],
|
||||
recent.source == .data,
|
||||
now().timeIntervalSince(recent.timestamp) < Self.dedupeWindow {
|
||||
// Consume: remove the matched opposite-source record.
|
||||
recentFinals[key] = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
upsert(id: id, role: role, text: text, isFinal: isFinal)
|
||||
if isFinal {
|
||||
history.add(role: role.rawValue, text: text)
|
||||
recordFinal(role: role, text: text, source: .primary)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
}
|
||||
}
|
||||
|
||||
func receiveDataTranscript(role: TranscriptEvent.Role, text: String) {
|
||||
closePartial(for: role)
|
||||
guard !text.isEmpty else { return }
|
||||
// When a data final arrives for a role that has an open partial, replace
|
||||
// the partial's text with the final and mark it final in place. The data
|
||||
// topic is the authoritative final; the user sees one row either way.
|
||||
if let index = events.lastIndex(where: { !$0.isFinal && $0.role == role }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = true
|
||||
recordFinal(role: role, text: text, source: .data)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
return
|
||||
}
|
||||
// No open partial: check whether the *primary* source already recorded
|
||||
// this exact final within the window (primary→data order). Same-source
|
||||
// (data) repeats are always allowed. When suppressing, consume (remove)
|
||||
// the matched record so a second same-source repeat from data is not
|
||||
// also suppressed — the next arrival establishes a new pair.
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
if let recent = recentFinals[key],
|
||||
recent.source == .primary,
|
||||
now().timeIntervalSince(recent.timestamp) < Self.dedupeWindow {
|
||||
// Consume: remove the matched opposite-source record.
|
||||
recentFinals[key] = nil
|
||||
return
|
||||
}
|
||||
upsert(id: "final-\(role.rawValue)-\(UUID().uuidString)", role: role, text: text, isFinal: true)
|
||||
history.add(role: role.rawValue, text: text)
|
||||
recordFinal(role: role, text: text, source: .data)
|
||||
trimEvents()
|
||||
syncPersistence()
|
||||
}
|
||||
|
||||
func clear() {
|
||||
/// Mirrors the canonical finals into the persisted snapshot so the UI and
|
||||
/// storage never diverge. Same-source repeats are preserved verbatim.
|
||||
private func syncPersistence() {
|
||||
let finals = events.filter(\.isFinal).map { (role: $0.role.rawValue, text: $0.text, timestamp: $0.timestamp) }
|
||||
history.sync(finals: finals)
|
||||
}
|
||||
|
||||
// MARK: Clear
|
||||
|
||||
/// Clears visible rows, dedupe state, and persistence atomically.
|
||||
func clearAll() {
|
||||
events.removeAll()
|
||||
}
|
||||
|
||||
func clearHistory() {
|
||||
recentFinals.removeAll()
|
||||
history.clear()
|
||||
}
|
||||
|
||||
private func closePartial(for role: TranscriptEvent.Role) {
|
||||
for index in events.indices where !events[index].isFinal && events[index].role == role {
|
||||
events[index].isFinal = true
|
||||
}
|
||||
}
|
||||
// MARK: Private
|
||||
|
||||
private func upsert(id: String, role: TranscriptEvent.Role, text: String, isFinal: Bool) {
|
||||
if let index = events.firstIndex(where: { $0.id == id }) {
|
||||
events[index].text = text
|
||||
events[index].isFinal = isFinal
|
||||
} else {
|
||||
events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: Date()))
|
||||
events.append(TranscriptEvent(id: id, role: role, text: text, isFinal: isFinal, timestamp: now()))
|
||||
}
|
||||
}
|
||||
|
||||
private func recordFinal(role: TranscriptEvent.Role, text: String, source: Source) {
|
||||
let key = Self.dedupeKey(role: role, text: text)
|
||||
recentFinals[key] = RecentFinal(source: source, timestamp: now())
|
||||
// Prune expired entries to bound the dictionary.
|
||||
let current = now()
|
||||
for (k, recent) in recentFinals where current.timeIntervalSince(recent.timestamp) >= Self.dedupeWindow {
|
||||
recentFinals[k] = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps at most `maxEvents` finals in the canonical collection. Partial
|
||||
/// (non-final) rows are never trimmed; only surplus old finals are dropped.
|
||||
private func trimEvents() {
|
||||
let finalCount = events.filter(\.isFinal).count
|
||||
guard finalCount > Self.maxEvents else { return }
|
||||
// Drop oldest finals until we're back to maxEvents.
|
||||
var toRemove = finalCount - Self.maxEvents
|
||||
var writeIndex = 0
|
||||
for event in events {
|
||||
if event.isFinal && toRemove > 0 {
|
||||
toRemove -= 1
|
||||
continue
|
||||
}
|
||||
events[writeIndex] = event
|
||||
writeIndex += 1
|
||||
}
|
||||
events.removeSubrange(writeIndex...)
|
||||
}
|
||||
|
||||
private static func dedupeKey(role: TranscriptEvent.Role, text: String) -> String {
|
||||
let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression)
|
||||
return "\(role.rawValue):\(normalized)"
|
||||
}
|
||||
}
|
||||
|
||||
extension TranscriptHandler: RoomDelegate {
|
||||
|
||||
@@ -7,12 +7,17 @@ struct HistoryEntry: Codable, Equatable {
|
||||
}
|
||||
|
||||
/// Last 50 final transcript entries persisted as JSON in UserDefaults.
|
||||
///
|
||||
/// Persistence-only store. The visible transcript is the single canonical
|
||||
/// collection owned by `TranscriptHandler.events`; this class provides the
|
||||
/// durable snapshot it initializes from and syncs to. It performs no dedupe —
|
||||
/// all suppression is source-aware and time-bounded in the handler.
|
||||
@MainActor
|
||||
final class TranscriptHistory: ObservableObject {
|
||||
@Published private(set) var entries: [HistoryEntry] = []
|
||||
|
||||
private static let key = "voice_history"
|
||||
private static let maxEntries = 50
|
||||
static let key = "voice_history"
|
||||
static let maxEntries = 50
|
||||
|
||||
init() {
|
||||
if let raw = UserDefaults.standard.string(forKey: Self.key),
|
||||
@@ -22,12 +27,14 @@ final class TranscriptHistory: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
func add(role: String, text: String) {
|
||||
guard !text.isEmpty else { return }
|
||||
entries.append(HistoryEntry(role: role, text: text, timestamp: Date()))
|
||||
if entries.count > Self.maxEntries {
|
||||
entries.removeFirst(entries.count - Self.maxEntries)
|
||||
}
|
||||
/// Replaces the persisted snapshot with exactly the finals currently in the
|
||||
/// canonical visible collection. Called by `TranscriptHandler` after every
|
||||
/// change so persistence and the UI never diverge. No dedupe is applied here:
|
||||
/// same-source repeats are legitimate entries.
|
||||
func sync(finals: [(role: String, text: String, timestamp: Date)]) {
|
||||
entries = Array(finals.suffix(Self.maxEntries).map {
|
||||
HistoryEntry(role: $0.role, text: $0.text, timestamp: $0.timestamp)
|
||||
})
|
||||
persist()
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,15 @@ enum VoiceSessionState: Equatable {
|
||||
case connecting
|
||||
case waitingAgent
|
||||
case ready
|
||||
/// Failure teardown in progress. Start is unavailable; the session is not
|
||||
/// yet safe for a fresh connect. Transitions to `.failed` when cleanup completes.
|
||||
case tearingDown
|
||||
case failed(String)
|
||||
|
||||
var isFailed: Bool {
|
||||
if case .failed = self { return true }
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
struct AgentInfo: Equatable {
|
||||
@@ -47,6 +55,36 @@ final class VoiceSession: ObservableObject {
|
||||
private var reconnectAttempted = false
|
||||
private var connectAttempts = 0
|
||||
private var agentWaitTask: Task<Void, Never>?
|
||||
/// Whether the user (or session mode) intentionally wants the microphone
|
||||
/// muted, independent of interruption-owned hardware state. Set by public
|
||||
/// mute actions (`setMuted(true)`, text-mode enable) after the operation
|
||||
/// succeeds; cleared by a successful normal mic enable (`enableMicrophone`).
|
||||
/// Interruption begin operations use this *after* awaiting the predecessor
|
||||
/// to decide whether to mute, rather than snapshotting transient `isMuted`
|
||||
/// before the predecessor completes (Iteration 08 Item 1).
|
||||
private(set) var userWantsMicMuted = false
|
||||
/// Whether the current audio interruption actually muted the mic. Set only
|
||||
/// *after* the mute operation completes successfully; cleared on `.ended`
|
||||
/// and `disconnect()`.
|
||||
private var micMutedByInterruption = false
|
||||
/// Sequence number for interruption-owned operations. Incremented on every
|
||||
/// `.began`; a `.ended` handler captures the sequence at `.began` time and
|
||||
/// only resumes if its sequence is still current (no newer `.began` has
|
||||
/// superseded it). This prevents out-of-order mute/unmute when interruptions
|
||||
/// overlap or arrive in rapid succession.
|
||||
private var interruptionSequence = 0
|
||||
/// The retained operation chain for microphone interruption operations.
|
||||
/// Both `.began` (mute) and `.ended` (unmute) are assigned to this chain so
|
||||
/// every new operation awaits its predecessor. This guarantees:
|
||||
/// - A newer `.began` orders after an in-flight old end/unmute and finishes
|
||||
/// by muting again (the old unmute cannot leave hardware enabled).
|
||||
/// - An `.ended` never launches unretained; it is part of the chain.
|
||||
/// Cancelled on disconnect/failure.
|
||||
private var micOpTask: Task<Void, Never>?
|
||||
/// Set before awaiting `room.disconnect()` during intentional teardown so
|
||||
/// the resulting `.disconnected` callback is recognized as expected (not a
|
||||
/// terminal failure) and ignored. Cleared when the new room connects.
|
||||
private var isTearingDown = false
|
||||
|
||||
init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore) {
|
||||
self.config = config
|
||||
@@ -74,11 +112,16 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
private func handleControlEvent(_ event: ControlChannel.InboundEvent) {
|
||||
guard state != .idle else { return }
|
||||
// Restrict inbound control-event application to active lifecycle states
|
||||
// so late events from a failed/tearing-down room cannot repopulate stale
|
||||
// fields (agent, models, mic state) into a fresh retry (Iteration 08 Item 3).
|
||||
guard state == .connecting || state == .waitingAgent || state == .ready else { return }
|
||||
switch event {
|
||||
case .models(let received, let selected):
|
||||
models = received
|
||||
if selectedModel == nil, let selected {
|
||||
// The persisted preference is deterministic; an early `models.selected`
|
||||
// event must not silently replace it before sync-on-join has run.
|
||||
if selectedModel == nil, settings.modelID == nil, let selected {
|
||||
selectedModel = selected
|
||||
settings.modelID = selected
|
||||
}
|
||||
@@ -96,11 +139,55 @@ final class VoiceSession: ObservableObject {
|
||||
|
||||
var isReady: Bool { state == .ready }
|
||||
|
||||
/// Clear all per-room/session ephemeral published fields so the failed or
|
||||
/// idle UI and any subsequent retry do not display stale identity, error,
|
||||
/// mute, model, or reconnect data from a torn-down room.
|
||||
///
|
||||
/// Deliberately preserved across reset: `settings` (durable user
|
||||
/// preferences), `transcriptHandler` events (conversation history), and
|
||||
/// `taskStore` (task history). These are not per-room ephemeral state.
|
||||
///
|
||||
/// Reused from `teardownForFailure` (both the initial pass and the final
|
||||
/// pass immediately before `.failed` publication) and `disconnect()`.
|
||||
private func resetEphemeralFields() {
|
||||
agent = nil
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
models = []
|
||||
selectedModel = nil
|
||||
isReconnecting = false
|
||||
reconnectAttempted = false
|
||||
micMutedByInterruption = false
|
||||
}
|
||||
|
||||
/// Pure predicate: whether a direct `RoomDelegate` callback that mutates
|
||||
/// published ephemeral fields (agent discovery, connection-state changes)
|
||||
/// should be accepted given the current lifecycle state and teardown flag.
|
||||
///
|
||||
/// Callbacks are accepted only for active lifecycle states (`.connecting`,
|
||||
/// `.waitingAgent`, `.ready`) AND only when `isTearingDown` is false. This
|
||||
/// prevents old-room callbacks delivered during failure teardown from
|
||||
/// repopulating ephemeral fields that were just cleared (Iteration 09 Item 1).
|
||||
func shouldAcceptRoomCallback(state: VoiceSessionState, isTearingDown: Bool) -> Bool {
|
||||
!isTearingDown && (state == .connecting || state == .waitingAgent || state == .ready)
|
||||
}
|
||||
|
||||
/// Start (or restart) a session. Accepts `.idle` and `.failed` so the
|
||||
/// Start button can retry after a failure without first resetting to idle.
|
||||
/// Rejects `.tearingDown`: failure cleanup must complete before a fresh
|
||||
/// connect is allowed, preventing the old failure task from tearing down
|
||||
/// the new room.
|
||||
func connect() async {
|
||||
guard state == .idle else { return }
|
||||
guard state == .idle || state.isFailed else { return }
|
||||
reconnectAttempted = false
|
||||
await start()
|
||||
}
|
||||
|
||||
/// Whether a failure teardown is currently in progress. Exposed for UI and
|
||||
/// test purposes; `connect()` rejects while this is true.
|
||||
var isFailureTearingDown: Bool { state == .tearingDown }
|
||||
|
||||
private func makeRoom() -> Room {
|
||||
let room = Room()
|
||||
room.add(delegate: self)
|
||||
@@ -149,7 +236,7 @@ final class VoiceSession: ObservableObject {
|
||||
return
|
||||
}
|
||||
logger.error("connect failed: \(String(describing: error), privacy: .public)")
|
||||
state = .failed(error.localizedDescription)
|
||||
await teardownForFailure(message: error.localizedDescription)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,15 +248,25 @@ final class VoiceSession: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
/// Intentional, idempotent room teardown. Sets the teardown flag *before*
|
||||
/// awaiting `room.disconnect()` so the resulting `.disconnected` callback is
|
||||
/// recognized as expected and ignored (not treated as a terminal failure).
|
||||
/// Safe to call when the room is already disconnected: LiveKit's
|
||||
/// `disconnect()` is a no-op in that case, and the flag is cleared either way.
|
||||
private func teardownRoom() async {
|
||||
isTearingDown = true
|
||||
await room.disconnect()
|
||||
room = makeRoom()
|
||||
isTearingDown = false
|
||||
}
|
||||
|
||||
private func handleAgentTimeout(generation: Int) async {
|
||||
guard generation == sessionGeneration, state == .waitingAgent else { return }
|
||||
guard !reconnectAttempted else {
|
||||
state = .failed("Agent never joined. Is the agent worker running?")
|
||||
// Final failure: tear down the live room, restore the idle timer,
|
||||
// and end the media scope so no stale room or wake lock survives.
|
||||
logger.log("agent timeout final, tearing down")
|
||||
await teardownForFailure(message: "Agent never joined. Is the agent worker running?")
|
||||
return
|
||||
}
|
||||
reconnectAttempted = true
|
||||
@@ -179,6 +276,53 @@ final class VoiceSession: ObservableObject {
|
||||
await start()
|
||||
}
|
||||
|
||||
/// Centralized failure teardown: mark teardown active before any await so
|
||||
/// `connect()` is rejected during cleanup, cancel wait work, invalidate the
|
||||
/// current generation (so stale callbacks become no-ops), clear all
|
||||
/// per-room/session ephemeral published fields (agent, mute state, model
|
||||
/// list, error state, interruption ownership), disconnect and recreate the
|
||||
/// room, stop conversation state, restore the idle timer, and end the media
|
||||
/// scope. Only after all cleanup completes does the session publish
|
||||
/// `.failed(message)`, making retry safe: the old room is fully torn down
|
||||
/// before Start becomes available again.
|
||||
///
|
||||
/// Deliberately preserved across failure: `settings` (durable user
|
||||
/// preferences), `transcriptHandler` events (conversation history), and
|
||||
/// `taskStore` (task history). These are not per-room ephemeral state.
|
||||
///
|
||||
/// Idempotent: if already tearing down or failed, keeps the first message.
|
||||
private func teardownForFailure(message: String) async {
|
||||
// Mark teardown active *before* any await so connect() rejects during
|
||||
// cleanup and the UI does not expose Start prematurely.
|
||||
if state == .tearingDown || state.isFailed { return }
|
||||
state = .tearingDown
|
||||
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = nil
|
||||
sessionGeneration += 1
|
||||
resetEphemeralFields()
|
||||
// Cancel any in-flight interruption operation and invalidate the
|
||||
// sequence so a pending .ended cannot resume the mic after teardown.
|
||||
micOpTask?.cancel()
|
||||
micOpTask = nil
|
||||
interruptionSequence += 1
|
||||
stateModel.stop()
|
||||
UIApplication.shared.isIdleTimerDisabled = false
|
||||
await mediaController.endConversation()
|
||||
await teardownRoom()
|
||||
|
||||
// Final reset immediately before publishing .failed: any old-room
|
||||
// callback delivered during the teardownRoom() await (participantDidConnect,
|
||||
// connection-state changes) may have repopulated ephemeral fields. This
|
||||
// second pass erases them so the published .failed state is clean
|
||||
// (Iteration 09 Item 1).
|
||||
resetEphemeralFields()
|
||||
|
||||
// Publish .failed only after all cleanup has completed. This makes the
|
||||
// state a truthful signal: retry is safe because the old room is gone.
|
||||
state = .failed(message)
|
||||
}
|
||||
|
||||
private func startAgentWait() {
|
||||
state = .waitingAgent
|
||||
let generation = sessionGeneration
|
||||
@@ -198,26 +342,52 @@ final class VoiceSession: ObservableObject {
|
||||
agentWaitTask?.cancel()
|
||||
state = .ready
|
||||
stateModel.start()
|
||||
logger.log("state=ready, enabling mic")
|
||||
let generation = sessionGeneration
|
||||
Task { await enableMicrophone() }
|
||||
// Honor the persisted interaction mode: text mode joins with the mic
|
||||
// off and the agent in text mode; voice mode enables the mic.
|
||||
if settings.textMode {
|
||||
isMuted = true
|
||||
userWantsMicMuted = true
|
||||
logger.log("state=ready (text mode), mic stays off")
|
||||
} else {
|
||||
logger.log("state=ready, enabling mic")
|
||||
Task { await enableMicrophone() }
|
||||
}
|
||||
Task { await syncSettingsOnJoin(generation: generation) }
|
||||
}
|
||||
|
||||
private func syncSettingsOnJoin(generation: Int) async {
|
||||
guard generation == sessionGeneration, isReady else { return }
|
||||
// Each sync operation is independent and best-effort: a failed set_voice
|
||||
// or invalid persisted model must not prevent set_mode from being sent.
|
||||
let voiceID = settings.voiceID
|
||||
do {
|
||||
try await controlChannel.setVoice(voiceID, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_voice sent: \(voiceID, privacy: .public)")
|
||||
if let model = selectedModel ?? settings.modelID {
|
||||
} catch {
|
||||
logger.error("sync-on-join set_voice failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
let model = settings.modelID ?? selectedModel
|
||||
if let model {
|
||||
do {
|
||||
try await controlChannel.setModel(model, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("sync-on-join set_model failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
// Sync the agent's conversation mode with the persisted setting.
|
||||
let mode = settings.textMode ? "text" : "voice"
|
||||
do {
|
||||
try await controlChannel.setMode(mode, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_mode sent: \(mode, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("sync-on-join failed: \(String(describing: error), privacy: .public)")
|
||||
logger.error("sync-on-join set_mode failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,12 +435,20 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func enableMicrophone() async {
|
||||
// Text mode invariant: the microphone must never be enabled while text
|
||||
// mode is active. This enforces the privacy contract at the API level,
|
||||
// not just in the view layer (Finding 4).
|
||||
guard !settings.textMode else {
|
||||
logger.log("enableMicrophone blocked: text mode active")
|
||||
return
|
||||
}
|
||||
let generation = sessionGeneration
|
||||
for attempt in 0..<3 {
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: true)
|
||||
guard generation == sessionGeneration else { return }
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
logger.log("mic enabled")
|
||||
return
|
||||
@@ -292,11 +470,21 @@ final class VoiceSession: ObservableObject {
|
||||
|
||||
func setMuted(_ muted: Bool) async {
|
||||
guard isReady else { return }
|
||||
// Text mode invariant: unmuting (enabling the mic) is not permitted
|
||||
// while text mode is active. Muting is always allowed.
|
||||
if !muted && settings.textMode {
|
||||
logger.log("setMuted(false) blocked: text mode active")
|
||||
return
|
||||
}
|
||||
let generation = sessionGeneration
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||
guard generation == sessionGeneration else { return }
|
||||
isMuted = muted
|
||||
// Track explicit user mute intent separately from hardware state.
|
||||
// Interruption begin operations use this after awaiting the
|
||||
// predecessor to decide whether to mute (Iteration 08 Item 1).
|
||||
userWantsMicMuted = muted
|
||||
} catch {
|
||||
guard generation == sessionGeneration else { return }
|
||||
micError = "Microphone error: \(error.localizedDescription)"
|
||||
@@ -315,6 +503,22 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a persisted text-mode change to the live session: mute + set_mode
|
||||
/// when enabling, set_mode + mic on when disabling. Called from
|
||||
/// ConversationView's `.onChange(of: settings.textMode)` so changes from
|
||||
/// either the conversation toggle or the Settings sheet apply to the
|
||||
/// current session.
|
||||
func applyTextMode(_ enabled: Bool) async {
|
||||
guard isReady else { return }
|
||||
if enabled {
|
||||
await setMuted(true)
|
||||
await setMode("text")
|
||||
} else {
|
||||
await setMode("voice")
|
||||
await enableMicrophone()
|
||||
}
|
||||
}
|
||||
|
||||
func sendText(_ text: String) async throws {
|
||||
guard isReady else { throw URLError(.notConnectedToInternet) }
|
||||
logger.log("sendText lk.chat chars=\(text.count)")
|
||||
@@ -322,43 +526,150 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func disconnect() async {
|
||||
// Set the teardown guard first so any .disconnected callback from our
|
||||
// own room.disconnect() is ignored (not treated as a terminal failure).
|
||||
isTearingDown = true
|
||||
agentWaitTask?.cancel()
|
||||
agentWaitTask = nil
|
||||
sessionGeneration += 1
|
||||
state = .idle
|
||||
agent = nil
|
||||
isMuted = false
|
||||
micError = nil
|
||||
models = []
|
||||
isReconnecting = false
|
||||
reconnectAttempted = false
|
||||
resetEphemeralFields()
|
||||
// Cancel any in-flight interruption operation and invalidate the
|
||||
// sequence so a pending .ended cannot resume the mic after disconnect.
|
||||
micOpTask?.cancel()
|
||||
micOpTask = nil
|
||||
interruptionSequence += 1
|
||||
stateModel.stop()
|
||||
UIApplication.shared.isIdleTimerDisabled = false
|
||||
await mediaController.endConversation()
|
||||
await teardownRoom()
|
||||
// Final reset before publishing .idle: any old-room callback delivered
|
||||
// during the teardownRoom() await may have repopulated ephemeral fields.
|
||||
// This ensures the idle state is clean for server replacement (Iteration 09).
|
||||
resetEphemeralFields()
|
||||
// Publish .idle only after media and room teardown have completed.
|
||||
// This makes idle a truthful readiness signal for server replacement:
|
||||
// the old session is fully torn down before the UI shows "ready to start".
|
||||
state = .idle
|
||||
}
|
||||
|
||||
// MARK: Audio session interruptions (phone call, Siri)
|
||||
|
||||
/// Handles audio session interruptions with a single retained operation
|
||||
/// chain (`micOpTask`) for both begin and end operations.
|
||||
///
|
||||
/// **Serialization invariants (Iteration 07 Item 2):**
|
||||
/// - Every new operation captures the prior `micOpTask` and awaits it before
|
||||
/// executing. This guarantees strict ordering: a newer `.began` orders
|
||||
/// after an in-flight old end/unmute and finishes by muting again, so a
|
||||
/// stale unmute cannot leave hardware enabled during a newer interruption.
|
||||
/// - `.began` increments `interruptionSequence`, then after predecessor
|
||||
/// completion mutes and records ownership (`micMutedByInterruption`) only
|
||||
/// after the mute succeeds.
|
||||
/// - `.ended` is also assigned to the retained chain (never unretained),
|
||||
/// awaits begin, and un-mutes only for matching sequence/generation/
|
||||
/// ownership/shouldResume.
|
||||
/// - Cancel/invalidate on disconnect or failure teardown.
|
||||
@objc private func handleAudioInterruption(_ notification: Notification) {
|
||||
guard let info = notification.userInfo else { return }
|
||||
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
||||
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
||||
switch type {
|
||||
case .began:
|
||||
let wasActive = state == .ready && !isMuted && !settings.textMode
|
||||
if wasActive {
|
||||
Task { await setMuted(true) }
|
||||
// Increment the sequence so any pending `.ended` from a prior
|
||||
// interruption sees a stale sequence and does not resume.
|
||||
interruptionSequence += 1
|
||||
let seq = interruptionSequence
|
||||
let generation = sessionGeneration
|
||||
let predecessor = micOpTask
|
||||
|
||||
// Assign the new begin operation to the retained chain. It awaits
|
||||
// the predecessor (which may be an in-flight end/unmute) *before*
|
||||
// deciding whether to mute. The mute decision uses the user's
|
||||
// explicit intent (`userWantsMicMuted`) and current text mode, NOT
|
||||
// a pre-await snapshot of transient `isMuted` which may still
|
||||
// reflect the predecessor's in-flight state (Iteration 08 Item 1).
|
||||
//
|
||||
// This guarantees: begin A → end A/unmute in flight → begin B.
|
||||
// B awaits A's unmute (which sets isMuted=false, userWantsMicMuted
|
||||
// already false from the successful enable). B then sees
|
||||
// userWantsMicMuted == false and textMode == false, so it mutes —
|
||||
// the final state is muted during interruption B.
|
||||
micOpTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
if let pred = predecessor { await pred.value }
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
// Reset ownership: a new interruption supersedes any prior claim.
|
||||
self.micMutedByInterruption = false
|
||||
// Decide whether to mute AFTER the predecessor completes, using
|
||||
// user intent (not a pre-await isMuted snapshot). The mic should
|
||||
// be muted during the interruption unless the user explicitly
|
||||
// wanted it muted already (in which case hardware is already off)
|
||||
// or text mode is active (mic must stay off by invariant).
|
||||
let shouldMute = !self.userWantsMicMuted && !self.settings.textMode
|
||||
&& self.state == .ready
|
||||
guard shouldMute else {
|
||||
logger.log("audio interruption began, no mute needed (userMuted=\(self.userWantsMicMuted), textMode=\(self.settings.textMode), state=\(String(describing: self.state)), seq=\(seq))")
|
||||
return
|
||||
}
|
||||
do {
|
||||
try await self.room.localParticipant.setMicrophone(enabled: false)
|
||||
// Mark ownership only after the mute actually completed and
|
||||
// the sequence is still current (no even newer .began).
|
||||
guard !Task.isCancelled,
|
||||
self.sessionGeneration == generation,
|
||||
seq == self.interruptionSequence else { return }
|
||||
self.isMuted = true
|
||||
self.micMutedByInterruption = true
|
||||
logger.log("interruption mute succeeded (seq=\(seq))")
|
||||
} catch {
|
||||
logger.log("interruption mute failed: \(error.localizedDescription) (seq=\(seq))")
|
||||
}
|
||||
}
|
||||
logger.log("audio interruption began, muted=\(wasActive)")
|
||||
logger.log("audio interruption began, seq=\(seq)")
|
||||
case .ended:
|
||||
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
||||
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
||||
if state == .ready, isMuted, !settings.textMode, shouldResume {
|
||||
Task { await setMuted(false) }
|
||||
logger.log("audio interruption ended, resuming mic")
|
||||
} else {
|
||||
let description = String(describing: state)
|
||||
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public))")
|
||||
// Capture the current sequence and generation before any async hop.
|
||||
let currentSeq = interruptionSequence
|
||||
let generation = sessionGeneration
|
||||
let predecessor = micOpTask
|
||||
|
||||
// Assign the end operation to the retained chain (never unretained).
|
||||
// It awaits the predecessor (the begin's mute) before deciding.
|
||||
micOpTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
if let pred = predecessor { await pred.value }
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
// Re-check all conditions after the await:
|
||||
// 1. The mute succeeded (flag set only on success).
|
||||
// 2. No newer .began has superseded this sequence.
|
||||
// 3. The session generation still matches.
|
||||
// 4. Text mode is off.
|
||||
// 5. .shouldResume is present.
|
||||
let shouldUnmute = self.micMutedByInterruption
|
||||
&& currentSeq == self.interruptionSequence
|
||||
&& self.sessionGeneration == generation
|
||||
&& self.state == .ready
|
||||
&& !self.settings.textMode
|
||||
&& shouldResume
|
||||
|
||||
// Clear the flag regardless of whether we resume, so a stale
|
||||
// true cannot cause an incorrect future resume.
|
||||
self.micMutedByInterruption = false
|
||||
|
||||
if shouldUnmute {
|
||||
do {
|
||||
try await self.room.localParticipant.setMicrophone(enabled: true)
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
self.isMuted = false
|
||||
logger.log("audio interruption ended, resuming mic (seq=\(currentSeq))")
|
||||
} catch {
|
||||
logger.log("interruption resume failed: \(error.localizedDescription) (seq=\(currentSeq))")
|
||||
}
|
||||
} else {
|
||||
let description = String(describing: self.state)
|
||||
logger.log("audio interruption ended, no resume (state=\(description, privacy: .public), shouldResume=\(shouldResume), seq=\(currentSeq))")
|
||||
}
|
||||
}
|
||||
@unknown default:
|
||||
break
|
||||
@@ -369,12 +680,33 @@ final class VoiceSession: ObservableObject {
|
||||
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
||||
}
|
||||
|
||||
/// Pure decision seam for interruption-begin mute logic (Iteration 08 Item 1).
|
||||
/// Extracted from `handleAudioInterruption` so the begin A → end A/unmute →
|
||||
/// begin B transient-state case can be tested deterministically without a
|
||||
/// live microphone or AVAudioSession.
|
||||
///
|
||||
/// Returns `true` if the begin operation should mute the mic, given the
|
||||
/// user's explicit intent and current session state *after* the predecessor
|
||||
/// has completed (not a pre-await snapshot).
|
||||
func interruptionBeginShouldMute(
|
||||
userWantsMicMuted: Bool,
|
||||
textMode: Bool,
|
||||
state: VoiceSessionState
|
||||
) -> Bool {
|
||||
!userWantsMicMuted && !textMode && state == .ready
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
extension VoiceSession: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, participantDidConnect participant: RemoteParticipant) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room, self.isAgentParticipant(participant) else { return }
|
||||
guard let self, self.room === room else { return }
|
||||
// Gate on active lifecycle states and teardown flag so old-room
|
||||
// callbacks delivered during failure teardown cannot repopulate
|
||||
// the agent field after it was cleared (Iteration 09 Item 1).
|
||||
guard self.shouldAcceptRoomCallback(state: self.state, isTearingDown: self.isTearingDown) else { return }
|
||||
guard self.isAgentParticipant(participant) else { return }
|
||||
self.foundAgent(participant)
|
||||
}
|
||||
}
|
||||
@@ -416,13 +748,35 @@ extension VoiceSession: RoomDelegate {
|
||||
nonisolated func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldConnectionState: ConnectionState) {
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room else { return }
|
||||
// Ignore all connection-state updates while intentional teardown is
|
||||
// in progress. The caller (teardownForFailure or disconnect) has
|
||||
// already set the correct terminal/transitional state; any callback
|
||||
// from the old room during the disconnect await must not mutate
|
||||
// published fields (Iteration 09 Item 1).
|
||||
if self.isTearingDown {
|
||||
logger.log("connection-state callback ignored during teardown: \(String(describing: connectionState))")
|
||||
return
|
||||
}
|
||||
switch (oldConnectionState, connectionState) {
|
||||
case (.connected, .reconnecting):
|
||||
// Only mutate published fields for active lifecycle states.
|
||||
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
||||
self.isReconnecting = true
|
||||
logger.log("connection lost, waiting for LiveKit reconnect")
|
||||
case (.reconnecting, .connected):
|
||||
guard self.state == .connecting || self.state == .waitingAgent || self.state == .ready else { return }
|
||||
self.isReconnecting = false
|
||||
logger.log("connection restored")
|
||||
case (_, .disconnected):
|
||||
// Terminal disconnect (not a transient reconnect). If the
|
||||
// session was active, tear down cleanly so no stale room or
|
||||
// wake lock survives. The user can retry via Start.
|
||||
if self.state == .ready || self.state == .waitingAgent {
|
||||
logger.log("terminal disconnect, tearing down session")
|
||||
await self.teardownForFailure(message: "Connection lost.")
|
||||
} else {
|
||||
self.isReconnecting = false
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
|
||||
@@ -182,16 +182,11 @@ final class MediaAPI {
|
||||
try await post("streams", body: ["room": room, "name": name, "url": url])
|
||||
}
|
||||
|
||||
func streamURL(trackID: String) async throws -> URL {
|
||||
let data = try await getRaw(url: baseURL.appendingPathComponent("stream").appending(path: trackID))
|
||||
guard let raw = String(data: data, encoding: .utf8), !raw.isEmpty else {
|
||||
throw MediaAPIError.decodeFailed
|
||||
}
|
||||
if let url = URL(string: raw) { return url }
|
||||
if raw.hasPrefix("/") {
|
||||
return serverBase.appendingPathComponent(String(raw.dropFirst()))
|
||||
}
|
||||
return serverBase.appendingPathComponent(raw)
|
||||
/// Returns the progressive-media endpoint URL for a track. The `/stream/{id}`
|
||||
/// path serves raw media bytes (not a text URL), so this constructs the
|
||||
/// endpoint without downloading or decoding the body.
|
||||
func streamURL(trackID: String) -> URL {
|
||||
baseURL.appendingPathComponent("stream").appending(path: trackID)
|
||||
}
|
||||
|
||||
func healthz() async throws -> Bool {
|
||||
@@ -218,11 +213,24 @@ final class MediaAPI {
|
||||
return data
|
||||
}
|
||||
|
||||
/// Fire-and-forget POST that only validates HTTP success. The media server
|
||||
/// returns `{"ok": true, ...snapshot}` for /play, /control, /report and
|
||||
/// /streams — callers don't need the body, so decoding it (as Data or any
|
||||
/// typed model) would be wasted work at best and a spurious failure mode.
|
||||
private func post(_ path: String, body: [String: Any]) async throws {
|
||||
_ = try await postDecoded(path, body: body, as: Data.self)
|
||||
_ = try await postRaw(path, body: body)
|
||||
}
|
||||
|
||||
private func postDecoded<T: Decodable>(_ path: String, body: [String: Any], as type: T.Type) async throws -> T {
|
||||
let data = try await postRaw(path, body: body)
|
||||
do {
|
||||
return try JSONDecoder().decode(T.self, from: data)
|
||||
} catch {
|
||||
throw MediaAPIError.decodeFailed
|
||||
}
|
||||
}
|
||||
|
||||
private func postRaw(_ path: String, body: [String: Any]) async throws -> Data {
|
||||
var request = URLRequest(url: baseURL.appendingPathComponent(path))
|
||||
request.httpMethod = "POST"
|
||||
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
@@ -236,10 +244,6 @@ final class MediaAPI {
|
||||
}
|
||||
throw MediaAPIError.httpStatus(http.statusCode, detail)
|
||||
}
|
||||
do {
|
||||
return try JSONDecoder().decode(T.self, from: data)
|
||||
} catch {
|
||||
throw MediaAPIError.decodeFailed
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
@@ -28,12 +28,14 @@ final class MediaPlayerController: ObservableObject {
|
||||
private var room: String?
|
||||
private var currentTrackID: String?
|
||||
private var loadedAudioPath: String?
|
||||
private var currentAsset: AVURLAsset?
|
||||
private var appliedSeekTs: Double = 0
|
||||
private var wasPlayingBeforeTalk = false
|
||||
private var lastReportDate = Date.distantPast
|
||||
private var pollTask: Task<Void, Never>?
|
||||
private var timeObserver: Any?
|
||||
private var statusCancellable: AnyCancellable?
|
||||
private var rateObservation: NSKeyValueObservation?
|
||||
|
||||
/// Bumped on every room-scope change so in-flight /state polls from the
|
||||
/// previous scope cannot reconcile into the new one (port of the web UI's
|
||||
@@ -55,22 +57,110 @@ final class MediaPlayerController: ObservableObject {
|
||||
if let timeObserver {
|
||||
player.removeTimeObserver(timeObserver)
|
||||
}
|
||||
rateObservation?.invalidate()
|
||||
NotificationCenter.default.removeObserver(self)
|
||||
}
|
||||
|
||||
/// Whether an audio session interruption is currently active. While true,
|
||||
/// reconciliation must not call player.play() (the next poll would restart
|
||||
/// audio during a phone call or Siri interaction).
|
||||
private var isInterruptionActive = false
|
||||
|
||||
/// Separate resume-suppression flag: set when an interruption ends without
|
||||
/// `.shouldResume` and the pause control to the server has not yet been
|
||||
/// confirmed. While true, reconciliation must not call player.play() even
|
||||
/// after `isInterruptionActive` is cleared. Cleared only by an explicit user
|
||||
/// play/resume action or a confirmed coherent paused state from the server.
|
||||
private var resumeSuppressed = false
|
||||
|
||||
/// Whether playback was actually active (rate > 0) when the interruption
|
||||
/// began. Used to decide whether a `.shouldResume` end should actually
|
||||
/// resume (a track that was already paused before the interruption must not
|
||||
/// be force-resumed).
|
||||
private var wasPlayingBeforeInterruption = false
|
||||
|
||||
/// Sequence number for media interruption operations. Incremented on every
|
||||
/// `.began`; a resume task captures the sequence and rechecks it immediately
|
||||
/// before `player.play()`. A newer `.began` makes an old resume task
|
||||
/// incapable of playing (Iteration 07 Item 5).
|
||||
private var mediaInterruptionSequence = 0
|
||||
|
||||
/// The retained operation chain for media interruption control work. Both
|
||||
/// pause and resume operations are assigned here so a newer begin cancels
|
||||
/// and supersedes prior control work.
|
||||
private var mediaInterruptionTask: Task<Void, Never>?
|
||||
|
||||
@objc private func handleInterruption(_ notification: Notification) {
|
||||
guard let info = notification.userInfo else { return }
|
||||
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
||||
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
||||
switch type {
|
||||
case .began:
|
||||
// Increment the sequence so any in-flight resume task from a prior
|
||||
// interruption sees a stale sequence and cannot play.
|
||||
mediaInterruptionSequence += 1
|
||||
let seq = mediaInterruptionSequence
|
||||
// Cancel/supersede prior control work (resume or pause) so it cannot
|
||||
// complete out of order.
|
||||
mediaInterruptionTask?.cancel()
|
||||
isInterruptionActive = true
|
||||
wasPlayingBeforeInterruption = !player.rate.isZero
|
||||
let wasPlaying = wasPlayingBeforeInterruption
|
||||
player.pause()
|
||||
logger.log("audio interruption began, paused media")
|
||||
logger.log("audio interruption began, paused media (wasPlaying=\(wasPlaying), seq=\(seq))")
|
||||
case .ended:
|
||||
defer { isInterruptionActive = false }
|
||||
applyVolume()
|
||||
if currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
|
||||
player.play()
|
||||
logger.log("audio interruption ended, resumed media")
|
||||
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
||||
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
||||
if shouldResume, wasPlayingBeforeInterruption, currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
|
||||
// Synchronize the server before resuming local playback so the
|
||||
// two never diverge. Capture generation/room/sequence before the
|
||||
// async hop. The task is retained in mediaInterruptionTask so a
|
||||
// newer .began can cancel it.
|
||||
let gen = pollGeneration
|
||||
let targetRoom = room
|
||||
let seq = mediaInterruptionSequence
|
||||
mediaInterruptionTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
if let targetRoom {
|
||||
do {
|
||||
try await self.api.control(room: targetRoom, action: "resume")
|
||||
logger.log("interruption resume: server synchronized (seq=\(seq))")
|
||||
} catch {
|
||||
logger.log("interruption resume: server sync failed (poll will retry): \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
// Recheck the sequence immediately before play: a newer .began
|
||||
// increments mediaInterruptionSequence, making this stale task
|
||||
// incapable of restarting audio during the newer interruption.
|
||||
guard !Task.isCancelled,
|
||||
self.mediaInterruptionSequence == seq,
|
||||
!self.isInterruptionActive,
|
||||
self.pollGeneration == gen,
|
||||
self.room == targetRoom else { return }
|
||||
self.player.play()
|
||||
logger.log("audio interruption ended, resumed media (seq=\(seq))")
|
||||
}
|
||||
} else {
|
||||
// No-resume path: keep suppression even if the pause control
|
||||
// fails. The flag is cleared only by an explicit user play/resume
|
||||
// action or a confirmed coherent paused state from the server.
|
||||
resumeSuppressed = true
|
||||
let seq = mediaInterruptionSequence
|
||||
if let room {
|
||||
mediaInterruptionTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
do {
|
||||
try await self.api.control(room: room, action: "pause")
|
||||
logger.log("audio interruption ended without resume, sent pause to server (seq=\(seq))")
|
||||
} catch {
|
||||
// Keep suppression even on failure — the next poll
|
||||
// will see desired=="paused" and clear it.
|
||||
logger.log("audio interruption ended without resume, pause control failed (suppression retained): \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@unknown default:
|
||||
break
|
||||
@@ -82,6 +172,7 @@ final class MediaPlayerController: ObservableObject {
|
||||
func attach(session: VoiceSession) {
|
||||
self.session = session
|
||||
observePlayer()
|
||||
observePlaybackState()
|
||||
applyVolume()
|
||||
}
|
||||
|
||||
@@ -106,7 +197,10 @@ final class MediaPlayerController: ObservableObject {
|
||||
}
|
||||
|
||||
/// Called when the conversation ends. If music was playing before the talk,
|
||||
/// hand the room back to the player.
|
||||
/// hand the room back to the player. The previous scope (room name) is kept
|
||||
/// so the panel can resume polling it via `startDefaultPolling()`; a fresh
|
||||
/// conversation re-scopes via `enterMediaScope`. Polling restarts here so
|
||||
/// the resumed state stays visible without the panel being open.
|
||||
func endConversation() async {
|
||||
stopPolling()
|
||||
pollGeneration += 1
|
||||
@@ -120,6 +214,16 @@ final class MediaPlayerController: ObservableObject {
|
||||
logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
startPolling()
|
||||
}
|
||||
|
||||
/// Drop the room scope entirely (app-level teardown). After this, polling
|
||||
/// targets the default room again.
|
||||
func resetScope() {
|
||||
stopPolling()
|
||||
pollGeneration += 1
|
||||
room = nil
|
||||
wasPlayingBeforeTalk = false
|
||||
}
|
||||
|
||||
private func startPolling() {
|
||||
@@ -137,10 +241,11 @@ final class MediaPlayerController: ObservableObject {
|
||||
pollTask = nil
|
||||
}
|
||||
|
||||
/// Start polling the default room when no session is active. Called from
|
||||
/// the Media Panel so play/stop state is visible without a voice session.
|
||||
/// Resume polling whenever the panel becomes visible and nothing is being
|
||||
/// polled yet — whether that's the default room (no session) or a previous
|
||||
/// conversation's room scope (kept after `endConversation`).
|
||||
func startDefaultPolling() {
|
||||
guard room == nil else { return }
|
||||
guard pollTask == nil else { return }
|
||||
startPolling()
|
||||
}
|
||||
|
||||
@@ -161,9 +266,16 @@ final class MediaPlayerController: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
private func reconcile(_ s: MediaState) {
|
||||
/// Reconciles the local player against a server state snapshot. Exposed
|
||||
/// as internal (not private) so tests can drive deterministic state
|
||||
/// transitions without a live server.
|
||||
func reconcile(_ s: MediaState) {
|
||||
choices = s.choices
|
||||
pendingJobs = s.pending
|
||||
// Keep the published desired state in sync with the server so the UI's
|
||||
// play/pause icon and status text reflect reality (the server is the
|
||||
// source of truth for per-room desired state).
|
||||
desired = s.desired ?? "stopped"
|
||||
|
||||
let track = s.track
|
||||
if let track, track.id != currentTrackID {
|
||||
@@ -189,14 +301,37 @@ final class MediaPlayerController: ObservableObject {
|
||||
}
|
||||
|
||||
private func applyDesiredState(_ s: MediaState, track: MediaTrack?) {
|
||||
if s.desired == "playing" {
|
||||
// audio_url presence means a playable stream exists — including a
|
||||
// still-growing download, which plays progressively.
|
||||
guard let desiredValue = s.desired else { return }
|
||||
switch desiredValue {
|
||||
case "playing":
|
||||
// Suppress play during an active audio interruption (phone call,
|
||||
// Siri). Without this guard the next 1.5s poll would restart audio
|
||||
// immediately after .began paused it. The player.rate.isZero check
|
||||
// makes this idempotent: once playing, subsequent polls are no-ops.
|
||||
// This also handles the late-audio case where desired=playing
|
||||
// arrives before audio_url is set; when the source is later wired
|
||||
// via wireSource(), the next poll retries play because rate is
|
||||
// still zero.
|
||||
// Also suppress while resumeSuppressed is true (no-resume
|
||||
// interruption ended, server not yet confirmed paused).
|
||||
guard !isInterruptionActive, !resumeSuppressed else { return }
|
||||
if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero {
|
||||
player.play()
|
||||
}
|
||||
} else if !player.rate.isZero {
|
||||
player.pause()
|
||||
case "paused", "stopped":
|
||||
if !player.rate.isZero {
|
||||
player.pause()
|
||||
}
|
||||
// The server has confirmed a coherent paused/stopped state. If we
|
||||
// were suppressing resume due to a no-resume interruption, the
|
||||
// pause control succeeded (or the user paused independently), so
|
||||
// it is safe to clear the suppression flag.
|
||||
if resumeSuppressed {
|
||||
resumeSuppressed = false
|
||||
logger.log("resume suppression cleared: server confirmed \(desiredValue)")
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,6 +343,7 @@ final class MediaPlayerController: ObservableObject {
|
||||
} else {
|
||||
// Downloading or failed: no stream yet. Reconcile wires the stream
|
||||
// URL in via polling as soon as one is available.
|
||||
clearCurrentAsset()
|
||||
player.replaceCurrentItem(with: nil)
|
||||
}
|
||||
}
|
||||
@@ -216,13 +352,27 @@ final class MediaPlayerController: ObservableObject {
|
||||
guard let url = URL(string: path) else { return }
|
||||
loadedAudioPath = path
|
||||
let pinned = mediaLoader.rewrite(url)
|
||||
// Remove the previous asset's loader mapping before wiring a new one so
|
||||
// the loaderURLs dictionary stays bounded (Finding 11 / Item 7).
|
||||
clearCurrentAsset()
|
||||
let asset = mediaLoader.assetWithPinnedURL(pinned)
|
||||
currentAsset = asset
|
||||
logger.log("wiring source \(pinned.absoluteString, privacy: .public)")
|
||||
let item = AVPlayerItem(asset: asset)
|
||||
observeItem(item)
|
||||
player.replaceCurrentItem(with: item)
|
||||
}
|
||||
|
||||
/// Removes the loader mapping for the current asset (if any) and clears the
|
||||
/// reference. Called before wiring a new source or clearing media so the
|
||||
/// resource loader's internal dictionaries stay bounded.
|
||||
private func clearCurrentAsset() {
|
||||
if let asset = currentAsset {
|
||||
mediaLoader.removeLoader(asset)
|
||||
currentAsset = nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: Player events
|
||||
|
||||
private func observePlayer() {
|
||||
@@ -248,6 +398,38 @@ final class MediaPlayerController: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Playback state observation
|
||||
|
||||
/// Observe the player's actual rate so every real play/pause transition is
|
||||
/// reported promptly, mirroring the web UI's `play`/`pause` listeners. The
|
||||
/// KVO block runs on an arbitrary queue; hop to MainActor before touching
|
||||
/// any controller state.
|
||||
private func observePlaybackState() {
|
||||
rateObservation = player.observe(\.rate, options: [.initial, .new]) { [weak self] player, _ in
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
guard self.currentTrackID != nil else { return }
|
||||
if player.rate > 0 {
|
||||
// Actual playback began (user play, interruption resume, or
|
||||
// reconcile-driven play). Report + publish the transition.
|
||||
self.lastReportDate = Date()
|
||||
self.report("playing")
|
||||
self.publishMediaState("playing")
|
||||
} else if player.currentItem != nil {
|
||||
// Rate dropped to zero with an item still loaded: a pause
|
||||
// that didn't originate from a user action (interruption,
|
||||
// reconcile). User-initiated pauses already report before
|
||||
// this fires; the lastReportDate guard makes duplicates no-ops.
|
||||
if Date().timeIntervalSince(self.lastReportDate) >= 1 {
|
||||
self.lastReportDate = Date()
|
||||
self.report("paused")
|
||||
self.publishMediaState("paused")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func observeItem(_ item: AVPlayerItem?) {
|
||||
NotificationCenter.default.removeObserver(
|
||||
self, name: .AVPlayerItemDidPlayToEndTime, object: nil
|
||||
@@ -270,6 +452,7 @@ final class MediaPlayerController: ObservableObject {
|
||||
}
|
||||
|
||||
@objc private func playerItemDidEnd(_ notification: Notification) {
|
||||
lastReportDate = Date()
|
||||
report("stopped")
|
||||
publishMediaState("stopped")
|
||||
Task { [weak self] in
|
||||
@@ -280,29 +463,38 @@ final class MediaPlayerController: ObservableObject {
|
||||
|
||||
// MARK: Reporting + publishing
|
||||
|
||||
private func report(_ state: String) {
|
||||
private func report(_ state: String, trackID: String? = nil) {
|
||||
guard let room else { return }
|
||||
let position = currentTime.isFinite ? currentTime : 0
|
||||
var durationValue: Double?
|
||||
if duration.isFinite, duration > 0 {
|
||||
durationValue = duration
|
||||
}
|
||||
let resolvedTrackID = trackID ?? currentTrackID
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
do {
|
||||
try await self.api.report(room: room, state: state, trackID: self.currentTrackID, position: position, duration: durationValue)
|
||||
try await self.api.report(room: room, state: state, trackID: resolvedTrackID, position: position, duration: durationValue)
|
||||
} catch {
|
||||
logger.log("report failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func publishMediaState(_ state: String) {
|
||||
private func publishMediaState(_ state: String, trackID: String? = nil) {
|
||||
guard let session else { return }
|
||||
// Capture the exact room and generation before spawning so a delayed
|
||||
// publish from an old lifecycle cannot escape into a new conversation
|
||||
// room (Finding 10).
|
||||
let targetRoom = session.room
|
||||
let generation = session.sessionGeneration
|
||||
let pollGen = pollGeneration
|
||||
let resolvedTrackID = trackID ?? currentTrackID
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
guard let self, self.pollGeneration == pollGen else { return }
|
||||
guard session.sessionGeneration == generation, session.room === targetRoom else { return }
|
||||
do {
|
||||
try await session.controlChannel.sendMediaState(state, trackID: self.currentTrackID, room: session.room)
|
||||
try await session.controlChannel.sendMediaState(state, trackID: resolvedTrackID, room: targetRoom)
|
||||
} catch {
|
||||
logger.log("media_state publish failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
@@ -336,9 +528,17 @@ final class MediaPlayerController: ObservableObject {
|
||||
func togglePlayPause() async {
|
||||
guard currentTrackID != nil, loadedAudioPath != nil else { return }
|
||||
if player.rate.isZero {
|
||||
// Explicit user play: clear any resume suppression (the user is
|
||||
// deliberately overriding the no-resume interruption state).
|
||||
resumeSuppressed = false
|
||||
// The rate observer reports + publishes the actual transition;
|
||||
// the control call updates server desired state.
|
||||
player.play()
|
||||
if let room { try? await api.control(room: room, action: "resume") }
|
||||
} else {
|
||||
lastReportDate = Date()
|
||||
report("paused")
|
||||
publishMediaState("paused")
|
||||
player.pause()
|
||||
if let room { try? await api.control(room: room, action: "pause") }
|
||||
}
|
||||
@@ -350,24 +550,31 @@ final class MediaPlayerController: ObservableObject {
|
||||
if target.isFinite {
|
||||
seek(to: target)
|
||||
}
|
||||
report(player.rate.isZero ? "paused" : "playing")
|
||||
}
|
||||
|
||||
func seek(to position: Double) {
|
||||
player.seek(to: CMTime(seconds: position, preferredTimescale: 600))
|
||||
currentTime = position
|
||||
report(player.rate.isZero ? "paused" : "playing")
|
||||
}
|
||||
|
||||
func stop() async {
|
||||
// Capture the track ID before clearing so the async report/publish
|
||||
// tasks can read a non-nil value even after clearMedia() runs.
|
||||
let trackID = currentTrackID
|
||||
report("stopped", trackID: trackID)
|
||||
clearMedia()
|
||||
if let room { try? await api.control(room: room, action: "stop") }
|
||||
publishMediaState("stopped")
|
||||
publishMediaState("stopped", trackID: trackID)
|
||||
}
|
||||
|
||||
private func clearMedia() {
|
||||
currentTrackID = nil
|
||||
track = nil
|
||||
loadedAudioPath = nil
|
||||
// Remove the loader mapping for the asset being cleared so the
|
||||
// resource loader's internal dictionaries stay bounded (Item 7).
|
||||
clearCurrentAsset()
|
||||
observeItem(nil)
|
||||
player.replaceCurrentItem(with: nil)
|
||||
currentTime = 0
|
||||
|
||||
@@ -9,17 +9,52 @@ private let logger = Logger(subsystem: "dev.shane.voice", category: "media")
|
||||
/// (ignoring registered URLProtocol subclasses), so the asset uses a custom
|
||||
/// `pinned-media` scheme and this delegate answers every data request by
|
||||
/// fetching the real https URL through TLSPinning.session(), forwarding Range.
|
||||
///
|
||||
/// **Lifecycle invariants (Finding 11 / Iteration 06 Item 7):**
|
||||
/// - Tasks are keyed by `ObjectIdentifier(loadingRequest)` directly, not a
|
||||
/// derived hash, so two distinct requests can never collide.
|
||||
/// - All mutable state (`loaderURLs`, `activeTasks`) is accessed only on the
|
||||
/// main queue: the AVAssetResourceLoader delegate is set with `queue: .main`
|
||||
/// and the pinned URLSession uses `delegateQueue: .main`, so both the
|
||||
/// resource-loading callbacks and the data-task completion handlers run on
|
||||
/// the same serialization domain. This is enforced in code, not by comment.
|
||||
/// - For HTTP 206, `contentLength` is set only when `Content-Range` provides a
|
||||
/// known total; it never falls back to the range's `expectedContentLength`.
|
||||
/// - For HTTP 200, `expectedContentLength` is already the full resource size.
|
||||
/// - Cleanup (task removal) occurs on success, error, cancellation, and item
|
||||
/// replacement via `removeLoader`.
|
||||
final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
|
||||
static let scheme = "pinned-media"
|
||||
|
||||
private let base: URL
|
||||
/// Maps each asset's resource loader to the real https URL it proxies.
|
||||
/// Accessed only on the main queue (the delegate queue).
|
||||
private var loaderURLs: [ObjectIdentifier: URL] = [:]
|
||||
/// Maps each loading request's identity to its upstream data task so
|
||||
/// cancellation can cancel the in-flight download. Keyed directly by
|
||||
/// `ObjectIdentifier(loadingRequest)` — no hash derivation, no collisions.
|
||||
private var activeTasks: [ObjectIdentifier: URLSessionDataTask] = [:]
|
||||
/// Shared pinned session, created once and invalidated on deinit.
|
||||
/// Uses `delegateQueue: .main` so all completion handlers (and therefore
|
||||
/// all mutations of `activeTasks`) run on the main queue — the same
|
||||
/// serialization domain as the AVAssetResourceLoader delegate callbacks
|
||||
/// (which are also dispatched to `.main`). This makes map access truly
|
||||
/// serialized in code, not just by convention.
|
||||
private lazy var pinnedSession: URLSession = {
|
||||
let config = URLSessionConfiguration.default
|
||||
config.timeoutIntervalForRequest = 30
|
||||
return URLSession(configuration: config, delegate: PinnedTrustDelegate(), delegateQueue: .main)
|
||||
}()
|
||||
|
||||
init(base: URL) {
|
||||
self.base = base
|
||||
}
|
||||
|
||||
deinit {
|
||||
pinnedSession.invalidateAndCancel()
|
||||
}
|
||||
|
||||
/// Maps a media URL (absolute https or /media/... path) to the pinned-media
|
||||
/// scheme so AVPlayer routes it through this delegate instead of loading
|
||||
/// https natively. Built as a string because URLComponents.url returns nil
|
||||
@@ -47,17 +82,38 @@ final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
return c.url ?? url
|
||||
}
|
||||
|
||||
/// Creates an AVURLAsset that routes through this loader's delegate. The
|
||||
/// mapping from the asset's resource loader to the real URL is recorded so
|
||||
/// `resourceLoader(_:shouldWaitForLoadingOfRequestedResource:)` can resolve
|
||||
/// it. All mutations happen on the main queue (the delegate queue).
|
||||
func assetWithPinnedURL(_ pinned: URL) -> AVURLAsset {
|
||||
let asset = AVURLAsset(url: pinned)
|
||||
loaderURLs[ObjectIdentifier(asset.resourceLoader)] = resolve(pinned)
|
||||
let realURL = resolve(pinned)
|
||||
// The delegate queue is .main, so this runs on the main thread.
|
||||
loaderURLs[ObjectIdentifier(asset.resourceLoader)] = realURL
|
||||
asset.resourceLoader.setDelegate(self, queue: .main)
|
||||
return asset
|
||||
}
|
||||
|
||||
/// Removes the loader mapping and any in-flight tasks for an asset that is
|
||||
/// no longer needed. Called by MediaPlayerController when replacing or
|
||||
/// clearing media items to bound the dictionaries (Finding 11).
|
||||
func removeLoader(_ asset: AVURLAsset) {
|
||||
let key = ObjectIdentifier(asset.resourceLoader)
|
||||
loaderURLs[key] = nil
|
||||
// Cancel any in-flight tasks. URLSessionDataTask has no isCancelled
|
||||
// property; calling cancel() on an already-cancelled task is a no-op.
|
||||
for (_, task) in activeTasks {
|
||||
task.cancel()
|
||||
}
|
||||
activeTasks.removeAll()
|
||||
}
|
||||
|
||||
func resourceLoader(
|
||||
_ resourceLoader: AVAssetResourceLoader,
|
||||
shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest
|
||||
) -> Bool {
|
||||
// All state access is on the main queue (the delegate queue).
|
||||
guard let dataRequest = loadingRequest.dataRequest else { return false }
|
||||
guard let url = loaderURLs[ObjectIdentifier(resourceLoader)] else {
|
||||
logger.log("no resolved URL for resource loader")
|
||||
@@ -75,7 +131,16 @@ final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
}
|
||||
logger.log("fetching \(url.absoluteString, privacy: .public) range=\(upstream.value(forHTTPHeaderField: "Range") ?? "-", privacy: .public)")
|
||||
|
||||
let task = TLSPinning.session().dataTask(with: upstream) { data, response, error in
|
||||
// Key the task by the loading request's identity directly — no hash
|
||||
// derivation, so two distinct requests can never collide.
|
||||
let requestKey = ObjectIdentifier(loadingRequest)
|
||||
let task = pinnedSession.dataTask(with: upstream) { [weak self] data, response, error in
|
||||
guard let self else { return }
|
||||
// Cleanup: remove the task from the active map regardless of outcome.
|
||||
// Runs on .main (the session's delegateQueue), same domain as all
|
||||
// other map mutations.
|
||||
self.activeTasks[requestKey] = nil
|
||||
|
||||
if let error {
|
||||
logger.log("fetch failed \(url.absoluteString, privacy: .public): \(error.localizedDescription, privacy: .public)")
|
||||
loadingRequest.finishLoading(with: error)
|
||||
@@ -85,10 +150,33 @@ final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
loadingRequest.finishLoading(with: URLError(.badServerResponse))
|
||||
return
|
||||
}
|
||||
// Reject non-200/206 responses (404, 500, etc.) instead of feeding
|
||||
// error JSON into AVPlayer as media bytes.
|
||||
guard http.statusCode == 200 || http.statusCode == 206 else {
|
||||
logger.log("unexpected status \(http.statusCode) for \(url.absoluteString, privacy: .public)")
|
||||
loadingRequest.finishLoading(with: URLError(.badServerResponse))
|
||||
return
|
||||
}
|
||||
if let info = loadingRequest.contentInformationRequest {
|
||||
info.contentType = http.mimeType ?? "application/octet-stream"
|
||||
if http.expectedContentLength >= 0 {
|
||||
info.contentLength = Int64(http.expectedContentLength)
|
||||
// Content-length handling (Iteration 05 Item 7):
|
||||
// - For 206: set full content length ONLY when Content-Range
|
||||
// provides a known total. Never fall back to the range's
|
||||
// expectedContentLength (that would tell AVPlayer the total
|
||||
// resource is only as large as this one range).
|
||||
// - For 200: expectedContentLength is already the full size.
|
||||
if http.statusCode == 206 {
|
||||
if let total = Self.parseTotalLength(http), total >= 0 {
|
||||
info.contentLength = total
|
||||
}
|
||||
// If Content-Range has no known total (e.g. "bytes 0-1023/*"),
|
||||
// do NOT set contentLength — AVPlayer will handle it as
|
||||
// unknown-length, which is correct for a range response.
|
||||
} else {
|
||||
// 200: expectedContentLength is the full resource size.
|
||||
if http.expectedContentLength >= 0 {
|
||||
info.contentLength = Int64(http.expectedContentLength)
|
||||
}
|
||||
}
|
||||
info.isByteRangeAccessSupported = true
|
||||
}
|
||||
@@ -97,6 +185,7 @@ final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
}
|
||||
loadingRequest.finishLoading()
|
||||
}
|
||||
activeTasks[requestKey] = task
|
||||
task.resume()
|
||||
return true
|
||||
}
|
||||
@@ -105,6 +194,27 @@ final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate {
|
||||
_ resourceLoader: AVAssetResourceLoader,
|
||||
didCancel loadingRequest: AVAssetResourceLoadingRequest
|
||||
) {
|
||||
logger.log("resource loader cancelled request")
|
||||
// Cancel the upstream data task so a seek or item swap doesn't keep
|
||||
// downloading cancelled ranges. Keyed by ObjectIdentifier directly.
|
||||
let key = ObjectIdentifier(loadingRequest)
|
||||
activeTasks[key]?.cancel()
|
||||
activeTasks[key] = nil
|
||||
logger.log("resource loader cancelled request, upstream task cancelled")
|
||||
}
|
||||
|
||||
/// Parses the total resource length from an HTTP 206 response's
|
||||
/// `Content-Range` header (e.g. "bytes 0-1023/4096" → 4096). Returns nil
|
||||
/// for non-206 responses, missing headers, unknown totals ("*"), or
|
||||
/// malformed values.
|
||||
static func parseTotalLength(_ response: HTTPURLResponse) -> Int64? {
|
||||
if response.statusCode == 206,
|
||||
let contentRange = response.value(forHTTPHeaderField: "Content-Range") {
|
||||
// Format: "bytes <start>-<end>/<total>" or "bytes <start>-<end>/*"
|
||||
let afterSlash = contentRange.split(separator: "/").last?.trimmingCharacters(in: .whitespaces)
|
||||
if let afterSlash, afterSlash != "*", let total = Int64(afterSlash) {
|
||||
return total
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -44,10 +44,21 @@ final class PinnedTrustDelegate: NSObject, URLSessionDelegate {
|
||||
completionHandler(.performDefaultHandling, nil)
|
||||
return
|
||||
}
|
||||
if TLSPinning.matchesPinnedCert(trust) {
|
||||
completionHandler(.useCredential, URLCredential(trust: trust))
|
||||
// Apply the fixed self-signed pin only to the pinned LAN deployment
|
||||
// host. Other explicitly configured HTTPS hosts use normal system
|
||||
// trust evaluation (Finding 12). This allows public deployments and
|
||||
// other LAN hosts with valid certificates while never weakening trust
|
||||
// for the default self-signed host.
|
||||
let host = challenge.protectionSpace.host
|
||||
if host == TLSPinning.host {
|
||||
if TLSPinning.matchesPinnedCert(trust) {
|
||||
completionHandler(.useCredential, URLCredential(trust: trust))
|
||||
} else {
|
||||
completionHandler(.cancelAuthenticationChallenge, nil)
|
||||
}
|
||||
} else {
|
||||
completionHandler(.cancelAuthenticationChallenge, nil)
|
||||
// Non-pinned host: let the system evaluate the certificate chain.
|
||||
completionHandler(.performDefaultHandling, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user