Archived
feat: authorize iOS Hermes accounts and secure session isolation
This commit is contained in:
snowspeeder committed
1 parent
9bdaea2408
commit
bebaf59466
23 files changed
+911
-185
No files matched your search
@@ -6,15 +6,8 @@ import LiveKit
|
||||
@MainActor
|
||||
final class ControlChannel: NSObject, RoomDelegate {
|
||||
|
||||
struct ModelInfo: Equatable {
|
||||
let id: String
|
||||
let name: String?
|
||||
}
|
||||
|
||||
enum InboundEvent {
|
||||
case agentReady
|
||||
case models([ModelInfo], selected: String?)
|
||||
case modelChanged(String)
|
||||
case taskEvent(TaskEventPayload)
|
||||
}
|
||||
|
||||
@@ -43,22 +36,6 @@ final class ControlChannel: NSObject, RoomDelegate {
|
||||
session?.agentReadyReceived()
|
||||
}
|
||||
return true
|
||||
case "models":
|
||||
var models: [ModelInfo] = []
|
||||
for item in json["models"]?.arrayValue ?? [] {
|
||||
if let obj = item.objectValue, let id = obj["id"]?.stringValue {
|
||||
models.append(ModelInfo(id: id, name: obj["name"]?.stringValue))
|
||||
} else if let id = item.stringValue {
|
||||
models.append(ModelInfo(id: id, name: nil))
|
||||
}
|
||||
}
|
||||
onEvent?(.models(models, selected: json["selected"]?.stringValue))
|
||||
return true
|
||||
case "model_changed":
|
||||
if let model = json["model"]?.stringValue {
|
||||
onEvent?(.modelChanged(model))
|
||||
}
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -97,14 +74,6 @@ final class ControlChannel: NSObject, RoomDelegate {
|
||||
try await publish(["type": "set_voice", "voice": voiceID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setModel(_ modelID: String, room: Room) async throws {
|
||||
try await publish(["type": "set_model", "model": modelID], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func getModels(room: Room) async throws {
|
||||
try await publish(["type": "get_models"], topic: "voice-control", room: room)
|
||||
}
|
||||
|
||||
func setMode(_ mode: String, room: Room) async throws {
|
||||
try await publish(["type": "set_mode", "mode": mode], topic: "conversation-control", room: room)
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ final class TranscriptHandler: ObservableObject {
|
||||
/// history; live events append/update here. Trimmed to 50 finals.
|
||||
@Published private(set) var events: [TranscriptEvent] = []
|
||||
|
||||
var acceptsRoom: ((Room) -> Bool)?
|
||||
let history: TranscriptHistory
|
||||
|
||||
// MARK: Cross-source dedupe state
|
||||
@@ -204,8 +205,9 @@ extension TranscriptHandler: RoomDelegate {
|
||||
TranscriptEvent(id: $0.id, role: resolvedRole, text: $0.text, isFinal: $0.isFinal, timestamp: $0.lastReceivedTime)
|
||||
}
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.acceptsRoom?(room) ?? true else { return }
|
||||
for event in events {
|
||||
self?.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal)
|
||||
self.receiveSegment(id: event.id, role: event.role, text: event.text, isFinal: event.isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,14 @@ struct HistoryEntry: Codable, Equatable {
|
||||
final class TranscriptHistory: ObservableObject {
|
||||
@Published private(set) var entries: [HistoryEntry] = []
|
||||
|
||||
static let key = "voice_history"
|
||||
static let key = "voice_history_isolated_tests"
|
||||
private let storageKey: String
|
||||
static let maxEntries = 50
|
||||
|
||||
init() {
|
||||
if let raw = AppPreferences.store.string(forKey: Self.key),
|
||||
init(origin: String? = nil, userID: String? = nil) {
|
||||
// Length-prefixed parts prevent ambiguous origin/account combinations.
|
||||
storageKey = origin.flatMap { origin in userID.map { "voice_history_v2:\(origin.utf8.count):\(origin):\($0)" } } ?? Self.key
|
||||
if let raw = AppPreferences.store.string(forKey: storageKey),
|
||||
let data = raw.data(using: .utf8),
|
||||
let decoded = try? JSONDecoder().decode([HistoryEntry].self, from: data) {
|
||||
entries = Array(decoded.suffix(Self.maxEntries))
|
||||
@@ -45,6 +48,6 @@ final class TranscriptHistory: ObservableObject {
|
||||
|
||||
private func persist() {
|
||||
guard let data = try? JSONEncoder().encode(entries) else { return }
|
||||
AppPreferences.store.set(String(decoding: data, as: UTF8.self), forKey: Self.key)
|
||||
AppPreferences.store.set(String(decoding: data, as: UTF8.self), forKey: storageKey)
|
||||
}
|
||||
}
|
||||
@@ -36,8 +36,8 @@ final class VoiceSession: ObservableObject {
|
||||
@Published private(set) var isMuted = false
|
||||
@Published private(set) var micError: String?
|
||||
@Published private(set) var isReconnecting = false
|
||||
@Published private(set) var models: [ControlChannel.ModelInfo] = []
|
||||
@Published private(set) var selectedModel: String?
|
||||
@Published private(set) var assistantActivity: String?
|
||||
private var workingRequests: [String: String] = [:]
|
||||
|
||||
/// Identity of the current room. Every async callback that could outlive a
|
||||
/// disconnect/reconnect compares its captured generation against this before
|
||||
@@ -45,7 +45,7 @@ final class VoiceSession: ObservableObject {
|
||||
private(set) var sessionGeneration = 0
|
||||
private(set) var room = Room()
|
||||
let controlChannel = ControlChannel()
|
||||
let transcriptHandler = TranscriptHandler()
|
||||
let transcriptHandler: TranscriptHandler
|
||||
let stateModel = ConversationStateModel()
|
||||
let taskStore = TaskStore()
|
||||
private let config: ServerConfig
|
||||
@@ -85,10 +85,15 @@ final class VoiceSession: ObservableObject {
|
||||
/// terminal failure) and ignored. Cleared when the new room connects.
|
||||
private var isTearingDown = false
|
||||
|
||||
init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore) {
|
||||
init(config: ServerConfig, tokenClient: TokenClient, settings: SettingsStore, userID: String? = nil) {
|
||||
transcriptHandler = TranscriptHandler(history: TranscriptHistory(origin: config.baseURL.absoluteString, userID: userID))
|
||||
self.config = config
|
||||
self.tokenClient = tokenClient
|
||||
self.settings = settings
|
||||
transcriptHandler.acceptsRoom = { [weak self] source in
|
||||
guard let self else { return false }
|
||||
return self.room === source && !self.isTearingDown && self.state == .ready
|
||||
}
|
||||
room.add(delegate: self)
|
||||
room.add(delegate: controlChannel)
|
||||
room.add(delegate: transcriptHandler)
|
||||
@@ -111,21 +116,8 @@ final class VoiceSession: ObservableObject {
|
||||
// Restrict inbound control-event application to active lifecycle states
|
||||
// so late events from a failed/tearing-down room cannot repopulate stale
|
||||
// fields (agent, models, mic state) into a fresh retry (Iteration 08 Item 3).
|
||||
guard state == .connecting || state == .waitingAgent || state == .ready else { return }
|
||||
guard !isTearingDown, state == .connecting || state == .waitingAgent || state == .ready else { return }
|
||||
switch event {
|
||||
case .models(let received, let selected):
|
||||
models = received
|
||||
// The persisted preference is deterministic; an early `models.selected`
|
||||
// event must not silently replace it before sync-on-join has run.
|
||||
if selectedModel == nil, settings.modelID == nil, let selected {
|
||||
selectedModel = selected
|
||||
settings.modelID = selected
|
||||
}
|
||||
logger.log("models received count=\(received.count) selected=\(selected ?? "nil", privacy: .public)")
|
||||
case .modelChanged(let model):
|
||||
selectedModel = model
|
||||
settings.modelID = model
|
||||
logger.log("model_changed applied: \(model, privacy: .public)")
|
||||
case .taskEvent(let payload):
|
||||
taskStore.apply(payload)
|
||||
default:
|
||||
@@ -133,6 +125,13 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
var onUnauthorized: (() async -> Void)? {
|
||||
get { tokenClient.onUnauthorized }
|
||||
set { tokenClient.onUnauthorized = newValue }
|
||||
}
|
||||
|
||||
var bootstrapBearerToken: String { tokenClient.bearerToken }
|
||||
|
||||
var isReady: Bool { state == .ready }
|
||||
|
||||
/// Clear all per-room/session ephemeral published fields so the failed or
|
||||
@@ -150,8 +149,8 @@ final class VoiceSession: ObservableObject {
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
models = []
|
||||
selectedModel = nil
|
||||
workingRequests.removeAll()
|
||||
assistantActivity = nil
|
||||
isReconnecting = false
|
||||
reconnectAttempted = false
|
||||
micMutedByInterruption = false
|
||||
@@ -206,7 +205,7 @@ final class VoiceSession: ObservableObject {
|
||||
let result: Result<Void, Error>
|
||||
do {
|
||||
let token = try await tokenClient.fetchToken()
|
||||
logger.log("token ok room=\(token.room, privacy: .public)")
|
||||
guard generation == sessionGeneration, !Task.isCancelled, !isTearingDown else { return }
|
||||
try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions())
|
||||
result = .success(())
|
||||
} catch {
|
||||
@@ -327,7 +326,7 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func agentReadyReceived() {
|
||||
guard state == .waitingAgent, sessionGeneration > 0 else {
|
||||
guard !isTearingDown, state == .waitingAgent, sessionGeneration > 0 else {
|
||||
logger.log("agent_ready ignored in state \(String(describing: self.state), privacy: .public)")
|
||||
return
|
||||
}
|
||||
@@ -351,7 +350,7 @@ final class VoiceSession: ObservableObject {
|
||||
private func syncSettingsOnJoin(generation: Int) async {
|
||||
guard generation == sessionGeneration, isReady else { return }
|
||||
// Each sync operation is independent and best-effort: a failed set_voice
|
||||
// or invalid persisted model must not prevent set_mode from being sent.
|
||||
// must not prevent set_mode from being sent. Hermes is server-managed.
|
||||
let voiceID = settings.voiceID
|
||||
do {
|
||||
try await controlChannel.setVoice(voiceID, room: room)
|
||||
@@ -361,17 +360,6 @@ final class VoiceSession: ObservableObject {
|
||||
logger.error("sync-on-join set_voice failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
let model = settings.modelID ?? selectedModel
|
||||
if let model {
|
||||
do {
|
||||
try await controlChannel.setModel(model, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("sync-on-join set_model sent: \(model, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("sync-on-join set_model failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
guard generation == sessionGeneration else { return }
|
||||
// Sync the agent's conversation mode with the persisted setting.
|
||||
let mode = settings.textMode ? "text" : "voice"
|
||||
do {
|
||||
@@ -396,19 +384,6 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
func selectModel(_ id: String) async {
|
||||
settings.modelID = id
|
||||
guard isReady else { return }
|
||||
let generation = sessionGeneration
|
||||
do {
|
||||
try await controlChannel.setModel(id, room: room)
|
||||
guard generation == sessionGeneration else { return }
|
||||
logger.log("set_model sent: \(id, privacy: .public)")
|
||||
} catch {
|
||||
logger.error("set_model failed: \(String(describing: error), privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
private func scanForAgent() {
|
||||
for participant in room.remoteParticipants.values {
|
||||
if isAgentParticipant(participant) {
|
||||
@@ -427,6 +402,18 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func enableMicrophone() async {
|
||||
let predecessor = micOpTask
|
||||
let generation = sessionGeneration
|
||||
let task = Task { @MainActor in
|
||||
await predecessor?.value
|
||||
guard !Task.isCancelled, generation == sessionGeneration, isReady, !settings.textMode else { return }
|
||||
await enableMicrophoneSerialized()
|
||||
}
|
||||
micOpTask = task
|
||||
await task.value
|
||||
}
|
||||
|
||||
private func enableMicrophoneSerialized() async {
|
||||
// Text mode invariant: the microphone must never be enabled while text
|
||||
// mode is active. This enforces the privacy contract at the API level,
|
||||
// not just in the view layer (Finding 4).
|
||||
@@ -436,9 +423,15 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
let generation = sessionGeneration
|
||||
for attempt in 0..<3 {
|
||||
guard generation == sessionGeneration, isReady, !settings.textMode, !Task.isCancelled else { return }
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: true)
|
||||
guard generation == sessionGeneration else { return }
|
||||
if settings.textMode || Task.isCancelled {
|
||||
try? await room.localParticipant.setMicrophone(enabled: false)
|
||||
isMuted = true
|
||||
return
|
||||
}
|
||||
isMuted = false
|
||||
userWantsMicMuted = false
|
||||
micError = nil
|
||||
@@ -461,6 +454,18 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func setMuted(_ muted: Bool) async {
|
||||
let predecessor = micOpTask
|
||||
let generation = sessionGeneration
|
||||
let task = Task { @MainActor in
|
||||
await predecessor?.value
|
||||
guard !Task.isCancelled, generation == sessionGeneration else { return }
|
||||
await setMutedSerialized(muted)
|
||||
}
|
||||
micOpTask = task
|
||||
await task.value
|
||||
}
|
||||
|
||||
private func setMutedSerialized(_ muted: Bool) async {
|
||||
guard isReady else { return }
|
||||
// Text mode invariant: unmuting (enabling the mic) is not permitted
|
||||
// while text mode is active. Muting is always allowed.
|
||||
@@ -472,6 +477,11 @@ final class VoiceSession: ObservableObject {
|
||||
do {
|
||||
try await room.localParticipant.setMicrophone(enabled: !muted)
|
||||
guard generation == sessionGeneration else { return }
|
||||
if !muted && settings.textMode {
|
||||
try? await room.localParticipant.setMicrophone(enabled: false)
|
||||
isMuted = true
|
||||
return
|
||||
}
|
||||
isMuted = muted
|
||||
// Track explicit user mute intent separately from hardware state.
|
||||
// Interruption begin operations use this after awaiting the
|
||||
@@ -502,11 +512,14 @@ final class VoiceSession: ObservableObject {
|
||||
/// current session.
|
||||
func applyTextMode(_ enabled: Bool) async {
|
||||
guard isReady else { return }
|
||||
guard enabled == settings.textMode else { return }
|
||||
if enabled {
|
||||
await setMuted(true)
|
||||
guard settings.textMode else { return }
|
||||
await setMode("text")
|
||||
} else {
|
||||
await setMode("voice")
|
||||
guard !settings.textMode else { return }
|
||||
await enableMicrophone()
|
||||
}
|
||||
}
|
||||
@@ -518,6 +531,7 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
|
||||
func disconnect() async {
|
||||
tokenClient.cancel()
|
||||
// Set the teardown guard first so any .disconnected callback from our
|
||||
// own room.disconnect() is ignored (not treated as a terminal failure).
|
||||
isTearingDown = true
|
||||
@@ -651,6 +665,11 @@ final class VoiceSession: ObservableObject {
|
||||
do {
|
||||
try await self.room.localParticipant.setMicrophone(enabled: true)
|
||||
guard !Task.isCancelled, self.sessionGeneration == generation else { return }
|
||||
if self.settings.textMode {
|
||||
try? await self.room.localParticipant.setMicrophone(enabled: false)
|
||||
self.isMuted = true
|
||||
return
|
||||
}
|
||||
self.isMuted = false
|
||||
logger.log("audio interruption ended, resuming mic (seq=\(currentSeq))")
|
||||
} catch {
|
||||
@@ -666,6 +685,20 @@ final class VoiceSession: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
func receiveAssistantProgress(_ data: Data) {
|
||||
struct Progress: Decodable { let type: String; let request_id: String; let status: String; let message: String }
|
||||
guard let progress = try? JSONDecoder().decode(Progress.self, from: data),
|
||||
progress.type == "assistant_progress", !progress.request_id.isEmpty else { return }
|
||||
switch progress.status {
|
||||
case "working": workingRequests[progress.request_id] = progress.message.isEmpty ? "Hope is working on your request…" : progress.message
|
||||
case "completed": workingRequests.removeValue(forKey: progress.request_id)
|
||||
case "failed": workingRequests.removeValue(forKey: progress.request_id)
|
||||
default: return
|
||||
}
|
||||
assistantActivity = workingRequests.values.sorted().first
|
||||
if progress.status == "failed" { assistantActivity = "Hope could not complete that request. Please try again." }
|
||||
}
|
||||
|
||||
func tickAudioLevels(local: Float, remote: Float) {
|
||||
stateModel.tick(localAudioLevel: local, remoteAudioLevel: remote)
|
||||
}
|
||||
@@ -718,9 +751,12 @@ extension VoiceSession: RoomDelegate {
|
||||
}
|
||||
|
||||
nonisolated func room(_ room: Room, participant: RemoteParticipant?, didReceiveData data: Data, forTopic topic: String, encryptionType: EncryptionType) {
|
||||
logger.log("data topic=\(topic, privacy: .public) bytes=\(data.count) payload=\(String(data: data.prefix(300), encoding: .utf8) ?? "<binary>", privacy: .public)")
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self, self.room === room else { return }
|
||||
guard let self, self.room === room, !self.isTearingDown, self.state == .ready || self.state == .waitingAgent else { return }
|
||||
if topic == "assistant-progress" {
|
||||
self.receiveAssistantProgress(data)
|
||||
return
|
||||
}
|
||||
// Secondary transcript source (finals only).
|
||||
if topic == "transcript",
|
||||
let json = try? JSONDecoder().decode([String: JSONValue].self, from: data),
|
||||
|
||||
Reference in new issue
Block a user