From af296da6b8b003073e1f5bf27c3917dc59fac365 Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sun, 30 Aug 2026 08:48:28 -0400 Subject: [PATCH] feat: background tasks panel (Phase 7) --- PLAN.md | 23 ++- .../Conversation/ConversationView.swift | 17 +++ Sources/Features/Tasks/TaskPanelView.swift | 137 ++++++++++++++++++ Sources/LiveKit/ControlChannel.swift | 18 ++- Sources/LiveKit/TaskStore.swift | 130 +++++++++++++++++ Sources/LiveKit/VoiceSession.swift | 3 + Voice.xcodeproj/project.pbxproj | 16 ++ 7 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 Sources/Features/Tasks/TaskPanelView.swift create mode 100644 Sources/LiveKit/TaskStore.swift diff --git a/PLAN.md b/PLAN.md index 8bb952d..ea0cfe5 100644 --- a/PLAN.md +++ b/PLAN.md @@ -26,7 +26,7 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X - [x] Phase 4: voice/model pickers with sync-on-join (commit ca2d26f) - [x] Phase 5: media player — commit 0af944e (webm playback blocked by iOS AVPlayer; see status below) - [x] Phase 6: media search, choices, library panel (commit cbf1830) -- [ ] Phase 7: background tasks panel (task_event fold-in) +- [x] Phase 7: background tasks panel (commit 63f2634) - [ ] Phase 8: polish/hardening/parity pass - [ ] Phase 9: docs & handoff @@ -130,6 +130,27 @@ Verified in simulator (log + UI + server evidence): - Add Stream: entered name + URL, tapped Add → POST /media/api/streams 400 (server rejected test URL) → error message "Could not add stream: Media API request failed with HTTP 400. url must be http(s)" displayed in red. - Polling without session: confirmed via AVPlayer webm decode error in logs (state was fetched and reconciled even though no voice session was active). +## Phase 7 status (2026-08-30 ~12:50) +Done: +- `TaskStore` (Sources/LiveKit/TaskStore.swift): @MainActor ObservableObject. Folds task_event payloads into TaskItem entries keyed by taskId — created (from nested `task` object or description), step (appends {role, content}, pending→running), completed (result + completedAt), failed (error). `sortedTasks` by createdAt desc; `activeCount` = pending+running (badge); `clear()`. +- `ControlChannel.handleTaskEvent` fixed to match the real wire format: agent publishes `{type:"task_event", task_id, event: created|step|completed|failed, ...}` — discriminator is `event`, not `type` (old code never matched). Parses nested `task.description`/`task.created_at`. Added `JSONValue.doubleValue`. +- `VoiceSession`: owns `taskStore`; `.taskEvent` in handleControlEvent → `taskStore.apply`. +- `TaskPanelView` (Sources/Features/Tasks/TaskPanelView.swift): cards sorted desc, status dot (gray/blue/green/red), description header, steps list with role labels + web-mirrored colors (thinking=blue, tool_call=indigo, tool_result=muted, text=primary), green result block, red error block. Empty state "Nothing running / Ask Hope to research or compare something." +- Toolbar: `tray.full` button in ConversationView with red count badge when activeCount > 0; pushes TaskPanelView (NavigationLink, matching the media panel pattern). +- DEBUG-only "Sample" toolbar button in the panel injects two synthetic tasks (one completed, one failed) for UI verification. + +Verified in simulator — REAL events (primary): +- Start → ready (text mode) → composer: "Research the history of jazz music in a background task…" → send. +- Agent published 8 real task_events on topic `tasks` (created + 6 steps incl. thinking/tool_call/tool_result/text + completed). Logs: `[dev.shane.voice:tasks] task created id=738e422c steps=0 … task completed id=738e422c steps=6`. +- Panel opened from toolbar shows the real card: description, all 6 role-styled steps (web_search tool_call visible), green result block with full jazz-history summary. Badge showed "1" while running (activeCount). + +Verified — SYNTHETIC events (secondary, via DEBUG Sample button): +- Injected sample-a (created→2 steps→completed) and sample-b (created→step→failed); both cards rendered with correct status dots, error block ("Search backend timed out after 30s"), and badge count. + +Notes: +- Screenshot captured at /tmp/taskpanel.png (model can't view images; verified via UI snapshot text + logs instead). +- No server/agent changes needed — the task dispatch pipeline already worked; only the iOS client was missing. + ## Subagent notes - Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md - Watch for loops (repeated identical commands) — kill and redirect if seen diff --git a/Sources/Features/Conversation/ConversationView.swift b/Sources/Features/Conversation/ConversationView.swift index d26d997..ee793d4 100644 --- a/Sources/Features/Conversation/ConversationView.swift +++ b/Sources/Features/Conversation/ConversationView.swift @@ -174,6 +174,23 @@ struct ConversationView: View { @ToolbarContentBuilder private var toolbarContent: some ToolbarContent { + ToolbarItem(placement: .topBarTrailing) { + NavigationLink { + TaskPanelView(taskStore: session.taskStore) + } label: { + Image(systemName: "tray.full") + .overlay(alignment: .topTrailing) { + if session.taskStore.activeCount > 0 { + Text("\(session.taskStore.activeCount)") + .font(.caption2.weight(.bold)) + .foregroundStyle(.white) + .padding(3) + .background(Color.red, in: Circle()) + .offset(x: 6, y: -6) + } + } + } + } ToolbarItem(placement: .topBarTrailing) { NavigationLink { MediaPanelView(playerController: session.mediaController, settings: settings) diff --git a/Sources/Features/Tasks/TaskPanelView.swift b/Sources/Features/Tasks/TaskPanelView.swift new file mode 100644 index 0000000..0b47c1b --- /dev/null +++ b/Sources/Features/Tasks/TaskPanelView.swift @@ -0,0 +1,137 @@ +import SwiftUI + +struct TaskPanelView: View { + @ObservedObject var taskStore: TaskStore + + var body: some View { + ScrollView { + VStack(spacing: 12) { + if taskStore.sortedTasks.isEmpty { + emptyState + } else { + ForEach(taskStore.sortedTasks) { task in + TaskCardView(task: task) + } + } + } + .padding() + } + .navigationTitle("Background tasks") + #if DEBUG + .toolbar { + ToolbarItem(placement: .topBarTrailing) { + Button("Sample", action: { taskStore.injectSampleEvents() }) + .font(.caption) + } + } + #endif + } + + private var emptyState: some View { + VStack(spacing: 8) { + Text("Nothing running") + .font(.headline) + Text("Ask Hope to research or compare something.") + .font(.subheadline) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 48) + } +} + +struct TaskCardView: View { + let task: TaskItem + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 8) { + statusDot + Text(task.description) + .font(.subheadline.weight(.semibold)) + .lineLimit(2) + Spacer() + } + if !task.steps.isEmpty { + VStack(alignment: .leading, spacing: 4) { + ForEach(task.steps) { step in + HStack(alignment: .top, spacing: 6) { + Text(stepLabel(step.role)) + .font(.caption2.weight(.semibold)) + .foregroundStyle(stepColor(step.role)) + Text(step.content ?? "") + .font(.caption) + .foregroundStyle(stepColor(step.role)) + .lineLimit(3) + } + } + } + } + if let result = task.result { + block(result, color: .green, background: Color.green.opacity(0.12)) + } + if let error = task.error { + block(error, color: .red, background: Color.red.opacity(0.12)) + } + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color(.secondarySystemBackground), in: RoundedRectangle(cornerRadius: 14)) + } + + private var statusDot: some View { + Circle() + .fill(dotColor) + .frame(width: 9, height: 9) + } + + private var dotColor: Color { + switch task.status { + case .pending: return .gray + case .running: return .blue + case .completed: return .green + case .failed: return .red + } + } + + private func block(_ text: String, color: Color, background: Color) -> some View { + Text(text) + .font(.subheadline) + .foregroundStyle(color) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + .background(background, in: RoundedRectangle(cornerRadius: 10)) + } + + private func stepLabel(_ role: String?) -> String { + switch role { + case "thinking": return "Thinking" + case "tool_call": return "Tool" + case "tool_result": return "Result" + case "text": return "Note" + default: return role ?? "Step" + } + } + + private func stepColor(_ role: String?) -> Color { + switch role { + case "thinking": return .blue + case "tool_call": return .indigo + case "tool_result": return .secondary + default: return .primary + } + } +} + +#Preview { + NavigationStack { + TaskPanelView(taskStore: { + let store = TaskStore() + #if DEBUG + store.injectSampleEvents() + #endif + return store + }()) + } +} diff --git a/Sources/LiveKit/ControlChannel.swift b/Sources/LiveKit/ControlChannel.swift index 30b4167..f10bd19 100644 --- a/Sources/LiveKit/ControlChannel.swift +++ b/Sources/LiveKit/ControlChannel.swift @@ -26,6 +26,7 @@ final class ControlChannel: NSObject, RoomDelegate { let content: String? let result: String? let error: String? + let createdAt: Double? } var onEvent: ((InboundEvent) -> Void)? @@ -63,19 +64,23 @@ final class ControlChannel: NSObject, RoomDelegate { } } - /// Inbound task_event handling (topic-agnostic per web UI). Returns true when consumed. + /// Inbound task_event handling (topic-agnostic per web UI). The agent publishes + /// {type:"task_event", task_id, event, ...} — the discriminator is `event`, not `type`. func handleTaskEvent(_ data: Data) -> Bool { guard let json = try? decoder.decode([String: JSONValue].self, from: data), - let type = json["type"]?.stringValue, + json["type"]?.stringValue == "task_event", + let type = json["event"]?.stringValue, ["created", "step", "completed", "failed"].contains(type) else { return false } + let taskObj = json["task"]?.objectValue ?? [:] let payload = TaskEventPayload( type: type, taskId: json["task_id"]?.stringValue, - description: json["description"]?.stringValue, + description: json["description"]?.stringValue ?? taskObj["description"]?.stringValue, role: json["role"]?.stringValue, content: json["content"]?.stringValue, result: json["result"]?.stringValue, - error: json["error"]?.stringValue + error: json["error"]?.stringValue, + createdAt: json["created_at"]?.doubleValue ?? taskObj["created_at"]?.doubleValue ) onEvent?(.taskEvent(payload)) return true @@ -137,6 +142,11 @@ indirect enum JSONValue: Decodable { return nil } + var doubleValue: Double? { + if case .number(let value) = self { return value } + return nil + } + var objectValue: [String: JSONValue]? { if case .object(let value) = self { return value } return nil diff --git a/Sources/LiveKit/TaskStore.swift b/Sources/LiveKit/TaskStore.swift new file mode 100644 index 0000000..b8ac22a --- /dev/null +++ b/Sources/LiveKit/TaskStore.swift @@ -0,0 +1,130 @@ +import Foundation +import os + +private let logger = Logger(subsystem: "dev.shane.voice", category: "tasks") + +struct TaskStep: Identifiable, Equatable { + let id = UUID() + let role: String? + let content: String? +} + +struct TaskItem: Identifiable, Equatable { + enum Status: Equatable { + case pending + case running + case completed + case failed + } + + let id: String + var description: String + var status: Status + var steps: [TaskStep] + var result: String? + var error: String? + var createdAt: Date + var completedAt: Date? +} + +/// Folds task_event messages into task items, mirroring the web UI's tasks Map. +@MainActor +final class TaskStore: ObservableObject { + @Published private(set) var tasks: [String: TaskItem] = [:] + + var sortedTasks: [TaskItem] { + tasks.values.sorted { $0.createdAt > $1.createdAt } + } + + var activeCount: Int { + tasks.values.filter { $0.status == .pending || $0.status == .running }.count + } + + func apply(_ payload: ControlChannel.TaskEventPayload) { + guard let taskId = payload.taskId, !taskId.isEmpty else { return } + switch payload.type { + case "created": + tasks[taskId] = TaskItem( + id: taskId, + description: payload.description ?? "Background task", + status: .pending, + steps: [], + result: nil, + error: nil, + createdAt: date(from: payload.createdAt), + completedAt: nil + ) + case "step": + var task = tasks[taskId] ?? TaskItem( + id: taskId, + description: payload.description ?? "Background task", + status: .pending, + steps: [], + result: nil, + error: nil, + createdAt: date(from: payload.createdAt), + completedAt: nil + ) + task.steps.append(TaskStep(role: payload.role, content: payload.content)) + if task.status == .pending { task.status = .running } + tasks[taskId] = task + case "completed": + var task = tasks[taskId] ?? TaskItem( + id: taskId, + description: payload.description ?? "Background task", + status: .pending, + steps: [], + result: nil, + error: nil, + createdAt: date(from: payload.createdAt), + completedAt: nil + ) + task.status = .completed + task.result = payload.result + task.completedAt = Date() + tasks[taskId] = task + case "failed": + var task = tasks[taskId] ?? TaskItem( + id: taskId, + description: payload.description ?? "Background task", + status: .pending, + steps: [], + result: nil, + error: nil, + createdAt: date(from: payload.createdAt), + completedAt: nil + ) + task.status = .failed + task.error = payload.error + task.completedAt = Date() + tasks[taskId] = task + default: + return + } + logger.log("task \(payload.type, privacy: .public) id=\(taskId, privacy: .public) steps=\(self.tasks[taskId]?.steps.count ?? 0, privacy: .public)") + } + + func clear() { + tasks.removeAll() + } + + #if DEBUG + func injectSampleEvents() { + let now = Date().timeIntervalSince1970 + apply(ControlChannel.TaskEventPayload(type: "created", taskId: "sample-a", description: "Research the best coffee roasters in Portland", role: nil, content: nil, result: nil, error: nil, createdAt: now - 45)) + apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-a", description: nil, role: "thinking", content: "Searching for Portland roasteries…", result: nil, error: nil, createdAt: nil)) + apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-a", description: nil, role: "tool_result", content: "Found 12 results across 3 review sites.", result: nil, error: nil, createdAt: nil)) + apply(ControlChannel.TaskEventPayload(type: "completed", taskId: "sample-a", description: nil, role: nil, content: nil, result: "Top three: Coava, Stumptown, and Heart Roasters — all scored 4.5+ across reviews.", error: nil, createdAt: nil)) + apply(ControlChannel.TaskEventPayload(type: "created", taskId: "sample-b", description: "Compare iPhone 17 vs Pixel 10 battery life", role: nil, content: nil, result: nil, error: nil, createdAt: now - 20)) + apply(ControlChannel.TaskEventPayload(type: "step", taskId: "sample-b", description: nil, role: "thinking", content: "Fetching benchmark articles…", result: nil, error: nil, createdAt: nil)) + apply(ControlChannel.TaskEventPayload(type: "failed", taskId: "sample-b", description: nil, role: nil, content: nil, result: nil, error: "Search backend timed out after 30s", createdAt: nil)) + } + #endif +} + +private extension TaskStore { + func date(from interval: Double?) -> Date { + guard let interval, interval > 0 else { return Date() } + return Date(timeIntervalSince1970: interval) + } +} diff --git a/Sources/LiveKit/VoiceSession.swift b/Sources/LiveKit/VoiceSession.swift index 687c884..bb57f49 100644 --- a/Sources/LiveKit/VoiceSession.swift +++ b/Sources/LiveKit/VoiceSession.swift @@ -33,6 +33,7 @@ final class VoiceSession: ObservableObject { let controlChannel = ControlChannel() let transcriptHandler = TranscriptHandler() let stateModel = ConversationStateModel() + let taskStore = TaskStore() let mediaController: MediaPlayerController private let config: ServerConfig private let tokenClient: TokenClient @@ -71,6 +72,8 @@ final class VoiceSession: ObservableObject { selectedModel = model settings.modelID = model logger.log("model_changed applied: \(model, privacy: .public)") + case .taskEvent(let payload): + taskStore.apply(payload) default: break } diff --git a/Voice.xcodeproj/project.pbxproj b/Voice.xcodeproj/project.pbxproj index 06e5d23..8fa1074 100644 --- a/Voice.xcodeproj/project.pbxproj +++ b/Voice.xcodeproj/project.pbxproj @@ -17,7 +17,9 @@ 73DA73811F8AEB2AFE340D10 /* MediaPlayerController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3964F565A6C6A06399EBA4DF /* MediaPlayerController.swift */; }; 89B47754E5C39418BFD839E6 /* TranscriptHistory.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68A45228EF59186CFC219F2E /* TranscriptHistory.swift */; }; 8DE1363CC0119B5AF9E4E117 /* ConnectionTestView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC4B9073EE8EAB51F8C55760 /* ConnectionTestView.swift */; }; + 943BCE971A01F220FAE9E250 /* TaskPanelView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E85BACA267ADD462306C9920 /* TaskPanelView.swift */; }; 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48F76881B4AE51DEE74511CC /* SettingsView.swift */; }; + A5868238CF4958D6D0C5908E /* TaskStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0B60FBBB9C47BB9ADACA7CA /* TaskStore.swift */; }; A86EFB492F40C26D4AA315E9 /* PinnedMediaResourceLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6DC3AC57ED58FC517E212F19 /* PinnedMediaResourceLoader.swift */; }; A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */; }; C00D1878C2AB3D8B6F9562B6 /* AgentMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */; }; @@ -48,8 +50,10 @@ A0C23305A6C55A83F8617D9D /* ConversationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConversationState.swift; sourceTree = ""; }; A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OrbView.swift; sourceTree = ""; }; AC4B9073EE8EAB51F8C55760 /* ConnectionTestView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionTestView.swift; sourceTree = ""; }; + C0B60FBBB9C47BB9ADACA7CA /* TaskStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TaskStore.swift; sourceTree = ""; }; D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscriptHandler.swift; sourceTree = ""; }; D9B40076934C28BA4B92D808 /* TLSPinning.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TLSPinning.swift; sourceTree = ""; }; + E85BACA267ADD462306C9920 /* TaskPanelView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TaskPanelView.swift; sourceTree = ""; }; EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSession.swift; sourceTree = ""; }; F5683993A1DE677121FFA1F3 /* ServerConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ServerConfig.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -89,6 +93,7 @@ 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */, 61C71C48230C683D199D6F4B /* ControlChannel.swift */, A0C23305A6C55A83F8617D9D /* ConversationState.swift */, + C0B60FBBB9C47BB9ADACA7CA /* TaskStore.swift */, D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */, 68A45228EF59186CFC219F2E /* TranscriptHistory.swift */, EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */, @@ -129,6 +134,7 @@ 84ADAF12A43A57888CA178A6 /* Conversation */, 0D2E974CBC286D7C886BA814 /* Media */, 0DF0591DF1D767F89099F62C /* Settings */, + F168B25F8B3C6F36B6E88228 /* Tasks */, ); path = Features; sourceTree = ""; @@ -161,6 +167,14 @@ path = App; sourceTree = ""; }; + F168B25F8B3C6F36B6E88228 /* Tasks */ = { + isa = PBXGroup; + children = ( + E85BACA267ADD462306C9920 /* TaskPanelView.swift */, + ); + path = Tasks; + sourceTree = ""; + }; FCC61E9990B71498171F9D34 /* Sources */ = { isa = PBXGroup; children = ( @@ -250,6 +264,8 @@ A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */, 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */, 720C26EC163AE46713B02DD8 /* TLSPinning.swift in Sources */, + 943BCE971A01F220FAE9E250 /* TaskPanelView.swift in Sources */, + A5868238CF4958D6D0C5908E /* TaskStore.swift in Sources */, D18AD347DFF72AB6361165BA /* TokenClient.swift in Sources */, D2E343C8B8657316A993698E /* TranscriptHandler.swift in Sources */, 89B47754E5C39418BFD839E6 /* TranscriptHistory.swift in Sources */,