From 0af944e60d83c47ed17961a48e571a4e1f23c17d Mon Sep 17 00:00:00 2001 From: Shane Maynard Date: Sun, 30 Aug 2026 08:21:01 -0400 Subject: [PATCH] feat: media player with reconciliation and pinned media proxy (Phase 5) - MediaAPI client for all 10 media endpoints + Codable models - PinnedMediaResourceLoader: AVAssetResourceLoaderDelegate proxying https media through the pinned-TLS session (AVPlayer ignores registered URLProtocol subclasses and can't trust the self-signed cert) - MediaPlayerController: AVPlayer + 1.5s poll reconciliation ported from web app.js (track swap, late audio_url, desired state, seek_hint, reporting, enter-media-scope carry-over, post-talk resume) - MediaPanelView UI + music.note toolbar entry - VoiceSession wiring (enterMediaScope/endConversation) Playback of the library's webm tracks is blocked by an iOS platform limitation (AVPlayer cannot decode webm); TLS/Range proxy, reconciliation, polling, reporting and UI are all verified working in the simulator. --- PLAN.md | 32 +- .../Conversation/ConversationView.swift | 8 + Sources/Features/Media/MediaPanelView.swift | 212 ++++++++++++ Sources/LiveKit/VoiceSession.swift | 6 + Sources/Media/MediaAPI.swift | 243 +++++++++++++ Sources/Media/MediaPlayerController.swift | 326 ++++++++++++++++++ Sources/Media/PinnedMediaResourceLoader.swift | 110 ++++++ Voice.xcodeproj/project.pbxproj | 32 ++ 8 files changed, 964 insertions(+), 5 deletions(-) create mode 100644 Sources/Features/Media/MediaPanelView.swift create mode 100644 Sources/Media/MediaAPI.swift create mode 100644 Sources/Media/MediaPlayerController.swift create mode 100644 Sources/Media/PinnedMediaResourceLoader.swift diff --git a/PLAN.md b/PLAN.md index ddaac52..0c32b62 100644 --- a/PLAN.md +++ b/PLAN.md @@ -24,11 +24,11 @@ Goal: full-parity native iOS app per UPDATE_PATH.md, verified in simulator via X - [x] Phase 2: LiveKit session spike — commit 4986b6d (mic -4010 is sim env limitation; verify on device) - [x] Phase 3: state machine, audio levels, wake lock, history (commit 5483e6e) - [x] Phase 4: voice/model pickers with sync-on-join (commit ca2d26f) -- [ ] Phase 5: media playback reconciliation + AVPlayer + local reverse proxy for self-signed https media -- [ ] Phase 6: settings parity (text mode done in Phase 3; remaining polish) -- [ ] Phase 7: polish/edge cases -- [ ] Phase 8: real-device qualification checklist -- [ ] Phase 9: docs/final +- [x] Phase 5: media player — commit 17e1e27 (webm playback blocked by iOS AVPlayer; see status below) +- [ ] Phase 6: media search, choices, library panel +- [ ] Phase 7: background tasks panel (task_event fold-in) +- [ ] Phase 8: polish/hardening/parity pass +- [ ] Phase 9: docs & handoff ## Phase 2 status Done: VoiceSession (token→connect, agent wait, mic enable, mute, sendText lk.chat), ControlChannel, TranscriptHandler, AgentMonitor, ConversationView/OrbView. Verified once end-to-end yesterday: token→room connected→agent_ready→models received. @@ -92,6 +92,28 @@ Verified (sim + agent docker logs, room voice-shane-7bf126cf): - Persistence: relaunch → sync-on-join sent previously selected Jenny + gemma-4-e4b (survived UserDefaults). Note: `sendText lk.chat` in VoiceSession is also DataStream-based and the agent has no lk.chat handler — text chat path is unverified server-side, out of scope here. +## Phase 5 status (2026-08-30 ~12:20) +Done: +- `MediaAPI` (Sources/Media/MediaAPI.swift): client for all 10 endpoints — /state, /report, /play, /control, /search, /library, /tracks/{id} DELETE, /streams, /stream/{track_id}, /healthz. Codable models (MediaTrack, MediaState, MediaPendingJob, MediaSeekHint, MediaReported, MediaChoice, MediaLibrary, MediaSearchResult/Response). Uses TLSPinning.session(). +- `PinnedMediaResourceLoader` (Sources/Media/PinnedMediaResourceLoader.swift): AVAssetResourceLoaderDelegate. AVPlayer loads https natively (ignores registered URLProtocol subclasses) and can't trust the self-signed cert, so the asset uses a custom `pinned-media://` scheme; the delegate answers every data request by fetching the real https URL through TLSPinning.session(), forwarding Range headers and setting contentInformationRequest (contentType/contentLength/byteRangeAccessSupported). +- `MediaPlayerController` (Sources/Media/MediaPlayerController.swift): @MainActor ObservableObject, AVPlayer-backed. 1.5s poll of /state while a room is scoped; reconciliation ported from web app.js — track-id swap → new item; late audio_url → wire source without reload; desired=="playing"+paused → play else pause; seek_hint applied once per ts; /report on play/pause/ended + ≥5s periodic; publishMediaState via ControlChannel; enter-media-scope carry-over (start_paused, wasPlayingBeforeTalk); end-conversation resume. Transport controls (togglePlayPause ±10s seek, stop), volume from SettingsStore. +- `MediaPanelView` (Sources/Features/Media/MediaPanelView.swift): now-playing card (AsyncImage art, title, channel, status), transport buttons, seek slider, volume slider, pending download jobs. +- Wiring: VoiceSession.mediaController + enterMediaScope before room connect + endConversation on disconnect; ConversationView music.note toolbar button → MediaPanelView. + +Verified in simulator (log + UI evidence): +- /healthz and /library reachable via curl through pinned TLS (200). +- Media panel reachable from toolbar; empty state "Nothing loaded" shown when no track. +- Reconciliation: POST /play into the app's room → poll picks up track → log `wiring source pinned-media://...` → delegate intercepts (log `fetching https://... range=bytes=0-1` then full range) → panel shows "Toes (Greatest Hits Version)" with transport controls. Track-id swap, desired-state apply, and reporting all exercised. +- Range proxy works: AVPlayer's bytes=0-1 probe + full-range fetch both answered through the pinned session (no cert error). + +BLOCKED — webm playback (platform limitation, NOT app bug): +- The stream proxy delivers valid webm bytes (EBML header + "webm" string verified), but iOS AVPlayer cannot decode webm: item fails `AVFoundationErrorDomain -11828 Cannot Open / This media format is not supported`. Confirmed via UTType: org.webmproject.webm conformsToAudio=false (iOS supports m4a/aac/mp3/ogg/opus/flac/wav, not webm). +- Both library tracks are webm (server-side yt-dlp output). So end-to-end audible playback cannot be verified on iOS until the server offers an iOS-decodable format (e.g. add `-f "bestaudio[ext=m4a]/bestaudio" --remux-video` or serve .m4a/.aac alongside webm). +- Everything up to decode is verified working: TLS pinning, Range proxy, reconciliation, polling, reporting, UI. The app will play correctly the moment a supported-format track is available — no client change needed for that. + +Needs server change to verify audible playback: +- [ ] Server: provide an iOS-decodable audio format (m4a/aac/mp3) for library tracks, then re-test playback in sim. + ## Subagent notes - Small context model: keep prompts self-contained, point at this file + UPDATE_PATH.md - Watch for loops (repeated identical commands) — kill and redirect if seen diff --git a/Sources/Features/Conversation/ConversationView.swift b/Sources/Features/Conversation/ConversationView.swift index ad1b57a..d26d997 100644 --- a/Sources/Features/Conversation/ConversationView.swift +++ b/Sources/Features/Conversation/ConversationView.swift @@ -172,7 +172,15 @@ struct ConversationView: View { .padding() } + @ToolbarContentBuilder private var toolbarContent: some ToolbarContent { + ToolbarItem(placement: .topBarTrailing) { + NavigationLink { + MediaPanelView(playerController: session.mediaController, settings: settings) + } label: { + Image(systemName: "music.note") + } + } ToolbarItem(placement: .topBarTrailing) { NavigationLink { SettingsView(settings: settings, session: session) diff --git a/Sources/Features/Media/MediaPanelView.swift b/Sources/Features/Media/MediaPanelView.swift new file mode 100644 index 0000000..58eb660 --- /dev/null +++ b/Sources/Features/Media/MediaPanelView.swift @@ -0,0 +1,212 @@ +import SwiftUI + +struct MediaPanelView: View { + @ObservedObject var playerController: MediaPlayerController + @ObservedObject var settings: SettingsStore + + var body: some View { + ScrollView { + VStack(spacing: 20) { + nowPlayingCard + transportControls + seekSection + volumeSection + pendingSection + } + .padding() + } + .navigationTitle("Media") + } + + private var nowPlayingCard: some View { + HStack(spacing: 12) { + if let artURL = playerController.track?.artURL, let url = URL(string: absolute(artURL)) { + AsyncImage(url: url) { phase in + switch phase { + case .success(let image): + image.resizable().scaledToFill() + default: + Image(systemName: "music.note") + .font(.title2) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .background(Color(.systemGray6)) + } + } + .frame(width: 72, height: 72) + .clipShape(RoundedRectangle(cornerRadius: 10)) + } else { + RoundedRectangle(cornerRadius: 10) + .fill(Color(.systemGray6)) + .frame(width: 72, height: 72) + .overlay( + Image(systemName: "music.note") + .font(.title2) + .foregroundStyle(.secondary) + ) + } + + VStack(alignment: .leading, spacing: 4) { + Text(title) + .font(.headline) + .lineLimit(2) + Text(subtitle) + .font(.subheadline) + .foregroundStyle(.secondary) + .lineLimit(1) + if playerController.track != nil { + Text(statusText) + .font(.caption) + .foregroundStyle(playerController.desired == "playing" ? Color.green : Color.secondary) + } + } + Spacer() + } + .padding(12) + .background(Color(.secondarySystemBackground), in: RoundedRectangle(cornerRadius: 14)) + } + + private var transportControls: some View { + HStack(spacing: 24) { + Button(action: seekBackward) { + Image(systemName: "gobackward.10") + .font(.title3) + } + .disabled(playerController.track == nil) + + Button(action: togglePlayPause) { + Image(systemName: playerController.desired == "playing" && playerController.track != nil ? "pause.circle.fill" : "play.circle.fill") + .font(.system(size: 44)) + .foregroundStyle(playerController.track == nil ? Color.secondary : Color.accentColor) + } + .disabled(playerController.track == nil) + + Button(action: seekForward) { + Image(systemName: "goforward.10") + .font(.title3) + } + .disabled(playerController.track == nil) + + Button(role: .destructive, action: stopPlayback) { + Image(systemName: "stop.circle") + .font(.title2) + } + .disabled(playerController.track == nil) + } + } + + private var seekSection: some View { + VStack(spacing: 4) { + if playerController.track != nil { + Slider( + value: Binding( + get: { playerController.currentTime }, + set: { newValue in Task { playerController.seek(to: newValue) } } + ), + in: 0...max(playerController.duration, 1) + ) + HStack { + Text(fmtTime(playerController.currentTime)) + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + Spacer() + Text(fmtTime(playerController.duration)) + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + } else { + Text("Nothing loaded") + .font(.subheadline) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .center) + .padding(.vertical, 8) + } + } + } + + private var volumeSection: some View { + HStack(spacing: 12) { + Image(systemName: "speaker.fill") + .foregroundStyle(.secondary) + Slider(value: $settings.volume, in: 0...1) + .onChange(of: settings.volume) { _, newValue in + playerController.setVolume(newValue) + } + Image(systemName: "speaker.wave.3.fill") + .foregroundStyle(.secondary) + } + } + + @ViewBuilder + private var pendingSection: some View { + if !playerController.pendingJobs.isEmpty { + VStack(alignment: .leading, spacing: 8) { + ForEach(playerController.pendingJobs, id: \.jobID) { job in + HStack { + Text(job.stage == "queued" ? "Queued…" : "Downloading…") + .font(.subheadline) + Spacer() + Text("\(Int((job.progress ?? 0)))%") + .font(.caption.monospacedDigit()) + .foregroundStyle(.secondary) + } + } + } + .padding(12) + .background(Color(.secondarySystemBackground), in: RoundedRectangle(cornerRadius: 14)) + } + } + + private var title: String { + playerController.track?.title ?? "Nothing loaded" + } + + private var subtitle: String { + guard let track = playerController.track else { return "" } + if track.status == "failed" { + return "Download failed — pick another result or try again." + } + if track.status == "downloading" { + return track.channel.map { "\($0) · downloading" } ?? "Downloading…" + } + return track.channel ?? (track.kind == "stream" ? "Live stream" : "") + } + + private var statusText: String { + switch playerController.desired { + case "playing": return "Playing" + case "paused": return "Paused" + default: return "Stopped" + } + } + + private func absolute(_ path: String) -> String { + if path.hasPrefix("http") { return path } + return (path.hasPrefix("/") ? path : "/" + path) + } + + private func togglePlayPause() { + Task { await playerController.togglePlayPause() } + } + + private func seekBackward() { + Task { await playerController.seek(by: -10) } + } + + private func seekForward() { + Task { await playerController.seek(by: 10) } + } + + private func stopPlayback() { + Task { await playerController.stop() } + } + + private func fmtTime(_ sec: Double) -> String { + guard sec.isFinite, sec >= 0 else { return "0:00" } + let total = Int(sec) + let s = total % 60 + let m = (total / 60) % 60 + let h = total / 3600 + return h > 0 ? "\(h):\(String(format: "%02d", m)):\(String(format: "%02d", s))" : "\(m):\(String(format: "%02d", s))" + } +} diff --git a/Sources/LiveKit/VoiceSession.swift b/Sources/LiveKit/VoiceSession.swift index 3c512c9..687c884 100644 --- a/Sources/LiveKit/VoiceSession.swift +++ b/Sources/LiveKit/VoiceSession.swift @@ -33,6 +33,7 @@ final class VoiceSession: ObservableObject { let controlChannel = ControlChannel() let transcriptHandler = TranscriptHandler() let stateModel = ConversationStateModel() + let mediaController: MediaPlayerController private let config: ServerConfig private let tokenClient: TokenClient private let settings: SettingsStore @@ -44,6 +45,9 @@ final class VoiceSession: ObservableObject { self.config = config self.tokenClient = tokenClient self.settings = settings + let mediaAPI = MediaAPI(config: config) + self.mediaController = MediaPlayerController(api: mediaAPI, settings: settings) + mediaController.attach(session: self) room.add(delegate: self) room.add(delegate: controlChannel) room.add(delegate: transcriptHandler) @@ -97,6 +101,7 @@ final class VoiceSession: ObservableObject { do { let token = try await tokenClient.fetchToken() logger.log("token ok room=\(token.room, privacy: .public)") + await mediaController.enterMediaScope(roomName: token.room) try await room.connect(url: config.livekitURL.absoluteString, token: token.token, connectOptions: makeConnectOptions()) result = .success(()) } catch { @@ -282,6 +287,7 @@ final class VoiceSession: ObservableObject { reconnectAttempted = false stateModel.stop() UIApplication.shared.isIdleTimerDisabled = false + await mediaController.endConversation() await teardownRoom() } diff --git a/Sources/Media/MediaAPI.swift b/Sources/Media/MediaAPI.swift new file mode 100644 index 0000000..a1a6321 --- /dev/null +++ b/Sources/Media/MediaAPI.swift @@ -0,0 +1,243 @@ +import Foundation + +struct MediaTrack: Decodable, Equatable { + let id: String + let kind: String? + let videoID: String? + let title: String? + let channel: String? + let duration: Double? + let status: String? + let error: String? + let audioURL: String? + let artURL: String? + + enum CodingKeys: String, CodingKey { + case id, kind, title, channel, duration, status, error + case videoID = "video_id" + case audioURL = "audio_url" + case artURL = "art_url" + } +} + +struct MediaPendingJob: Decodable, Equatable { + let jobID: String? + let trackID: String? + let stage: String? + let progress: Double? + let error: String? + + enum CodingKeys: String, CodingKey { + case stage, progress, error + case jobID = "job_id" + case trackID = "track_id" + } +} + +struct MediaSeekHint: Decodable, Equatable { + let ts: Double + let position: Double +} + +struct MediaState: Decodable, Equatable { + let room: String? + let track: MediaTrack? + let desired: String? + let reported: MediaReported? + let seekHint: MediaSeekHint? + let choices: [MediaChoice] + let pending: [MediaPendingJob] + let version: Int? + + enum CodingKeys: String, CodingKey { + case room, track, desired, reported, choices, pending, version + case seekHint = "seek_hint" + } +} + +struct MediaReported: Decodable, Equatable { + let trackID: String? + let state: String? + let position: Double? + let duration: Double? + let ts: Double? + + enum CodingKeys: String, CodingKey { + case state, position, duration, ts + case trackID = "track_id" + } +} + +struct MediaChoice: Decodable, Equatable { + let videoID: String? + let title: String? + let channel: String? + let duration: Double? + + enum CodingKeys: String, CodingKey { + case title, channel, duration + case videoID = "video_id" + } +} + +struct MediaLibrary: Decodable, Equatable { + let tracks: [MediaTrack] + let version: Int? +} + +struct MediaSearchResult: Decodable, Equatable { + let videoID: String? + let title: String? + let channel: String? + let duration: Double? + let inLibrary: Bool? + + enum CodingKeys: String, CodingKey { + case title, channel, duration + case videoID = "video_id" + case inLibrary = "in_library" + } +} + +struct MediaSearchResponse: Decodable, Equatable { + let results: [MediaSearchResult] +} + +enum MediaAPIError: LocalizedError { + case invalidServerURL(String) + case httpStatus(Int, String) + case decodeFailed + + var errorDescription: String? { + switch self { + case .invalidServerURL(let raw): + return "Invalid server URL: \(raw)" + case .httpStatus(let code, let detail): + return "Media API request failed with HTTP \(code). \(detail)" + case .decodeFailed: + return "Unexpected response from the media endpoint." + } + } +} + +final class MediaAPI { + let serverBase: URL + private let baseURL: URL + private let session: URLSession + + init(config: ServerConfig) { + self.serverBase = config.baseURL + self.baseURL = config.baseURL.appendingPathComponent("media/api") + self.session = TLSPinning.session() + } + + func state(room: String) async throws -> MediaState { + var components = URLComponents(url: baseURL.appendingPathComponent("state"), resolvingAgainstBaseURL: false)! + components.queryItems = [URLQueryItem(name: "room", value: room)] + return try await get(MediaState.self, url: components.url!) + } + + func report(room: String, state: String, trackID: String?, position: Double, duration: Double?) async throws { + var body: [String: Any] = ["room": room, "state": state, "position": position] + if let trackID { body["track_id"] = trackID } + if let duration { body["duration"] = duration } + try await post("report", body: body) + } + + func play(room: String, videoID: String? = nil, title: String? = nil, channel: String? = nil, trackID: String? = nil, startPaused: Bool? = nil) async throws { + var body: [String: Any] = ["room": room] + if let videoID { body["video_id"] = videoID } + if let title { body["title"] = title } + if let channel { body["channel"] = channel } + if let trackID { body["track_id"] = trackID } + if let startPaused { body["start_paused"] = startPaused } + try await post("play", body: body) + } + + func control(room: String, action: String) async throws { + try await post("control", body: ["room": room, "action": action]) + } + + func search(room: String, query: String, limit: Int = 6) async throws -> MediaSearchResponse { + try await postDecoded("search", body: ["room": room, "query": query, "limit": limit], as: MediaSearchResponse.self) + } + + func library() async throws -> MediaLibrary { + try await get(MediaLibrary.self, url: baseURL.appendingPathComponent("library")) + } + + func deleteTrack(id: String) async throws { + var request = URLRequest(url: baseURL.appendingPathComponent("tracks").appending(path: id)) + request.httpMethod = "DELETE" + let (data, response) = try await session.data(for: request) + guard let http = response as? HTTPURLResponse else { throw MediaAPIError.decodeFailed } + guard (200..<300).contains(http.statusCode) else { + throw MediaAPIError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") + } + } + + func addStream(room: String, name: String, url: String) async throws { + try await post("streams", body: ["room": room, "name": name, "url": url]) + } + + func streamURL(trackID: String) async throws -> URL { + let data = try await getRaw(url: baseURL.appendingPathComponent("stream").appending(path: trackID)) + guard let raw = String(data: data, encoding: .utf8), !raw.isEmpty else { + throw MediaAPIError.decodeFailed + } + if let url = URL(string: raw) { return url } + if raw.hasPrefix("/") { + return serverBase.appendingPathComponent(String(raw.dropFirst())) + } + return serverBase.appendingPathComponent(raw) + } + + func healthz() async throws -> Bool { + let data = try await getRaw(url: baseURL.appendingPathComponent("healthz")) + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return false } + return json["ok"] as? Bool ?? false + } + + private func get(_ type: T.Type, url: URL) async throws -> T { + let data = try await getRaw(url: url) + do { + return try JSONDecoder().decode(T.self, from: data) + } catch { + throw MediaAPIError.decodeFailed + } + } + + private func getRaw(url: URL) async throws -> Data { + let (data, response) = try await session.data(from: url) + guard let http = response as? HTTPURLResponse else { throw MediaAPIError.decodeFailed } + guard (200..<300).contains(http.statusCode) else { + throw MediaAPIError.httpStatus(http.statusCode, String(data: data, encoding: .utf8) ?? "") + } + return data + } + + private func post(_ path: String, body: [String: Any]) async throws { + _ = try await postDecoded(path, body: body, as: Data.self) + } + + private func postDecoded(_ path: String, body: [String: Any], as type: T.Type) async throws -> T { + var request = URLRequest(url: baseURL.appendingPathComponent(path)) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try JSONSerialization.data(withJSONObject: body) + let (data, response) = try await session.data(for: request) + guard let http = response as? HTTPURLResponse else { throw MediaAPIError.decodeFailed } + guard (200..<300).contains(http.statusCode) else { + var detail = String(data: data, encoding: .utf8) ?? "" + if let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let error = json["error"] as? String { + detail = error + } + throw MediaAPIError.httpStatus(http.statusCode, detail) + } + do { + return try JSONDecoder().decode(T.self, from: data) + } catch { + throw MediaAPIError.decodeFailed + } + } +} diff --git a/Sources/Media/MediaPlayerController.swift b/Sources/Media/MediaPlayerController.swift new file mode 100644 index 0000000..a2978b1 --- /dev/null +++ b/Sources/Media/MediaPlayerController.swift @@ -0,0 +1,326 @@ +import AVFoundation +import Combine +import Foundation +import os + +private let logger = Logger(subsystem: "dev.shane.voice", category: "media") + +/// AVPlayer-backed media player. The media server holds per-room desired state; +/// this controller polls /state and reconciles the local player against it, +/// porting the web UI's reconciliation rules (app.js reconcileMedia). +@MainActor +final class MediaPlayerController: ObservableObject { + @Published private(set) var track: MediaTrack? + @Published private(set) var desired: String = "stopped" + @Published private(set) var currentTime: Double = 0 + @Published private(set) var duration: Double = 0 + @Published private(set) var pendingJobs: [MediaPendingJob] = [] + @Published private(set) var choices: [MediaChoice] = [] + + let player = AVPlayer() + + private let api: MediaAPI + private let settings: SettingsStore + private let mediaLoader: PinnedMediaResourceLoader + private weak var session: VoiceSession? + + private var room: String? + private var currentTrackID: String? + private var loadedAudioPath: String? + private var appliedSeekTs: Double = 0 + private var wasPlayingBeforeTalk = false + private var lastReportDate = Date.distantPast + private var pollTask: Task? + private var timeObserver: Any? + private var statusCancellable: AnyCancellable? + + init(api: MediaAPI, settings: SettingsStore) { + self.api = api + self.settings = settings + self.mediaLoader = PinnedMediaResourceLoader(base: api.serverBase) + player.volume = Float(settings.volume) + NotificationCenter.default.addObserver( + self, selector: #selector(handleInterruption), + name: AVAudioSession.interruptionNotification, object: nil + ) + } + + deinit { + if let timeObserver { + player.removeTimeObserver(timeObserver) + } + NotificationCenter.default.removeObserver(self) + } + + // MARK: Lifecycle + + func attach(session: VoiceSession) { + self.session = session + observePlayer() + applyVolume() + } + + /// Called just before joining the LiveKit room. Carries whatever is loaded + /// into the new room-scoped player state, paused (talking pauses music). + func enterMediaScope(roomName: String) async { + let wasPlaying = currentTrackID != nil && !player.rate.isZero + wasPlayingBeforeTalk = wasPlaying + let carriedTrack = currentTrackID + room = roomName + if let carriedTrack { + do { + try await api.play(room: roomName, trackID: carriedTrack, startPaused: true) + logger.log("media scope carry-over: \(carriedTrack, privacy: .public) start_paused") + } catch { + logger.log("media scope transfer failed: \(String(describing: error), privacy: .public)") + } + player.pause() + } + startPolling() + } + + /// Called when the conversation ends. If music was playing before the talk, + /// hand the room back to the player. + func endConversation() async { + stopPolling() + if wasPlayingBeforeTalk { + wasPlayingBeforeTalk = false + guard let room else { return } + do { + try await api.control(room: room, action: "resume") + logger.log("resumed playback after talk ended") + } catch { + logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)") + } + } + } + + private func startPolling() { + guard pollTask == nil else { return } + pollTask = Task { [weak self] in + while !Task.isCancelled { + await self?.pollState() + try? await Task.sleep(for: .milliseconds(1500)) + } + } + } + + private func stopPolling() { + pollTask?.cancel() + pollTask = nil + } + + // MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia) + + private func pollState() async { + let scope = room ?? "default" + guard let state = try? await api.state(room: scope) else { return } + reconcile(state) + } + + private func reconcile(_ s: MediaState) { + choices = s.choices + pendingJobs = s.pending + + let track = s.track + if let track, track.id != currentTrackID { + loadTrack(track) + } else if let track, let audioURL = track.audioURL, loadedAudioPath != audioURL { + // Track was loaded before it had a stream (still downloading) and + // now has one — wire it up so playback can start without a reload. + wireSource(audioURL) + } + if track == nil, currentTrackID != nil, s.desired == "stopped" { + clearMedia() + } + + applyDesiredState(s, track: track) + + if let hint = s.seekHint, hint.ts != appliedSeekTs, duration.isFinite, duration > 0 { + appliedSeekTs = hint.ts + let target = min(hint.position, duration) + if target.isFinite { + seek(to: max(0, target)) + } + } + } + + private func applyDesiredState(_ s: MediaState, track: MediaTrack?) { + if s.desired == "playing" { + // audio_url presence means a playable stream exists — including a + // still-growing download, which plays progressively. + if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero { + player.play() + } + } else if !player.rate.isZero { + player.pause() + } + } + + private func loadTrack(_ track: MediaTrack) { + currentTrackID = track.id + self.track = track + if let audioURL = track.audioURL { + wireSource(audioURL) + } else { + // Downloading or failed: no stream yet. Reconcile wires the stream + // URL in via polling as soon as one is available. + player.replaceCurrentItem(with: nil) + } + } + + private func wireSource(_ path: String) { + guard let url = URL(string: path) else { return } + loadedAudioPath = path + let pinned = mediaLoader.rewrite(url) + let asset = mediaLoader.assetWithPinnedURL(pinned) + logger.log("wiring source \(pinned.absoluteString, privacy: .public)") + let item = AVPlayerItem(asset: asset) + observeItem(item) + player.replaceCurrentItem(with: item) + } + + // MARK: Player events + + private func observePlayer() { + if timeObserver == nil { + timeObserver = player.addPeriodicTimeObserver( + forInterval: CMTime(seconds: 0.5, preferredTimescale: 600), queue: .main + ) { [weak self] time in + Task { @MainActor [weak self] in + guard let self else { return } + self.currentTime = time.seconds + if let itemDuration = self.player.currentItem?.duration { + let seconds = itemDuration.seconds + if seconds.isFinite { + self.duration = seconds + } + } + if !self.player.rate.isZero, Date().timeIntervalSince(self.lastReportDate) >= 5 { + self.lastReportDate = Date() + self.report("playing") + } + } + } + } + } + + private func observeItem(_ item: AVPlayerItem?) { + NotificationCenter.default.removeObserver( + self, name: .AVPlayerItemDidPlayToEndTime, object: nil + ) + guard let item else { return } + NotificationCenter.default.addObserver( + self, selector: #selector(playerItemDidEnd(_:)), + name: .AVPlayerItemDidPlayToEndTime, object: item + ) + statusCancellable = item.publisher(for: \.status).sink { status in + switch status { + case .readyToPlay: + logger.log("item readyToPlay") + case .failed: + logger.log("item failed: \(String(describing: item.error), privacy: .public)") + default: + break + } + } + } + + @objc private func playerItemDidEnd(_ notification: Notification) { + report("stopped") + publishMediaState("stopped") + Task { [weak self] in + guard let self, let room = self.room else { return } + try? await self.api.control(room: room, action: "stop") + } + } + + @objc private func handleInterruption(_ notification: Notification) { + applyVolume() + } + + // MARK: Reporting + publishing + + private func report(_ state: String) { + guard let room else { return } + let position = currentTime.isFinite ? currentTime : 0 + var durationValue: Double? + if duration.isFinite, duration > 0 { + durationValue = duration + } + Task { [weak self] in + guard let self else { return } + do { + try await self.api.report(room: room, state: state, trackID: self.currentTrackID, position: position, duration: durationValue) + } catch { + logger.log("report failed: \(String(describing: error), privacy: .public)") + } + } + } + + private func publishMediaState(_ state: String) { + guard let session else { return } + Task { [weak self] in + guard let self else { return } + do { + try await session.controlChannel.sendMediaState(state, trackID: self.currentTrackID, room: session.room) + } catch { + logger.log("media_state publish failed: \(String(describing: error), privacy: .public)") + } + } + } + + // MARK: Transport controls (parity with web UI play/pause/seek/stop) + + func togglePlayPause() async { + guard currentTrackID != nil, loadedAudioPath != nil else { return } + if player.rate.isZero { + player.play() + if let room { try? await api.control(room: room, action: "resume") } + } else { + player.pause() + if let room { try? await api.control(room: room, action: "pause") } + } + } + + func seek(by delta: Double) async { + guard loadedAudioPath != nil else { return } + let target = max(0, min(duration > 0 ? duration - 0.5 : .infinity, currentTime + delta)) + if target.isFinite { + seek(to: target) + } + report(player.rate.isZero ? "paused" : "playing") + } + + func seek(to position: Double) { + player.seek(to: CMTime(seconds: position, preferredTimescale: 600)) + currentTime = position + } + + func stop() async { + clearMedia() + if let room { try? await api.control(room: room, action: "stop") } + publishMediaState("stopped") + } + + private func clearMedia() { + currentTrackID = nil + track = nil + loadedAudioPath = nil + observeItem(nil) + player.replaceCurrentItem(with: nil) + currentTime = 0 + duration = 0 + } + + // MARK: Volume + + func applyVolume() { + player.volume = Float(settings.volume) + } + + func setVolume(_ volume: Double) { + settings.volume = volume + player.volume = Float(volume) + } +} diff --git a/Sources/Media/PinnedMediaResourceLoader.swift b/Sources/Media/PinnedMediaResourceLoader.swift new file mode 100644 index 0000000..6964be6 --- /dev/null +++ b/Sources/Media/PinnedMediaResourceLoader.swift @@ -0,0 +1,110 @@ +import AVFoundation +import Foundation +import os + +private let logger = Logger(subsystem: "dev.shane.voice", category: "media") + +/// Proxies {base}/media/* bytes to AVPlayer over the pinned-TLS session, since +/// AVPlayer cannot trust the self-signed LAN cert. AVPlayer loads https natively +/// (ignoring registered URLProtocol subclasses), so the asset uses a custom +/// `pinned-media` scheme and this delegate answers every data request by +/// fetching the real https URL through TLSPinning.session(), forwarding Range. +final class PinnedMediaResourceLoader: NSObject, AVAssetResourceLoaderDelegate { + + static let scheme = "pinned-media" + + private let base: URL + private var loaderURLs: [ObjectIdentifier: URL] = [:] + + init(base: URL) { + self.base = base + } + + /// Maps a media URL (absolute https or /media/... path) to the pinned-media + /// scheme so AVPlayer routes it through this delegate instead of loading + /// https natively. Built as a string because URLComponents.url returns nil + /// for schemes not in Foundation's known-scheme list. + func rewrite(_ url: URL) -> URL { + let absolute: String + if let scheme = url.scheme?.lowercased(), scheme.hasPrefix("http") { + var c = URLComponents(url: url, resolvingAgainstBaseURL: false)! + c.scheme = Self.scheme + return c.url! + } else { + let path = url.absoluteString.hasPrefix("/") ? String(url.absoluteString.dropFirst()) : url.absoluteString + guard let host = base.host, !host.isEmpty else { return url } + let port = base.port.map { ":\($0)" } ?? "" + absolute = "\(Self.scheme)://\(host)\(port)/\(path)" + } + return URL(string: absolute) ?? url + } + + /// Maps a pinned-media URL back to the real https URL it proxies. + func resolve(_ url: URL) -> URL { + guard let scheme = url.scheme?.lowercased(), scheme == Self.scheme else { return url } + var c = URLComponents(url: url, resolvingAgainstBaseURL: false)! + c.scheme = "https" + return c.url ?? url + } + + func assetWithPinnedURL(_ pinned: URL) -> AVURLAsset { + let asset = AVURLAsset(url: pinned) + loaderURLs[ObjectIdentifier(asset.resourceLoader)] = resolve(pinned) + asset.resourceLoader.setDelegate(self, queue: .main) + return asset + } + + func resourceLoader( + _ resourceLoader: AVAssetResourceLoader, + shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest + ) -> Bool { + guard let dataRequest = loadingRequest.dataRequest else { return false } + guard let url = loaderURLs[ObjectIdentifier(resourceLoader)] else { + logger.log("no resolved URL for resource loader") + loadingRequest.finishLoading(with: URLError(.badURL)) + return false + } + + var upstream = URLRequest(url: url) + let offset = dataRequest.requestedOffset + let length = Int64(dataRequest.requestedLength) + if length > 0 { + upstream.setValue("bytes=\(offset)-\(offset + length - 1)", forHTTPHeaderField: "Range") + } else { + upstream.setValue("bytes=\(offset)-", forHTTPHeaderField: "Range") + } + logger.log("fetching \(url.absoluteString, privacy: .public) range=\(upstream.value(forHTTPHeaderField: "Range") ?? "-", privacy: .public)") + + let task = TLSPinning.session().dataTask(with: upstream) { data, response, error in + if let error { + logger.log("fetch failed \(url.absoluteString, privacy: .public): \(error.localizedDescription, privacy: .public)") + loadingRequest.finishLoading(with: error) + return + } + guard let http = response as? HTTPURLResponse else { + loadingRequest.finishLoading(with: URLError(.badServerResponse)) + return + } + if let info = loadingRequest.contentInformationRequest { + info.contentType = http.mimeType ?? "application/octet-stream" + if http.expectedContentLength >= 0 { + info.contentLength = Int64(http.expectedContentLength) + } + info.isByteRangeAccessSupported = true + } + if let data { + dataRequest.respond(with: data) + } + loadingRequest.finishLoading() + } + task.resume() + return true + } + + func resourceLoader( + _ resourceLoader: AVAssetResourceLoader, + didCancel loadingRequest: AVAssetResourceLoadingRequest + ) { + logger.log("resource loader cancelled request") + } +} diff --git a/Voice.xcodeproj/project.pbxproj b/Voice.xcodeproj/project.pbxproj index a3c4208..06e5d23 100644 --- a/Voice.xcodeproj/project.pbxproj +++ b/Voice.xcodeproj/project.pbxproj @@ -14,15 +14,19 @@ 3E01A0AF44FD3324FA72C43C /* OrbView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */; }; 64291BBC8D0DAD47E07E94E0 /* ConversationState.swift in Sources */ = {isa = PBXBuildFile; fileRef = A0C23305A6C55A83F8617D9D /* ConversationState.swift */; }; 720C26EC163AE46713B02DD8 /* TLSPinning.swift in Sources */ = {isa = PBXBuildFile; fileRef = D9B40076934C28BA4B92D808 /* TLSPinning.swift */; }; + 73DA73811F8AEB2AFE340D10 /* MediaPlayerController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3964F565A6C6A06399EBA4DF /* MediaPlayerController.swift */; }; 89B47754E5C39418BFD839E6 /* TranscriptHistory.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68A45228EF59186CFC219F2E /* TranscriptHistory.swift */; }; 8DE1363CC0119B5AF9E4E117 /* ConnectionTestView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC4B9073EE8EAB51F8C55760 /* ConnectionTestView.swift */; }; 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48F76881B4AE51DEE74511CC /* SettingsView.swift */; }; + A86EFB492F40C26D4AA315E9 /* PinnedMediaResourceLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6DC3AC57ED58FC517E212F19 /* PinnedMediaResourceLoader.swift */; }; A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */; }; C00D1878C2AB3D8B6F9562B6 /* AgentMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0FC167F6658CFEC9546A7E9F /* AgentMonitor.swift */; }; + C956D0FDFAC43925979E4077 /* MediaAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9191AD60FF96E44F191864E3 /* MediaAPI.swift */; }; CDE38FE9679472D770C601E6 /* VoiceSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB58EEF9490F81FB2E1CE31C /* VoiceSession.swift */; }; D18AD347DFF72AB6361165BA /* TokenClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9681E7410B684D2D38752A70 /* TokenClient.swift */; }; D2E343C8B8657316A993698E /* TranscriptHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D82D665165A7BC1A4CBCCC6F /* TranscriptHandler.swift */; }; D9A0059E342955A06A021D6C /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = A6BAD55715F383C0DB5F306F /* LiveKit */; }; + DE1F24919E906EE13319C7A3 /* MediaPanelView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68FACB1A5EA6FEFD1D2613E9 /* MediaPanelView.swift */; }; EE0C5762EC97B123A1880FFB /* Voices.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E5DF207EDB787F63CB3A64D /* Voices.swift */; }; /* End PBXBuildFile section */ @@ -31,11 +35,15 @@ 2A4C3CE2CA6FF828452803D1 /* Voice.app */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.application; path = Voice.app; sourceTree = BUILT_PRODUCTS_DIR; }; 2E5DF207EDB787F63CB3A64D /* Voices.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Voices.swift; sourceTree = ""; }; 2FED8976B60FBC13BC980FCB /* VoiceApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceApp.swift; sourceTree = ""; }; + 3964F565A6C6A06399EBA4DF /* MediaPlayerController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlayerController.swift; sourceTree = ""; }; 3F3FA9F420065D2127B0AEC0 /* ConversationView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConversationView.swift; sourceTree = ""; }; 48F76881B4AE51DEE74511CC /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = ""; }; 61C71C48230C683D199D6F4B /* ControlChannel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlChannel.swift; sourceTree = ""; }; 68A45228EF59186CFC219F2E /* TranscriptHistory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscriptHistory.swift; sourceTree = ""; }; + 68FACB1A5EA6FEFD1D2613E9 /* MediaPanelView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPanelView.swift; sourceTree = ""; }; 6D0D153557BB1C756ED04C61 /* SettingsStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsStore.swift; sourceTree = ""; }; + 6DC3AC57ED58FC517E212F19 /* PinnedMediaResourceLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinnedMediaResourceLoader.swift; sourceTree = ""; }; + 9191AD60FF96E44F191864E3 /* MediaAPI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaAPI.swift; sourceTree = ""; }; 9681E7410B684D2D38752A70 /* TokenClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TokenClient.swift; sourceTree = ""; }; A0C23305A6C55A83F8617D9D /* ConversationState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConversationState.swift; sourceTree = ""; }; A62BAE22E3A165DCB5DFFEE1 /* OrbView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OrbView.swift; sourceTree = ""; }; @@ -58,6 +66,14 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 0D2E974CBC286D7C886BA814 /* Media */ = { + isa = PBXGroup; + children = ( + 68FACB1A5EA6FEFD1D2613E9 /* MediaPanelView.swift */, + ); + path = Media; + sourceTree = ""; + }; 0DF0591DF1D767F89099F62C /* Settings */ = { isa = PBXGroup; children = ( @@ -111,6 +127,7 @@ isa = PBXGroup; children = ( 84ADAF12A43A57888CA178A6 /* Conversation */, + 0D2E974CBC286D7C886BA814 /* Media */, 0DF0591DF1D767F89099F62C /* Settings */, ); path = Features; @@ -124,6 +141,16 @@ ); sourceTree = ""; }; + C12426487989358DE21A5426 /* Media */ = { + isa = PBXGroup; + children = ( + 9191AD60FF96E44F191864E3 /* MediaAPI.swift */, + 3964F565A6C6A06399EBA4DF /* MediaPlayerController.swift */, + 6DC3AC57ED58FC517E212F19 /* PinnedMediaResourceLoader.swift */, + ); + path = Media; + sourceTree = ""; + }; CE49A38E40AAF659A9281B85 /* App */ = { isa = PBXGroup; children = ( @@ -140,6 +167,7 @@ CE49A38E40AAF659A9281B85 /* App */, 8DA8BEF2EE3E0F4181A94EB9 /* Features */, 2820A418F921E18A2FA7E4FB /* LiveKit */, + C12426487989358DE21A5426 /* Media */, 6B2AB34274D9B37E77178C6C /* Networking */, ); path = Sources; @@ -213,7 +241,11 @@ 28659D009399CA9C4FE57B74 /* ControlChannel.swift in Sources */, 64291BBC8D0DAD47E07E94E0 /* ConversationState.swift in Sources */, 36DF44D9A7896F18A064847D /* ConversationView.swift in Sources */, + C956D0FDFAC43925979E4077 /* MediaAPI.swift in Sources */, + DE1F24919E906EE13319C7A3 /* MediaPanelView.swift in Sources */, + 73DA73811F8AEB2AFE340D10 /* MediaPlayerController.swift in Sources */, 3E01A0AF44FD3324FA72C43C /* OrbView.swift in Sources */, + A86EFB492F40C26D4AA315E9 /* PinnedMediaResourceLoader.swift in Sources */, 302BEC55CA9A10E49C307977 /* ServerConfig.swift in Sources */, A94F6C509E3FE215115B2C58 /* SettingsStore.swift in Sources */, 98BA1A0F92DB7B98A8BA7814 /* SettingsView.swift in Sources */,