import AVFoundation import Combine import Foundation import os private let logger = Logger(subsystem: "dev.shane.voice", category: "media") /// AVPlayer-backed media player. The media server holds per-room desired state; /// this controller polls /state and reconciles the local player against it, /// porting the web UI's reconciliation rules (app.js reconcileMedia). @MainActor final class MediaPlayerController: ObservableObject { @Published private(set) var track: MediaTrack? @Published private(set) var desired: String = "stopped" @Published private(set) var currentTime: Double = 0 @Published private(set) var duration: Double = 0 @Published private(set) var pendingJobs: [MediaPendingJob] = [] @Published private(set) var choices: [MediaChoice] = [] @Published private(set) var stateError: String? let player = AVPlayer() private let api: MediaAPI private let settings: SettingsStore private let mediaLoader: PinnedMediaResourceLoader private weak var session: VoiceSession? private var room: String? private var currentTrackID: String? private var loadedAudioPath: String? private var currentAsset: AVURLAsset? private var appliedSeekTs: Double = 0 private var wasPlayingBeforeTalk = false private var lastReportDate = Date.distantPast private var pollTask: Task? private var timeObserver: Any? private var statusCancellable: AnyCancellable? private var rateObservation: NSKeyValueObservation? /// Bumped on every room-scope change so in-flight /state polls from the /// previous scope cannot reconcile into the new one (port of the web UI's /// `=== room` checks). private var pollGeneration = 0 init(api: MediaAPI, settings: SettingsStore) { self.api = api self.settings = settings self.mediaLoader = PinnedMediaResourceLoader(base: api.serverBase) player.volume = Float(settings.volume) NotificationCenter.default.addObserver( self, selector: #selector(handleInterruption), name: AVAudioSession.interruptionNotification, object: nil ) } deinit { if let timeObserver { player.removeTimeObserver(timeObserver) } rateObservation?.invalidate() NotificationCenter.default.removeObserver(self) } /// Whether an audio session interruption is currently active. While true, /// reconciliation must not call player.play() (the next poll would restart /// audio during a phone call or Siri interaction). private var isInterruptionActive = false /// Separate resume-suppression flag: set when an interruption ends without /// `.shouldResume` and the pause control to the server has not yet been /// confirmed. While true, reconciliation must not call player.play() even /// after `isInterruptionActive` is cleared. Cleared only by an explicit user /// play/resume action or a confirmed coherent paused state from the server. private var resumeSuppressed = false /// Whether playback was actually active (rate > 0) when the interruption /// began. Used to decide whether a `.shouldResume` end should actually /// resume (a track that was already paused before the interruption must not /// be force-resumed). private var wasPlayingBeforeInterruption = false /// Sequence number for media interruption operations. Incremented on every /// `.began`; a resume task captures the sequence and rechecks it immediately /// before `player.play()`. A newer `.began` makes an old resume task /// incapable of playing (Iteration 07 Item 5). private var mediaInterruptionSequence = 0 /// The retained operation chain for media interruption control work. Both /// pause and resume operations are assigned here so a newer begin cancels /// and supersedes prior control work. private var mediaInterruptionTask: Task? @objc private func handleInterruption(_ notification: Notification) { guard let info = notification.userInfo else { return } guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt, let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return } switch type { case .began: // Increment the sequence so any in-flight resume task from a prior // interruption sees a stale sequence and cannot play. mediaInterruptionSequence += 1 let seq = mediaInterruptionSequence // Cancel/supersede prior control work (resume or pause) so it cannot // complete out of order. mediaInterruptionTask?.cancel() isInterruptionActive = true wasPlayingBeforeInterruption = !player.rate.isZero let wasPlaying = wasPlayingBeforeInterruption player.pause() logger.log("audio interruption began, paused media (wasPlaying=\(wasPlaying), seq=\(seq))") case .ended: defer { isInterruptionActive = false } applyVolume() let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0 let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume) if shouldResume, wasPlayingBeforeInterruption, currentTrackID != nil, desired == "playing", loadedAudioPath != nil { // Synchronize the server before resuming local playback so the // two never diverge. Capture generation/room/sequence before the // async hop. The task is retained in mediaInterruptionTask so a // newer .began can cancel it. let gen = pollGeneration let targetRoom = room let seq = mediaInterruptionSequence mediaInterruptionTask = Task { [weak self] in guard let self else { return } if let targetRoom { do { try await self.api.control(room: targetRoom, action: "resume") logger.log("interruption resume: server synchronized (seq=\(seq))") } catch { logger.log("interruption resume: server sync failed (poll will retry): \(error.localizedDescription)") } } // Recheck the sequence immediately before play: a newer .began // increments mediaInterruptionSequence, making this stale task // incapable of restarting audio during the newer interruption. guard !Task.isCancelled, self.mediaInterruptionSequence == seq, !self.isInterruptionActive, self.pollGeneration == gen, self.room == targetRoom else { return } self.player.play() logger.log("audio interruption ended, resumed media (seq=\(seq))") } } else { // No-resume path: keep suppression even if the pause control // fails. The flag is cleared only by an explicit user play/resume // action or a confirmed coherent paused state from the server. resumeSuppressed = true let seq = mediaInterruptionSequence if let room { mediaInterruptionTask = Task { [weak self] in guard let self else { return } do { try await self.api.control(room: room, action: "pause") logger.log("audio interruption ended without resume, sent pause to server (seq=\(seq))") } catch { // Keep suppression even on failure — the next poll // will see desired=="paused" and clear it. logger.log("audio interruption ended without resume, pause control failed (suppression retained): \(error.localizedDescription)") } } } } @unknown default: break } } // MARK: Lifecycle func attach(session: VoiceSession) { self.session = session observePlayer() observePlaybackState() applyVolume() } /// Called just before joining the LiveKit room. Carries whatever is loaded /// into the new room-scoped player state, paused (talking pauses music). func enterMediaScope(roomName: String) async { pollGeneration += 1 let wasPlaying = currentTrackID != nil && !player.rate.isZero wasPlayingBeforeTalk = wasPlaying let carriedTrack = currentTrackID room = roomName if let carriedTrack { do { try await api.play(room: roomName, trackID: carriedTrack, startPaused: true) logger.log("media scope carry-over: \(carriedTrack, privacy: .public) start_paused") } catch { logger.log("media scope transfer failed: \(String(describing: error), privacy: .public)") } player.pause() } startPolling() } /// Called when the conversation ends. If music was playing before the talk, /// hand the room back to the player. The previous scope (room name) is kept /// so the panel can resume polling it via `startDefaultPolling()`; a fresh /// conversation re-scopes via `enterMediaScope`. Polling restarts here so /// the resumed state stays visible without the panel being open. func endConversation() async { stopPolling() pollGeneration += 1 if wasPlayingBeforeTalk { wasPlayingBeforeTalk = false guard let room else { return } do { try await api.control(room: room, action: "resume") logger.log("resumed playback after talk ended") } catch { logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)") } } startPolling() } /// Drop the room scope entirely (app-level teardown). After this, polling /// targets the default room again. func resetScope() { stopPolling() pollGeneration += 1 room = nil wasPlayingBeforeTalk = false } private func startPolling() { guard pollTask == nil else { return } pollTask = Task { [weak self] in while !Task.isCancelled { await self?.pollState() try? await Task.sleep(for: .milliseconds(1500)) } } } private func stopPolling() { pollTask?.cancel() pollTask = nil } /// Resume polling whenever the panel becomes visible and nothing is being /// polled yet — whether that's the default room (no session) or a previous /// conversation's room scope (kept after `endConversation`). func startDefaultPolling() { guard pollTask == nil else { return } startPolling() } // MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia) private func pollState() async { let generation = pollGeneration let scope = room ?? "default" do { let state = try await api.state(room: scope) guard generation == pollGeneration else { return } stateError = nil reconcile(state) } catch { guard generation == pollGeneration else { return } logger.log("poll failed: \(String(describing: error), privacy: .public)") stateError = "Media server unreachable: \(error.localizedDescription)" } } /// Reconciles the local player against a server state snapshot. Exposed /// as internal (not private) so tests can drive deterministic state /// transitions without a live server. func reconcile(_ s: MediaState) { choices = s.choices pendingJobs = s.pending // Keep the published desired state in sync with the server so the UI's // play/pause icon and status text reflect reality (the server is the // source of truth for per-room desired state). desired = s.desired ?? "stopped" let track = s.track if let track, track.id != currentTrackID { loadTrack(track) } else if let track, let audioURL = track.audioURL, loadedAudioPath != audioURL { // Track was loaded before it had a stream (still downloading) and // now has one — wire it up so playback can start without a reload. wireSource(audioURL) } if track == nil, currentTrackID != nil, s.desired == "stopped" { clearMedia() } applyDesiredState(s, track: track) if let hint = s.seekHint, hint.ts != appliedSeekTs, duration.isFinite, duration > 0 { appliedSeekTs = hint.ts let target = min(hint.position, duration) if target.isFinite { seek(to: max(0, target)) } } } private func applyDesiredState(_ s: MediaState, track: MediaTrack?) { guard let desiredValue = s.desired else { return } switch desiredValue { case "playing": // Suppress play during an active audio interruption (phone call, // Siri). Without this guard the next 1.5s poll would restart audio // immediately after .began paused it. The player.rate.isZero check // makes this idempotent: once playing, subsequent polls are no-ops. // This also handles the late-audio case where desired=playing // arrives before audio_url is set; when the source is later wired // via wireSource(), the next poll retries play because rate is // still zero. // Also suppress while resumeSuppressed is true (no-resume // interruption ended, server not yet confirmed paused). guard !isInterruptionActive, !resumeSuppressed else { return } if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero { player.play() } case "paused", "stopped": if !player.rate.isZero { player.pause() } // The server has confirmed a coherent paused/stopped state. If we // were suppressing resume due to a no-resume interruption, the // pause control succeeded (or the user paused independently), so // it is safe to clear the suppression flag. if resumeSuppressed { resumeSuppressed = false logger.log("resume suppression cleared: server confirmed \(desiredValue)") } default: break } } private func loadTrack(_ track: MediaTrack) { currentTrackID = track.id self.track = track if let audioURL = track.audioURL { wireSource(audioURL) } else { // Downloading or failed: no stream yet. Reconcile wires the stream // URL in via polling as soon as one is available. clearCurrentAsset() player.replaceCurrentItem(with: nil) } } private func wireSource(_ path: String) { guard let url = URL(string: path) else { return } loadedAudioPath = path let pinned = mediaLoader.rewrite(url) // Remove the previous asset's loader mapping before wiring a new one so // the loaderURLs dictionary stays bounded (Finding 11 / Item 7). clearCurrentAsset() let asset = mediaLoader.assetWithPinnedURL(pinned) currentAsset = asset logger.log("wiring source \(pinned.absoluteString, privacy: .public)") let item = AVPlayerItem(asset: asset) observeItem(item) player.replaceCurrentItem(with: item) } /// Removes the loader mapping for the current asset (if any) and clears the /// reference. Called before wiring a new source or clearing media so the /// resource loader's internal dictionaries stay bounded. private func clearCurrentAsset() { if let asset = currentAsset { mediaLoader.removeLoader(asset) currentAsset = nil } } // MARK: Player events private func observePlayer() { if timeObserver == nil { timeObserver = player.addPeriodicTimeObserver( forInterval: CMTime(seconds: 0.5, preferredTimescale: 600), queue: .main ) { [weak self] time in Task { @MainActor [weak self] in guard let self else { return } self.currentTime = time.seconds if let itemDuration = self.player.currentItem?.duration { let seconds = itemDuration.seconds if seconds.isFinite { self.duration = seconds } } if !self.player.rate.isZero, Date().timeIntervalSince(self.lastReportDate) >= 5 { self.lastReportDate = Date() self.report("playing") } } } } } // MARK: - Playback state observation /// Observe the player's actual rate so every real play/pause transition is /// reported promptly, mirroring the web UI's `play`/`pause` listeners. The /// KVO block runs on an arbitrary queue; hop to MainActor before touching /// any controller state. private func observePlaybackState() { rateObservation = player.observe(\.rate, options: [.initial, .new]) { [weak self] player, _ in Task { @MainActor [weak self] in guard let self else { return } guard self.currentTrackID != nil else { return } if player.rate > 0 { // Actual playback began (user play, interruption resume, or // reconcile-driven play). Report + publish the transition. self.lastReportDate = Date() self.report("playing") self.publishMediaState("playing") } else if player.currentItem != nil { // Rate dropped to zero with an item still loaded: a pause // that didn't originate from a user action (interruption, // reconcile). User-initiated pauses already report before // this fires; the lastReportDate guard makes duplicates no-ops. if Date().timeIntervalSince(self.lastReportDate) >= 1 { self.lastReportDate = Date() self.report("paused") self.publishMediaState("paused") } } } } } private func observeItem(_ item: AVPlayerItem?) { NotificationCenter.default.removeObserver( self, name: .AVPlayerItemDidPlayToEndTime, object: nil ) guard let item else { return } NotificationCenter.default.addObserver( self, selector: #selector(playerItemDidEnd(_:)), name: .AVPlayerItemDidPlayToEndTime, object: item ) statusCancellable = item.publisher(for: \.status).sink { status in switch status { case .readyToPlay: logger.log("item readyToPlay") case .failed: logger.log("item failed: \(String(describing: item.error), privacy: .public)") default: break } } } @objc private func playerItemDidEnd(_ notification: Notification) { lastReportDate = Date() report("stopped") publishMediaState("stopped") Task { [weak self] in guard let self, let room = self.room else { return } try? await self.api.control(room: room, action: "stop") } } // MARK: Reporting + publishing private func report(_ state: String, trackID: String? = nil) { guard let room else { return } let position = currentTime.isFinite ? currentTime : 0 var durationValue: Double? if duration.isFinite, duration > 0 { durationValue = duration } let resolvedTrackID = trackID ?? currentTrackID Task { [weak self] in guard let self else { return } do { try await self.api.report(room: room, state: state, trackID: resolvedTrackID, position: position, duration: durationValue) } catch { logger.log("report failed: \(String(describing: error), privacy: .public)") } } } private func publishMediaState(_ state: String, trackID: String? = nil) { guard let session else { return } // Capture the exact room and generation before spawning so a delayed // publish from an old lifecycle cannot escape into a new conversation // room (Finding 10). let targetRoom = session.room let generation = session.sessionGeneration let pollGen = pollGeneration let resolvedTrackID = trackID ?? currentTrackID Task { [weak self] in guard let self, self.pollGeneration == pollGen else { return } guard session.sessionGeneration == generation, session.room === targetRoom else { return } do { try await session.controlChannel.sendMediaState(state, trackID: resolvedTrackID, room: targetRoom) } catch { logger.log("media_state publish failed: \(String(describing: error), privacy: .public)") } } } // MARK: Play requests (search results, choices, library) func playVideo(videoID: String?, title: String?, channel: String?) async { let targetRoom = room ?? "default" do { try await api.play(room: targetRoom, videoID: videoID, title: title, channel: channel) logger.log("play requested video_id=\(videoID ?? "nil", privacy: .public) room=\(targetRoom, privacy: .public)") } catch { logger.log("play failed: \(String(describing: error), privacy: .public)") } } func playTrack(id: String) async { let targetRoom = room ?? "default" do { try await api.play(room: targetRoom, trackID: id) logger.log("play requested track_id=\(id, privacy: .public) room=\(targetRoom, privacy: .public)") } catch { logger.log("play failed: \(String(describing: error), privacy: .public)") } } // MARK: Transport controls (parity with web UI play/pause/seek/stop) func togglePlayPause() async { guard currentTrackID != nil, loadedAudioPath != nil else { return } if player.rate.isZero { // Explicit user play: clear any resume suppression (the user is // deliberately overriding the no-resume interruption state). resumeSuppressed = false // The rate observer reports + publishes the actual transition; // the control call updates server desired state. player.play() if let room { try? await api.control(room: room, action: "resume") } } else { lastReportDate = Date() report("paused") publishMediaState("paused") player.pause() if let room { try? await api.control(room: room, action: "pause") } } } func seek(by delta: Double) async { guard loadedAudioPath != nil else { return } let target = max(0, min(duration > 0 ? duration - 0.5 : .infinity, currentTime + delta)) if target.isFinite { seek(to: target) } } func seek(to position: Double) { player.seek(to: CMTime(seconds: position, preferredTimescale: 600)) currentTime = position report(player.rate.isZero ? "paused" : "playing") } func stop() async { // Capture the track ID before clearing so the async report/publish // tasks can read a non-nil value even after clearMedia() runs. let trackID = currentTrackID report("stopped", trackID: trackID) clearMedia() if let room { try? await api.control(room: room, action: "stop") } publishMediaState("stopped", trackID: trackID) } private func clearMedia() { currentTrackID = nil track = nil loadedAudioPath = nil // Remove the loader mapping for the asset being cleared so the // resource loader's internal dictionaries stay bounded (Item 7). clearCurrentAsset() observeItem(nil) player.replaceCurrentItem(with: nil) currentTime = 0 duration = 0 stateError = nil } // MARK: Volume func applyVolume() { player.volume = Float(settings.volume) } func setVolume(_ volume: Double) { settings.volume = volume player.volume = Float(volume) } }