This repository has been archived on 2026-10-04. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
hope-voice-ios/Sources/Media/MediaPlayerController.swift
T

634 lines
27 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import AVFoundation
import Combine
import Foundation
import os
private let logger = Logger(subsystem: "dev.shane.voice", category: "media")
/// AVPlayer-backed media player. The media server holds per-room desired state;
/// this controller polls /state and reconciles the local player against it,
/// porting the web UI's reconciliation rules (app.js reconcileMedia).
@MainActor
final class MediaPlayerController: ObservableObject {
@Published private(set) var track: MediaTrack?
@Published private(set) var desired: String = "stopped"
@Published private(set) var currentTime: Double = 0
@Published private(set) var duration: Double = 0
@Published private(set) var pendingJobs: [MediaPendingJob] = []
@Published private(set) var choices: [MediaChoice] = []
@Published private(set) var stateError: String?
@Published private(set) var playbackError: String?
let player = AVPlayer()
private let api: MediaAPI
private let settings: SettingsStore
private let mediaLoader: PinnedMediaResourceLoader
private weak var session: VoiceSession?
private var room: String?
private var currentTrackID: String?
private var loadedAudioPath: String?
private var currentAsset: AVURLAsset?
private var appliedSeekTs: Double = 0
private var wasPlayingBeforeTalk = false
private var lastReportDate = Date.distantPast
private var pollTask: Task<Void, Never>?
private var timeObserver: Any?
private var statusCancellable: AnyCancellable?
private var rateObservation: NSKeyValueObservation?
/// Bumped on every room-scope change so in-flight /state polls from the
/// previous scope cannot reconcile into the new one (port of the web UI's
/// `=== room` checks).
private var pollGeneration = 0
init(api: MediaAPI, settings: SettingsStore) {
self.api = api
self.settings = settings
self.mediaLoader = PinnedMediaResourceLoader(base: api.serverBase)
player.volume = Float(settings.volume)
NotificationCenter.default.addObserver(
self, selector: #selector(handleInterruption),
name: AVAudioSession.interruptionNotification, object: nil
)
}
deinit {
if let timeObserver {
player.removeTimeObserver(timeObserver)
}
rateObservation?.invalidate()
NotificationCenter.default.removeObserver(self)
}
/// Whether an audio session interruption is currently active. While true,
/// reconciliation must not call player.play() (the next poll would restart
/// audio during a phone call or Siri interaction).
private var isInterruptionActive = false
/// Separate resume-suppression flag: set when an interruption ends without
/// `.shouldResume` and the pause control to the server has not yet been
/// confirmed. While true, reconciliation must not call player.play() even
/// after `isInterruptionActive` is cleared. Cleared only by an explicit user
/// play/resume action or a confirmed coherent paused state from the server.
private var resumeSuppressed = false
/// Whether playback was actually active (rate > 0) when the interruption
/// began. Used to decide whether a `.shouldResume` end should actually
/// resume (a track that was already paused before the interruption must not
/// be force-resumed).
private var wasPlayingBeforeInterruption = false
/// Sequence number for media interruption operations. Incremented on every
/// `.began`; a resume task captures the sequence and rechecks it immediately
/// before `player.play()`. A newer `.began` makes an old resume task
/// incapable of playing (Iteration 07 Item 5).
private var mediaInterruptionSequence = 0
/// The retained operation chain for media interruption control work. Both
/// pause and resume operations are assigned here so a newer begin cancels
/// and supersedes prior control work.
private var mediaInterruptionTask: Task<Void, Never>?
@objc private func handleInterruption(_ notification: Notification) {
guard let info = notification.userInfo else { return }
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
switch type {
case .began:
// Increment the sequence so any in-flight resume task from a prior
// interruption sees a stale sequence and cannot play.
mediaInterruptionSequence += 1
let seq = mediaInterruptionSequence
// Cancel/supersede prior control work (resume or pause) so it cannot
// complete out of order.
mediaInterruptionTask?.cancel()
isInterruptionActive = true
wasPlayingBeforeInterruption = !player.rate.isZero
let wasPlaying = wasPlayingBeforeInterruption
player.pause()
logger.log("audio interruption began, paused media (wasPlaying=\(wasPlaying), seq=\(seq))")
case .ended:
defer { isInterruptionActive = false }
applyVolume()
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
if shouldResume, wasPlayingBeforeInterruption, currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
// Synchronize the server before resuming local playback so the
// two never diverge. Capture generation/room/sequence before the
// async hop. The task is retained in mediaInterruptionTask so a
// newer .began can cancel it.
let gen = pollGeneration
let targetRoom = room
let seq = mediaInterruptionSequence
mediaInterruptionTask = Task { [weak self] in
guard let self else { return }
if let targetRoom {
do {
try await self.api.control(room: targetRoom, action: "resume")
logger.log("interruption resume: server synchronized (seq=\(seq))")
} catch {
logger.log("interruption resume: server sync failed (poll will retry): \(error.localizedDescription)")
}
}
// Recheck the sequence immediately before play: a newer .began
// increments mediaInterruptionSequence, making this stale task
// incapable of restarting audio during the newer interruption.
guard !Task.isCancelled,
self.mediaInterruptionSequence == seq,
!self.isInterruptionActive,
self.pollGeneration == gen,
self.room == targetRoom else { return }
self.player.play()
logger.log("audio interruption ended, resumed media (seq=\(seq))")
}
} else {
// No-resume path: keep suppression even if the pause control
// fails. The flag is cleared only by an explicit user play/resume
// action or a confirmed coherent paused state from the server.
resumeSuppressed = true
let seq = mediaInterruptionSequence
if let room {
mediaInterruptionTask = Task { [weak self] in
guard let self else { return }
do {
try await self.api.control(room: room, action: "pause")
logger.log("audio interruption ended without resume, sent pause to server (seq=\(seq))")
} catch {
// Keep suppression even on failure — the next poll
// will see desired=="paused" and clear it.
logger.log("audio interruption ended without resume, pause control failed (suppression retained): \(error.localizedDescription)")
}
}
}
}
@unknown default:
break
}
}
// MARK: Lifecycle
func attach(session: VoiceSession) {
self.session = session
observePlayer()
observePlaybackState()
applyVolume()
}
/// Called just before joining the LiveKit room. Carries whatever is loaded
/// into the new room-scoped player state, paused (talking pauses music).
func enterMediaScope(roomName: String) async {
pollGeneration += 1
let wasPlaying = currentTrackID != nil && !player.rate.isZero
wasPlayingBeforeTalk = wasPlaying
let carriedTrack = currentTrackID
room = roomName
if let carriedTrack {
do {
try await api.play(room: roomName, trackID: carriedTrack, startPaused: true)
logger.log("media scope carry-over: \(carriedTrack, privacy: .public) start_paused")
} catch {
logger.log("media scope transfer failed: \(String(describing: error), privacy: .public)")
}
player.pause()
}
startPolling()
}
/// Called when the conversation ends. If music was playing before the talk,
/// hand the room back to the player. The previous scope (room name) is kept
/// so the panel can resume polling it via `startDefaultPolling()`; a fresh
/// conversation re-scopes via `enterMediaScope`. Polling restarts here so
/// the resumed state stays visible without the panel being open.
func endConversation() async {
stopPolling()
pollGeneration += 1
if wasPlayingBeforeTalk {
wasPlayingBeforeTalk = false
guard let room else { return }
do {
try await api.control(room: room, action: "resume")
logger.log("resumed playback after talk ended")
} catch {
logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)")
}
}
startPolling()
}
/// Drop the room scope entirely (app-level teardown). After this, polling
/// targets the default room again.
func resetScope() {
stopPolling()
pollGeneration += 1
room = nil
wasPlayingBeforeTalk = false
clearMedia()
}
private func startPolling() {
guard pollTask == nil else { return }
pollTask = Task { [weak self] in
while !Task.isCancelled {
await self?.pollState()
try? await Task.sleep(for: .milliseconds(1500))
}
}
}
private func stopPolling() {
pollTask?.cancel()
pollTask = nil
}
/// Resume polling whenever the panel becomes visible and nothing is being
/// polled yet — whether that's the default room (no session) or a previous
/// conversation's room scope (kept after `endConversation`).
func startDefaultPolling() {
guard pollTask == nil else { return }
startPolling()
}
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
private func pollState() async {
let generation = pollGeneration
let scope = room ?? "default"
do {
let state = try await api.state(room: scope)
guard generation == pollGeneration else { return }
stateError = nil
reconcile(state)
} catch {
guard generation == pollGeneration else { return }
logger.log("poll failed: \(String(describing: error), privacy: .public)")
stateError = "Media server unreachable: \(error.localizedDescription)"
}
}
/// Reconciles the local player against a server state snapshot. Exposed
/// as internal (not private) so tests can drive deterministic state
/// transitions without a live server.
func reconcile(_ s: MediaState) {
choices = s.choices
pendingJobs = s.pending
// Keep the published desired state in sync with the server so the UI's
// play/pause icon and status text reflect reality (the server is the
// source of truth for per-room desired state).
desired = s.desired ?? "stopped"
let track = s.track
self.track = track
if let track, track.id != currentTrackID {
loadTrack(track)
} else if let track, let audioURL = track.audioURL, loadedAudioPath != audioURL {
// Track was loaded before it had a stream (still downloading) and
// now has one — wire it up so playback can start without a reload.
wireSource(audioURL)
}
if track == nil, currentTrackID != nil, s.desired == "stopped" {
clearMedia()
}
applyDesiredState(s, track: track)
if let hint = s.seekHint, hint.ts != appliedSeekTs, duration.isFinite, duration > 0 {
appliedSeekTs = hint.ts
let target = min(hint.position, duration)
if target.isFinite {
seek(to: max(0, target))
}
}
}
private func applyDesiredState(_ s: MediaState, track: MediaTrack?) {
guard let desiredValue = s.desired else { return }
switch desiredValue {
case "playing":
// Suppress play during an active audio interruption (phone call,
// Siri). Without this guard the next 1.5s poll would restart audio
// immediately after .began paused it. The player.rate.isZero check
// makes this idempotent: once playing, subsequent polls are no-ops.
// This also handles the late-audio case where desired=playing
// arrives before audio_url is set; when the source is later wired
// via wireSource(), the next poll retries play because rate is
// still zero.
// Also suppress while resumeSuppressed is true (no-resume
// interruption ended, server not yet confirmed paused).
guard !isInterruptionActive, !resumeSuppressed else { return }
if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero {
player.play()
}
case "paused", "stopped":
if !player.rate.isZero {
player.pause()
}
// The server has confirmed a coherent paused/stopped state. If we
// were suppressing resume due to a no-resume interruption, the
// pause control succeeded (or the user paused independently), so
// it is safe to clear the suppression flag.
if resumeSuppressed {
resumeSuppressed = false
logger.log("resume suppression cleared: server confirmed \(desiredValue)")
}
default:
break
}
}
private func loadTrack(_ track: MediaTrack) {
currentTrackID = track.id
self.track = track
if let audioURL = track.audioURL {
wireSource(audioURL)
} else {
// Downloading or failed: no stream yet. Reconcile wires the stream
// URL in via polling as soon as one is available.
clearCurrentAsset()
player.replaceCurrentItem(with: nil)
}
}
private func wireSource(_ path: String) {
// A complete file is required for the server's cached AAC conversion.
// Keep showing download progress; the next ready-state poll wires it.
guard track?.kind != "youtube" || track?.status != "downloading" else { return }
guard var components = URLComponents(string: path) else { return }
if track?.kind == "youtube", components.path.hasPrefix("/media/api/stream/") {
var items = components.queryItems ?? []
items.removeAll { $0.name == "format" }
items.append(URLQueryItem(name: "format", value: "m4a"))
components.queryItems = items
}
guard let url = components.url else { return }
loadedAudioPath = path
let pinned = mediaLoader.rewrite(url)
// Remove the previous asset's loader mapping before wiring a new one so
// the loaderURLs dictionary stays bounded (Finding 11 / Item 7).
clearCurrentAsset()
let asset = mediaLoader.assetWithPinnedURL(pinned)
currentAsset = asset
logger.log("wiring source \(pinned.absoluteString, privacy: .public)")
let item = AVPlayerItem(asset: asset)
observeItem(item)
player.replaceCurrentItem(with: item)
}
/// Removes the loader mapping for the current asset (if any) and clears the
/// reference. Called before wiring a new source or clearing media so the
/// resource loader's internal dictionaries stay bounded.
private func clearCurrentAsset() {
if let asset = currentAsset {
mediaLoader.removeLoader(asset)
currentAsset = nil
}
}
// MARK: Player events
private func observePlayer() {
if timeObserver == nil {
timeObserver = player.addPeriodicTimeObserver(
forInterval: CMTime(seconds: 0.5, preferredTimescale: 600), queue: .main
) { [weak self] time in
Task { @MainActor [weak self] in
guard let self else { return }
self.currentTime = time.seconds
if let itemDuration = self.player.currentItem?.duration {
let seconds = itemDuration.seconds
if seconds.isFinite {
self.duration = seconds
}
}
if !self.player.rate.isZero, Date().timeIntervalSince(self.lastReportDate) >= 5 {
self.lastReportDate = Date()
self.report("playing")
}
}
}
}
}
// MARK: - Playback state observation
/// Observe the player's actual rate so every real play/pause transition is
/// reported promptly, mirroring the web UI's `play`/`pause` listeners. The
/// KVO block runs on an arbitrary queue; hop to MainActor before touching
/// any controller state.
private func observePlaybackState() {
rateObservation = player.observe(\.rate, options: [.initial, .new]) { [weak self] player, _ in
Task { @MainActor [weak self] in
guard let self else { return }
guard self.currentTrackID != nil else { return }
if player.rate > 0 {
// Actual playback began (user play, interruption resume, or
// reconcile-driven play). Report + publish the transition.
self.lastReportDate = Date()
self.report("playing")
self.publishMediaState("playing")
} else if player.currentItem != nil {
// Rate dropped to zero with an item still loaded: a pause
// that didn't originate from a user action (interruption,
// reconcile). User-initiated pauses already report before
// this fires; the lastReportDate guard makes duplicates no-ops.
if Date().timeIntervalSince(self.lastReportDate) >= 1 {
self.lastReportDate = Date()
self.report("paused")
self.publishMediaState("paused")
}
}
}
}
}
private func observeItem(_ item: AVPlayerItem?) {
NotificationCenter.default.removeObserver(
self, name: .AVPlayerItemDidPlayToEndTime, object: nil
)
guard let item else { return }
NotificationCenter.default.addObserver(
self, selector: #selector(playerItemDidEnd(_:)),
name: .AVPlayerItemDidPlayToEndTime, object: item
)
statusCancellable = item.publisher(for: \.status).sink { [weak self, weak item] status in
Task { @MainActor in
guard let self, let item, self.player.currentItem === item else { return }
switch status {
case .readyToPlay:
self.playbackError = nil
case .failed:
self.playbackError = "This audio couldn’t play on your iPhone. Try another track or an MP3/AAC station."
logger.log("item failed: \(String(describing: item.error), privacy: .public)")
default: break
}
}
}
}
@objc private func playerItemDidEnd(_ notification: Notification) {
lastReportDate = Date()
report("stopped")
publishMediaState("stopped")
Task { [weak self] in
guard let self else { return }
try? await self.api.control(room: self.room ?? "default", action: "stop")
}
}
// MARK: Reporting + publishing
private func report(_ state: String, trackID: String? = nil) {
let room = room ?? "default"
let position = currentTime.isFinite ? currentTime : 0
var durationValue: Double?
if duration.isFinite, duration > 0 {
durationValue = duration
}
let resolvedTrackID = trackID ?? currentTrackID
Task { [weak self] in
guard let self else { return }
do {
try await self.api.report(room: room, state: state, trackID: resolvedTrackID, position: position, duration: durationValue)
} catch {
logger.log("report failed: \(String(describing: error), privacy: .public)")
}
}
}
private func publishMediaState(_ state: String, trackID: String? = nil) {
guard let session else { return }
// Capture the exact room and generation before spawning so a delayed
// publish from an old lifecycle cannot escape into a new conversation
// room (Finding 10).
let targetRoom = session.room
let generation = session.sessionGeneration
let pollGen = pollGeneration
let resolvedTrackID = trackID ?? currentTrackID
Task { [weak self] in
guard let self, self.pollGeneration == pollGen else { return }
guard session.sessionGeneration == generation, session.room === targetRoom else { return }
do {
try await session.controlChannel.sendMediaState(state, trackID: resolvedTrackID, room: targetRoom)
} catch {
logger.log("media_state publish failed: \(String(describing: error), privacy: .public)")
}
}
}
// MARK: Play requests (search results, choices, library)
func playVideo(videoID: String?, title: String?, channel: String?) async {
let targetRoom = room ?? "default"
let generation = pollGeneration
playbackError = nil
do {
try await api.play(room: targetRoom, videoID: videoID, title: title, channel: channel)
logger.log("play requested video_id=\(videoID ?? "nil", privacy: .public) room=\(targetRoom, privacy: .public)")
} catch {
guard generation == pollGeneration else { return }
playbackError = "Couldn’t play this song. Check your connection and try again."
logger.log("play failed: \(String(describing: error), privacy: .public)")
}
}
func playTrack(id: String) async {
let targetRoom = room ?? "default"
let generation = pollGeneration
playbackError = nil
do {
try await api.play(room: targetRoom, trackID: id)
logger.log("play requested track_id=\(id, privacy: .public) room=\(targetRoom, privacy: .public)")
} catch {
guard generation == pollGeneration else { return }
playbackError = "Couldn’t play this track. Check your connection and try again."
logger.log("play failed: \(String(describing: error), privacy: .public)")
}
}
// MARK: Transport controls (parity with web UI play/pause/seek/stop)
func togglePlayPause() async {
guard currentTrackID != nil, loadedAudioPath != nil else { return }
if player.rate.isZero {
// Explicit user play: clear any resume suppression (the user is
// deliberately overriding the no-resume interruption state).
resumeSuppressed = false
// The rate observer reports + publishes the actual transition;
// the control call updates server desired state.
player.play()
await sendControl("resume")
} else {
lastReportDate = Date()
report("paused")
publishMediaState("paused")
player.pause()
await sendControl("pause")
}
}
func seek(by delta: Double) async {
guard loadedAudioPath != nil else { return }
let target = max(0, min(duration > 0 ? duration - 0.5 : .infinity, currentTime + delta))
if target.isFinite {
seek(to: target)
}
}
func seek(to position: Double) {
guard position.isFinite, position >= 0 else { return }
player.seek(to: CMTime(seconds: position, preferredTimescale: 600))
currentTime = position
report(player.rate.isZero ? "paused" : "playing")
}
func stop() async {
// Capture the track ID before clearing so the async report/publish
// tasks can read a non-nil value even after clearMedia() runs.
let trackID = currentTrackID
report("stopped", trackID: trackID)
clearMedia()
await sendControl("stop")
publishMediaState("stopped", trackID: trackID)
}
private func clearMedia() {
playbackError = nil
currentTrackID = nil
track = nil
loadedAudioPath = nil
// Remove the loader mapping for the asset being cleared so the
// resource loader's internal dictionaries stay bounded (Item 7).
clearCurrentAsset()
observeItem(nil)
player.replaceCurrentItem(with: nil)
currentTime = 0
duration = 0
stateError = nil
}
private func sendControl(_ action: String) async {
let generation = pollGeneration
do {
try await api.control(room: room ?? "default", action: action)
guard generation == pollGeneration else { return }
playbackError = nil
} catch {
guard generation == pollGeneration else { return }
playbackError = "Couldn’t update playback on the server. Check your connection and try again."
}
}
// MARK: Volume
func applyVolume() {
player.volume = Float(settings.volume)
}
func setVolume(_ volume: Double) {
settings.volume = volume
player.volume = Float(volume)
}
}