Archived
596 lines
25 KiB
Swift
596 lines
25 KiB
Swift
import AVFoundation
|
|
import Combine
|
|
import Foundation
|
|
import os
|
|
|
|
private let logger = Logger(subsystem: "dev.shane.voice", category: "media")
|
|
|
|
/// AVPlayer-backed media player. The media server holds per-room desired state;
|
|
/// this controller polls /state and reconciles the local player against it,
|
|
/// porting the web UI's reconciliation rules (app.js reconcileMedia).
|
|
@MainActor
|
|
final class MediaPlayerController: ObservableObject {
|
|
@Published private(set) var track: MediaTrack?
|
|
@Published private(set) var desired: String = "stopped"
|
|
@Published private(set) var currentTime: Double = 0
|
|
@Published private(set) var duration: Double = 0
|
|
@Published private(set) var pendingJobs: [MediaPendingJob] = []
|
|
@Published private(set) var choices: [MediaChoice] = []
|
|
@Published private(set) var stateError: String?
|
|
|
|
let player = AVPlayer()
|
|
|
|
private let api: MediaAPI
|
|
private let settings: SettingsStore
|
|
private let mediaLoader: PinnedMediaResourceLoader
|
|
private weak var session: VoiceSession?
|
|
|
|
private var room: String?
|
|
private var currentTrackID: String?
|
|
private var loadedAudioPath: String?
|
|
private var currentAsset: AVURLAsset?
|
|
private var appliedSeekTs: Double = 0
|
|
private var wasPlayingBeforeTalk = false
|
|
private var lastReportDate = Date.distantPast
|
|
private var pollTask: Task<Void, Never>?
|
|
private var timeObserver: Any?
|
|
private var statusCancellable: AnyCancellable?
|
|
private var rateObservation: NSKeyValueObservation?
|
|
|
|
/// Bumped on every room-scope change so in-flight /state polls from the
|
|
/// previous scope cannot reconcile into the new one (port of the web UI's
|
|
/// `=== room` checks).
|
|
private var pollGeneration = 0
|
|
|
|
init(api: MediaAPI, settings: SettingsStore) {
|
|
self.api = api
|
|
self.settings = settings
|
|
self.mediaLoader = PinnedMediaResourceLoader(base: api.serverBase)
|
|
player.volume = Float(settings.volume)
|
|
NotificationCenter.default.addObserver(
|
|
self, selector: #selector(handleInterruption),
|
|
name: AVAudioSession.interruptionNotification, object: nil
|
|
)
|
|
}
|
|
|
|
deinit {
|
|
if let timeObserver {
|
|
player.removeTimeObserver(timeObserver)
|
|
}
|
|
rateObservation?.invalidate()
|
|
NotificationCenter.default.removeObserver(self)
|
|
}
|
|
|
|
/// Whether an audio session interruption is currently active. While true,
|
|
/// reconciliation must not call player.play() (the next poll would restart
|
|
/// audio during a phone call or Siri interaction).
|
|
private var isInterruptionActive = false
|
|
|
|
/// Separate resume-suppression flag: set when an interruption ends without
|
|
/// `.shouldResume` and the pause control to the server has not yet been
|
|
/// confirmed. While true, reconciliation must not call player.play() even
|
|
/// after `isInterruptionActive` is cleared. Cleared only by an explicit user
|
|
/// play/resume action or a confirmed coherent paused state from the server.
|
|
private var resumeSuppressed = false
|
|
|
|
/// Whether playback was actually active (rate > 0) when the interruption
|
|
/// began. Used to decide whether a `.shouldResume` end should actually
|
|
/// resume (a track that was already paused before the interruption must not
|
|
/// be force-resumed).
|
|
private var wasPlayingBeforeInterruption = false
|
|
|
|
/// Sequence number for media interruption operations. Incremented on every
|
|
/// `.began`; a resume task captures the sequence and rechecks it immediately
|
|
/// before `player.play()`. A newer `.began` makes an old resume task
|
|
/// incapable of playing (Iteration 07 Item 5).
|
|
private var mediaInterruptionSequence = 0
|
|
|
|
/// The retained operation chain for media interruption control work. Both
|
|
/// pause and resume operations are assigned here so a newer begin cancels
|
|
/// and supersedes prior control work.
|
|
private var mediaInterruptionTask: Task<Void, Never>?
|
|
|
|
@objc private func handleInterruption(_ notification: Notification) {
|
|
guard let info = notification.userInfo else { return }
|
|
guard let typeValue = info[AVAudioSessionInterruptionTypeKey] as? UInt,
|
|
let type = AVAudioSession.InterruptionType(rawValue: typeValue) else { return }
|
|
switch type {
|
|
case .began:
|
|
// Increment the sequence so any in-flight resume task from a prior
|
|
// interruption sees a stale sequence and cannot play.
|
|
mediaInterruptionSequence += 1
|
|
let seq = mediaInterruptionSequence
|
|
// Cancel/supersede prior control work (resume or pause) so it cannot
|
|
// complete out of order.
|
|
mediaInterruptionTask?.cancel()
|
|
isInterruptionActive = true
|
|
wasPlayingBeforeInterruption = !player.rate.isZero
|
|
let wasPlaying = wasPlayingBeforeInterruption
|
|
player.pause()
|
|
logger.log("audio interruption began, paused media (wasPlaying=\(wasPlaying), seq=\(seq))")
|
|
case .ended:
|
|
defer { isInterruptionActive = false }
|
|
applyVolume()
|
|
let optionsValue = info[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0
|
|
let shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume)
|
|
if shouldResume, wasPlayingBeforeInterruption, currentTrackID != nil, desired == "playing", loadedAudioPath != nil {
|
|
// Synchronize the server before resuming local playback so the
|
|
// two never diverge. Capture generation/room/sequence before the
|
|
// async hop. The task is retained in mediaInterruptionTask so a
|
|
// newer .began can cancel it.
|
|
let gen = pollGeneration
|
|
let targetRoom = room
|
|
let seq = mediaInterruptionSequence
|
|
mediaInterruptionTask = Task { [weak self] in
|
|
guard let self else { return }
|
|
if let targetRoom {
|
|
do {
|
|
try await self.api.control(room: targetRoom, action: "resume")
|
|
logger.log("interruption resume: server synchronized (seq=\(seq))")
|
|
} catch {
|
|
logger.log("interruption resume: server sync failed (poll will retry): \(error.localizedDescription)")
|
|
}
|
|
}
|
|
// Recheck the sequence immediately before play: a newer .began
|
|
// increments mediaInterruptionSequence, making this stale task
|
|
// incapable of restarting audio during the newer interruption.
|
|
guard !Task.isCancelled,
|
|
self.mediaInterruptionSequence == seq,
|
|
!self.isInterruptionActive,
|
|
self.pollGeneration == gen,
|
|
self.room == targetRoom else { return }
|
|
self.player.play()
|
|
logger.log("audio interruption ended, resumed media (seq=\(seq))")
|
|
}
|
|
} else {
|
|
// No-resume path: keep suppression even if the pause control
|
|
// fails. The flag is cleared only by an explicit user play/resume
|
|
// action or a confirmed coherent paused state from the server.
|
|
resumeSuppressed = true
|
|
let seq = mediaInterruptionSequence
|
|
if let room {
|
|
mediaInterruptionTask = Task { [weak self] in
|
|
guard let self else { return }
|
|
do {
|
|
try await self.api.control(room: room, action: "pause")
|
|
logger.log("audio interruption ended without resume, sent pause to server (seq=\(seq))")
|
|
} catch {
|
|
// Keep suppression even on failure — the next poll
|
|
// will see desired=="paused" and clear it.
|
|
logger.log("audio interruption ended without resume, pause control failed (suppression retained): \(error.localizedDescription)")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@unknown default:
|
|
break
|
|
}
|
|
}
|
|
|
|
// MARK: Lifecycle
|
|
|
|
func attach(session: VoiceSession) {
|
|
self.session = session
|
|
observePlayer()
|
|
observePlaybackState()
|
|
applyVolume()
|
|
}
|
|
|
|
/// Called just before joining the LiveKit room. Carries whatever is loaded
|
|
/// into the new room-scoped player state, paused (talking pauses music).
|
|
func enterMediaScope(roomName: String) async {
|
|
pollGeneration += 1
|
|
let wasPlaying = currentTrackID != nil && !player.rate.isZero
|
|
wasPlayingBeforeTalk = wasPlaying
|
|
let carriedTrack = currentTrackID
|
|
room = roomName
|
|
if let carriedTrack {
|
|
do {
|
|
try await api.play(room: roomName, trackID: carriedTrack, startPaused: true)
|
|
logger.log("media scope carry-over: \(carriedTrack, privacy: .public) start_paused")
|
|
} catch {
|
|
logger.log("media scope transfer failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
player.pause()
|
|
}
|
|
startPolling()
|
|
}
|
|
|
|
/// Called when the conversation ends. If music was playing before the talk,
|
|
/// hand the room back to the player. The previous scope (room name) is kept
|
|
/// so the panel can resume polling it via `startDefaultPolling()`; a fresh
|
|
/// conversation re-scopes via `enterMediaScope`. Polling restarts here so
|
|
/// the resumed state stays visible without the panel being open.
|
|
func endConversation() async {
|
|
stopPolling()
|
|
pollGeneration += 1
|
|
if wasPlayingBeforeTalk {
|
|
wasPlayingBeforeTalk = false
|
|
guard let room else { return }
|
|
do {
|
|
try await api.control(room: room, action: "resume")
|
|
logger.log("resumed playback after talk ended")
|
|
} catch {
|
|
logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
startPolling()
|
|
}
|
|
|
|
/// Drop the room scope entirely (app-level teardown). After this, polling
|
|
/// targets the default room again.
|
|
func resetScope() {
|
|
stopPolling()
|
|
pollGeneration += 1
|
|
room = nil
|
|
wasPlayingBeforeTalk = false
|
|
}
|
|
|
|
private func startPolling() {
|
|
guard pollTask == nil else { return }
|
|
pollTask = Task { [weak self] in
|
|
while !Task.isCancelled {
|
|
await self?.pollState()
|
|
try? await Task.sleep(for: .milliseconds(1500))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func stopPolling() {
|
|
pollTask?.cancel()
|
|
pollTask = nil
|
|
}
|
|
|
|
/// Resume polling whenever the panel becomes visible and nothing is being
|
|
/// polled yet — whether that's the default room (no session) or a previous
|
|
/// conversation's room scope (kept after `endConversation`).
|
|
func startDefaultPolling() {
|
|
guard pollTask == nil else { return }
|
|
startPolling()
|
|
}
|
|
|
|
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
|
|
|
|
private func pollState() async {
|
|
let generation = pollGeneration
|
|
let scope = room ?? "default"
|
|
do {
|
|
let state = try await api.state(room: scope)
|
|
guard generation == pollGeneration else { return }
|
|
stateError = nil
|
|
reconcile(state)
|
|
} catch {
|
|
guard generation == pollGeneration else { return }
|
|
logger.log("poll failed: \(String(describing: error), privacy: .public)")
|
|
stateError = "Media server unreachable: \(error.localizedDescription)"
|
|
}
|
|
}
|
|
|
|
/// Reconciles the local player against a server state snapshot. Exposed
|
|
/// as internal (not private) so tests can drive deterministic state
|
|
/// transitions without a live server.
|
|
func reconcile(_ s: MediaState) {
|
|
choices = s.choices
|
|
pendingJobs = s.pending
|
|
// Keep the published desired state in sync with the server so the UI's
|
|
// play/pause icon and status text reflect reality (the server is the
|
|
// source of truth for per-room desired state).
|
|
desired = s.desired ?? "stopped"
|
|
|
|
let track = s.track
|
|
if let track, track.id != currentTrackID {
|
|
loadTrack(track)
|
|
} else if let track, let audioURL = track.audioURL, loadedAudioPath != audioURL {
|
|
// Track was loaded before it had a stream (still downloading) and
|
|
// now has one — wire it up so playback can start without a reload.
|
|
wireSource(audioURL)
|
|
}
|
|
if track == nil, currentTrackID != nil, s.desired == "stopped" {
|
|
clearMedia()
|
|
}
|
|
|
|
applyDesiredState(s, track: track)
|
|
|
|
if let hint = s.seekHint, hint.ts != appliedSeekTs, duration.isFinite, duration > 0 {
|
|
appliedSeekTs = hint.ts
|
|
let target = min(hint.position, duration)
|
|
if target.isFinite {
|
|
seek(to: max(0, target))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func applyDesiredState(_ s: MediaState, track: MediaTrack?) {
|
|
guard let desiredValue = s.desired else { return }
|
|
switch desiredValue {
|
|
case "playing":
|
|
// Suppress play during an active audio interruption (phone call,
|
|
// Siri). Without this guard the next 1.5s poll would restart audio
|
|
// immediately after .began paused it. The player.rate.isZero check
|
|
// makes this idempotent: once playing, subsequent polls are no-ops.
|
|
// This also handles the late-audio case where desired=playing
|
|
// arrives before audio_url is set; when the source is later wired
|
|
// via wireSource(), the next poll retries play because rate is
|
|
// still zero.
|
|
// Also suppress while resumeSuppressed is true (no-resume
|
|
// interruption ended, server not yet confirmed paused).
|
|
guard !isInterruptionActive, !resumeSuppressed else { return }
|
|
if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero {
|
|
player.play()
|
|
}
|
|
case "paused", "stopped":
|
|
if !player.rate.isZero {
|
|
player.pause()
|
|
}
|
|
// The server has confirmed a coherent paused/stopped state. If we
|
|
// were suppressing resume due to a no-resume interruption, the
|
|
// pause control succeeded (or the user paused independently), so
|
|
// it is safe to clear the suppression flag.
|
|
if resumeSuppressed {
|
|
resumeSuppressed = false
|
|
logger.log("resume suppression cleared: server confirmed \(desiredValue)")
|
|
}
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
|
|
private func loadTrack(_ track: MediaTrack) {
|
|
currentTrackID = track.id
|
|
self.track = track
|
|
if let audioURL = track.audioURL {
|
|
wireSource(audioURL)
|
|
} else {
|
|
// Downloading or failed: no stream yet. Reconcile wires the stream
|
|
// URL in via polling as soon as one is available.
|
|
clearCurrentAsset()
|
|
player.replaceCurrentItem(with: nil)
|
|
}
|
|
}
|
|
|
|
private func wireSource(_ path: String) {
|
|
guard let url = URL(string: path) else { return }
|
|
loadedAudioPath = path
|
|
let pinned = mediaLoader.rewrite(url)
|
|
// Remove the previous asset's loader mapping before wiring a new one so
|
|
// the loaderURLs dictionary stays bounded (Finding 11 / Item 7).
|
|
clearCurrentAsset()
|
|
let asset = mediaLoader.assetWithPinnedURL(pinned)
|
|
currentAsset = asset
|
|
logger.log("wiring source \(pinned.absoluteString, privacy: .public)")
|
|
let item = AVPlayerItem(asset: asset)
|
|
observeItem(item)
|
|
player.replaceCurrentItem(with: item)
|
|
}
|
|
|
|
/// Removes the loader mapping for the current asset (if any) and clears the
|
|
/// reference. Called before wiring a new source or clearing media so the
|
|
/// resource loader's internal dictionaries stay bounded.
|
|
private func clearCurrentAsset() {
|
|
if let asset = currentAsset {
|
|
mediaLoader.removeLoader(asset)
|
|
currentAsset = nil
|
|
}
|
|
}
|
|
|
|
// MARK: Player events
|
|
|
|
private func observePlayer() {
|
|
if timeObserver == nil {
|
|
timeObserver = player.addPeriodicTimeObserver(
|
|
forInterval: CMTime(seconds: 0.5, preferredTimescale: 600), queue: .main
|
|
) { [weak self] time in
|
|
Task { @MainActor [weak self] in
|
|
guard let self else { return }
|
|
self.currentTime = time.seconds
|
|
if let itemDuration = self.player.currentItem?.duration {
|
|
let seconds = itemDuration.seconds
|
|
if seconds.isFinite {
|
|
self.duration = seconds
|
|
}
|
|
}
|
|
if !self.player.rate.isZero, Date().timeIntervalSince(self.lastReportDate) >= 5 {
|
|
self.lastReportDate = Date()
|
|
self.report("playing")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Playback state observation
|
|
|
|
/// Observe the player's actual rate so every real play/pause transition is
|
|
/// reported promptly, mirroring the web UI's `play`/`pause` listeners. The
|
|
/// KVO block runs on an arbitrary queue; hop to MainActor before touching
|
|
/// any controller state.
|
|
private func observePlaybackState() {
|
|
rateObservation = player.observe(\.rate, options: [.initial, .new]) { [weak self] player, _ in
|
|
Task { @MainActor [weak self] in
|
|
guard let self else { return }
|
|
guard self.currentTrackID != nil else { return }
|
|
if player.rate > 0 {
|
|
// Actual playback began (user play, interruption resume, or
|
|
// reconcile-driven play). Report + publish the transition.
|
|
self.lastReportDate = Date()
|
|
self.report("playing")
|
|
self.publishMediaState("playing")
|
|
} else if player.currentItem != nil {
|
|
// Rate dropped to zero with an item still loaded: a pause
|
|
// that didn't originate from a user action (interruption,
|
|
// reconcile). User-initiated pauses already report before
|
|
// this fires; the lastReportDate guard makes duplicates no-ops.
|
|
if Date().timeIntervalSince(self.lastReportDate) >= 1 {
|
|
self.lastReportDate = Date()
|
|
self.report("paused")
|
|
self.publishMediaState("paused")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private func observeItem(_ item: AVPlayerItem?) {
|
|
NotificationCenter.default.removeObserver(
|
|
self, name: .AVPlayerItemDidPlayToEndTime, object: nil
|
|
)
|
|
guard let item else { return }
|
|
NotificationCenter.default.addObserver(
|
|
self, selector: #selector(playerItemDidEnd(_:)),
|
|
name: .AVPlayerItemDidPlayToEndTime, object: item
|
|
)
|
|
statusCancellable = item.publisher(for: \.status).sink { status in
|
|
switch status {
|
|
case .readyToPlay:
|
|
logger.log("item readyToPlay")
|
|
case .failed:
|
|
logger.log("item failed: \(String(describing: item.error), privacy: .public)")
|
|
default:
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
@objc private func playerItemDidEnd(_ notification: Notification) {
|
|
lastReportDate = Date()
|
|
report("stopped")
|
|
publishMediaState("stopped")
|
|
Task { [weak self] in
|
|
guard let self, let room = self.room else { return }
|
|
try? await self.api.control(room: room, action: "stop")
|
|
}
|
|
}
|
|
|
|
// MARK: Reporting + publishing
|
|
|
|
private func report(_ state: String, trackID: String? = nil) {
|
|
guard let room else { return }
|
|
let position = currentTime.isFinite ? currentTime : 0
|
|
var durationValue: Double?
|
|
if duration.isFinite, duration > 0 {
|
|
durationValue = duration
|
|
}
|
|
let resolvedTrackID = trackID ?? currentTrackID
|
|
Task { [weak self] in
|
|
guard let self else { return }
|
|
do {
|
|
try await self.api.report(room: room, state: state, trackID: resolvedTrackID, position: position, duration: durationValue)
|
|
} catch {
|
|
logger.log("report failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
}
|
|
|
|
private func publishMediaState(_ state: String, trackID: String? = nil) {
|
|
guard let session else { return }
|
|
// Capture the exact room and generation before spawning so a delayed
|
|
// publish from an old lifecycle cannot escape into a new conversation
|
|
// room (Finding 10).
|
|
let targetRoom = session.room
|
|
let generation = session.sessionGeneration
|
|
let pollGen = pollGeneration
|
|
let resolvedTrackID = trackID ?? currentTrackID
|
|
Task { [weak self] in
|
|
guard let self, self.pollGeneration == pollGen else { return }
|
|
guard session.sessionGeneration == generation, session.room === targetRoom else { return }
|
|
do {
|
|
try await session.controlChannel.sendMediaState(state, trackID: resolvedTrackID, room: targetRoom)
|
|
} catch {
|
|
logger.log("media_state publish failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: Play requests (search results, choices, library)
|
|
|
|
func playVideo(videoID: String?, title: String?, channel: String?) async {
|
|
let targetRoom = room ?? "default"
|
|
do {
|
|
try await api.play(room: targetRoom, videoID: videoID, title: title, channel: channel)
|
|
logger.log("play requested video_id=\(videoID ?? "nil", privacy: .public) room=\(targetRoom, privacy: .public)")
|
|
} catch {
|
|
logger.log("play failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
|
|
func playTrack(id: String) async {
|
|
let targetRoom = room ?? "default"
|
|
do {
|
|
try await api.play(room: targetRoom, trackID: id)
|
|
logger.log("play requested track_id=\(id, privacy: .public) room=\(targetRoom, privacy: .public)")
|
|
} catch {
|
|
logger.log("play failed: \(String(describing: error), privacy: .public)")
|
|
}
|
|
}
|
|
|
|
// MARK: Transport controls (parity with web UI play/pause/seek/stop)
|
|
|
|
func togglePlayPause() async {
|
|
guard currentTrackID != nil, loadedAudioPath != nil else { return }
|
|
if player.rate.isZero {
|
|
// Explicit user play: clear any resume suppression (the user is
|
|
// deliberately overriding the no-resume interruption state).
|
|
resumeSuppressed = false
|
|
// The rate observer reports + publishes the actual transition;
|
|
// the control call updates server desired state.
|
|
player.play()
|
|
if let room { try? await api.control(room: room, action: "resume") }
|
|
} else {
|
|
lastReportDate = Date()
|
|
report("paused")
|
|
publishMediaState("paused")
|
|
player.pause()
|
|
if let room { try? await api.control(room: room, action: "pause") }
|
|
}
|
|
}
|
|
|
|
func seek(by delta: Double) async {
|
|
guard loadedAudioPath != nil else { return }
|
|
let target = max(0, min(duration > 0 ? duration - 0.5 : .infinity, currentTime + delta))
|
|
if target.isFinite {
|
|
seek(to: target)
|
|
}
|
|
}
|
|
|
|
func seek(to position: Double) {
|
|
player.seek(to: CMTime(seconds: position, preferredTimescale: 600))
|
|
currentTime = position
|
|
report(player.rate.isZero ? "paused" : "playing")
|
|
}
|
|
|
|
func stop() async {
|
|
// Capture the track ID before clearing so the async report/publish
|
|
// tasks can read a non-nil value even after clearMedia() runs.
|
|
let trackID = currentTrackID
|
|
report("stopped", trackID: trackID)
|
|
clearMedia()
|
|
if let room { try? await api.control(room: room, action: "stop") }
|
|
publishMediaState("stopped", trackID: trackID)
|
|
}
|
|
|
|
private func clearMedia() {
|
|
currentTrackID = nil
|
|
track = nil
|
|
loadedAudioPath = nil
|
|
// Remove the loader mapping for the asset being cleared so the
|
|
// resource loader's internal dictionaries stay bounded (Item 7).
|
|
clearCurrentAsset()
|
|
observeItem(nil)
|
|
player.replaceCurrentItem(with: nil)
|
|
currentTime = 0
|
|
duration = 0
|
|
stateError = nil
|
|
}
|
|
|
|
// MARK: Volume
|
|
|
|
func applyVolume() {
|
|
player.volume = Float(settings.volume)
|
|
}
|
|
|
|
func setVolume(_ volume: Double) {
|
|
settings.volume = volume
|
|
player.volume = Float(volume)
|
|
}
|
|
}
|