This repository has been archived on 2026-10-04. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
hope-voice-ios/Sources/Media/MediaPlayerController.swift
T
snowspeeder 0af944e60d feat: media player with reconciliation and pinned media proxy (Phase 5)
- MediaAPI client for all 10 media endpoints + Codable models
- PinnedMediaResourceLoader: AVAssetResourceLoaderDelegate proxying
  https media through the pinned-TLS session (AVPlayer ignores registered
  URLProtocol subclasses and can't trust the self-signed cert)
- MediaPlayerController: AVPlayer + 1.5s poll reconciliation ported from
  web app.js (track swap, late audio_url, desired state, seek_hint,
  reporting, enter-media-scope carry-over, post-talk resume)
- MediaPanelView UI + music.note toolbar entry
- VoiceSession wiring (enterMediaScope/endConversation)

Playback of the library's webm tracks is blocked by an iOS platform
limitation (AVPlayer cannot decode webm); TLS/Range proxy, reconciliation,
polling, reporting and UI are all verified working in the simulator.
2026-08-30 08:21:08 -04:00

327 lines
11 KiB
Swift

import AVFoundation
import Combine
import Foundation
import os
private let logger = Logger(subsystem: "dev.shane.voice", category: "media")
/// AVPlayer-backed media player. The media server holds per-room desired state;
/// this controller polls /state and reconciles the local player against it,
/// porting the web UI's reconciliation rules (app.js reconcileMedia).
@MainActor
final class MediaPlayerController: ObservableObject {
@Published private(set) var track: MediaTrack?
@Published private(set) var desired: String = "stopped"
@Published private(set) var currentTime: Double = 0
@Published private(set) var duration: Double = 0
@Published private(set) var pendingJobs: [MediaPendingJob] = []
@Published private(set) var choices: [MediaChoice] = []
let player = AVPlayer()
private let api: MediaAPI
private let settings: SettingsStore
private let mediaLoader: PinnedMediaResourceLoader
private weak var session: VoiceSession?
private var room: String?
private var currentTrackID: String?
private var loadedAudioPath: String?
private var appliedSeekTs: Double = 0
private var wasPlayingBeforeTalk = false
private var lastReportDate = Date.distantPast
private var pollTask: Task<Void, Never>?
private var timeObserver: Any?
private var statusCancellable: AnyCancellable?
init(api: MediaAPI, settings: SettingsStore) {
self.api = api
self.settings = settings
self.mediaLoader = PinnedMediaResourceLoader(base: api.serverBase)
player.volume = Float(settings.volume)
NotificationCenter.default.addObserver(
self, selector: #selector(handleInterruption),
name: AVAudioSession.interruptionNotification, object: nil
)
}
deinit {
if let timeObserver {
player.removeTimeObserver(timeObserver)
}
NotificationCenter.default.removeObserver(self)
}
// MARK: Lifecycle
func attach(session: VoiceSession) {
self.session = session
observePlayer()
applyVolume()
}
/// Called just before joining the LiveKit room. Carries whatever is loaded
/// into the new room-scoped player state, paused (talking pauses music).
func enterMediaScope(roomName: String) async {
let wasPlaying = currentTrackID != nil && !player.rate.isZero
wasPlayingBeforeTalk = wasPlaying
let carriedTrack = currentTrackID
room = roomName
if let carriedTrack {
do {
try await api.play(room: roomName, trackID: carriedTrack, startPaused: true)
logger.log("media scope carry-over: \(carriedTrack, privacy: .public) start_paused")
} catch {
logger.log("media scope transfer failed: \(String(describing: error), privacy: .public)")
}
player.pause()
}
startPolling()
}
/// Called when the conversation ends. If music was playing before the talk,
/// hand the room back to the player.
func endConversation() async {
stopPolling()
if wasPlayingBeforeTalk {
wasPlayingBeforeTalk = false
guard let room else { return }
do {
try await api.control(room: room, action: "resume")
logger.log("resumed playback after talk ended")
} catch {
logger.log("post-talk resume failed (poll will retry): \(String(describing: error), privacy: .public)")
}
}
}
private func startPolling() {
guard pollTask == nil else { return }
pollTask = Task { [weak self] in
while !Task.isCancelled {
await self?.pollState()
try? await Task.sleep(for: .milliseconds(1500))
}
}
}
private func stopPolling() {
pollTask?.cancel()
pollTask = nil
}
// MARK: Poll + reconcile (ported from app.js pollMediaState/reconcileMedia)
private func pollState() async {
let scope = room ?? "default"
guard let state = try? await api.state(room: scope) else { return }
reconcile(state)
}
private func reconcile(_ s: MediaState) {
choices = s.choices
pendingJobs = s.pending
let track = s.track
if let track, track.id != currentTrackID {
loadTrack(track)
} else if let track, let audioURL = track.audioURL, loadedAudioPath != audioURL {
// Track was loaded before it had a stream (still downloading) and
// now has one — wire it up so playback can start without a reload.
wireSource(audioURL)
}
if track == nil, currentTrackID != nil, s.desired == "stopped" {
clearMedia()
}
applyDesiredState(s, track: track)
if let hint = s.seekHint, hint.ts != appliedSeekTs, duration.isFinite, duration > 0 {
appliedSeekTs = hint.ts
let target = min(hint.position, duration)
if target.isFinite {
seek(to: max(0, target))
}
}
}
private func applyDesiredState(_ s: MediaState, track: MediaTrack?) {
if s.desired == "playing" {
// audio_url presence means a playable stream exists — including a
// still-growing download, which plays progressively.
if let track, track.audioURL != nil, loadedAudioPath != nil, player.rate.isZero {
player.play()
}
} else if !player.rate.isZero {
player.pause()
}
}
private func loadTrack(_ track: MediaTrack) {
currentTrackID = track.id
self.track = track
if let audioURL = track.audioURL {
wireSource(audioURL)
} else {
// Downloading or failed: no stream yet. Reconcile wires the stream
// URL in via polling as soon as one is available.
player.replaceCurrentItem(with: nil)
}
}
private func wireSource(_ path: String) {
guard let url = URL(string: path) else { return }
loadedAudioPath = path
let pinned = mediaLoader.rewrite(url)
let asset = mediaLoader.assetWithPinnedURL(pinned)
logger.log("wiring source \(pinned.absoluteString, privacy: .public)")
let item = AVPlayerItem(asset: asset)
observeItem(item)
player.replaceCurrentItem(with: item)
}
// MARK: Player events
private func observePlayer() {
if timeObserver == nil {
timeObserver = player.addPeriodicTimeObserver(
forInterval: CMTime(seconds: 0.5, preferredTimescale: 600), queue: .main
) { [weak self] time in
Task { @MainActor [weak self] in
guard let self else { return }
self.currentTime = time.seconds
if let itemDuration = self.player.currentItem?.duration {
let seconds = itemDuration.seconds
if seconds.isFinite {
self.duration = seconds
}
}
if !self.player.rate.isZero, Date().timeIntervalSince(self.lastReportDate) >= 5 {
self.lastReportDate = Date()
self.report("playing")
}
}
}
}
}
private func observeItem(_ item: AVPlayerItem?) {
NotificationCenter.default.removeObserver(
self, name: .AVPlayerItemDidPlayToEndTime, object: nil
)
guard let item else { return }
NotificationCenter.default.addObserver(
self, selector: #selector(playerItemDidEnd(_:)),
name: .AVPlayerItemDidPlayToEndTime, object: item
)
statusCancellable = item.publisher(for: \.status).sink { status in
switch status {
case .readyToPlay:
logger.log("item readyToPlay")
case .failed:
logger.log("item failed: \(String(describing: item.error), privacy: .public)")
default:
break
}
}
}
@objc private func playerItemDidEnd(_ notification: Notification) {
report("stopped")
publishMediaState("stopped")
Task { [weak self] in
guard let self, let room = self.room else { return }
try? await self.api.control(room: room, action: "stop")
}
}
@objc private func handleInterruption(_ notification: Notification) {
applyVolume()
}
// MARK: Reporting + publishing
private func report(_ state: String) {
guard let room else { return }
let position = currentTime.isFinite ? currentTime : 0
var durationValue: Double?
if duration.isFinite, duration > 0 {
durationValue = duration
}
Task { [weak self] in
guard let self else { return }
do {
try await self.api.report(room: room, state: state, trackID: self.currentTrackID, position: position, duration: durationValue)
} catch {
logger.log("report failed: \(String(describing: error), privacy: .public)")
}
}
}
private func publishMediaState(_ state: String) {
guard let session else { return }
Task { [weak self] in
guard let self else { return }
do {
try await session.controlChannel.sendMediaState(state, trackID: self.currentTrackID, room: session.room)
} catch {
logger.log("media_state publish failed: \(String(describing: error), privacy: .public)")
}
}
}
// MARK: Transport controls (parity with web UI play/pause/seek/stop)
func togglePlayPause() async {
guard currentTrackID != nil, loadedAudioPath != nil else { return }
if player.rate.isZero {
player.play()
if let room { try? await api.control(room: room, action: "resume") }
} else {
player.pause()
if let room { try? await api.control(room: room, action: "pause") }
}
}
func seek(by delta: Double) async {
guard loadedAudioPath != nil else { return }
let target = max(0, min(duration > 0 ? duration - 0.5 : .infinity, currentTime + delta))
if target.isFinite {
seek(to: target)
}
report(player.rate.isZero ? "paused" : "playing")
}
func seek(to position: Double) {
player.seek(to: CMTime(seconds: position, preferredTimescale: 600))
currentTime = position
}
func stop() async {
clearMedia()
if let room { try? await api.control(room: room, action: "stop") }
publishMediaState("stopped")
}
private func clearMedia() {
currentTrackID = nil
track = nil
loadedAudioPath = nil
observeItem(nil)
player.replaceCurrentItem(with: nil)
currentTime = 0
duration = 0
}
// MARK: Volume
func applyVolume() {
player.volume = Float(settings.volume)
}
func setVolume(_ volume: Double) {
settings.volume = volume
player.volume = Float(volume)
}
}