MTAudioProcessingTap - prepare{} and process{} callbacks are not executing

Question

I am working on loading a (local) movie into AVPlayer and applying processing to the audio track with an audioTapProcessor. So far I've found great GitHub examples here, here, and here. I'm using the "tap cookie" approach used in the last link and in an answer to this previous question.

Audio & video playback are working fine. However, my tapPrepare and tapProcess callbacks are not being called, but Init and Finalize are. So I'm doing something both right and wrong. relevant code attached -- Any help appreciated!

import Foundation
import AVFoundation
import AudioToolbox
import MediaToolbox
import CoreAudioTypes

class PlayerViewController: UIViewController {
    
    class TapCookie {
        weak var content: PlayerViewController?
        deinit {
            print("TapCookie deinit")   // appears after tapFinalize
        }
    }
    // MARK: Properties
    var playerAsset: AVURLAsset?
    var playerItem: AVPlayerItem! = nil
    var audioProcessingFormat: AudioStreamBasicDescription?
    private var tracksObserver: NSKeyValueObservation?
    
    // MARK: Button to trigger actions
    @IBAction func selectVideo(_ sender: Any) {
        // starts doing stuff:
        // - select a video file from device, extract movieURL string ...
        playerAsset = AVURLAsset(url: movieURL)
        playerItem = AVPlayerItem(url: movieURL)
        //... then send asset to AVPlayer (not shown)
        
        // set up audioProcessingTap
        tracksObserver = playerItem.observe(\AVPlayerItem.tracks, options: [.initial, .new]) {
            [unowned self] item, change in
            installTap(playerItem: playerItem)
        }
    }
    func installTap(playerItem: AVPlayerItem) {
        let cookie = TapCookie()
        cookie.content = self
                
        var callbacks = MTAudioProcessingTapCallbacks(
            version: kMTAudioProcessingTapCallbacksVersion_0,
            clientInfo: UnsafeMutableRawPointer(Unmanaged.passRetained(cookie).toOpaque()),
            init: tapInit,
            finalize: tapFinalize,
            prepare: tapPrepare,
            unprepare: tapUnprepare,
            process: tapProcess)
        
        var tap: Unmanaged<MTAudioProcessingTap>?
        let err = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap)
        assert(noErr == err);
        // tapInit successfully called after MTAudioProcessingTapCreate
                
        let audioMix = AVMutableAudioMix()
        let audioTrack = playerItem.asset.tracks(withMediaType: AVMediaType.audio).first! //use first audio track
        let inputParams = AVMutableAudioMixInputParameters(track: audioTrack)
        
        inputParams.audioTapProcessor = tap?.takeRetainedValue()
        audioMix.inputParameters = [inputParams]
        playerItem.audioMix = audioMix
    }
    // MARK: install tap callbacks

    let tapInit: MTAudioProcessingTapInitCallback = {
        (tap, clientInfo, tapStorageOut) in
        
        tapStorageOut.pointee = clientInfo
        print("tapInit tap: \(tap)\n clientInfo: \(String(describing: clientInfo))\n tapStorageOut: \(tapStorageOut)\n")
    }
    // tapPrepare not called !!
    let tapPrepare: MTAudioProcessingTapPrepareCallback = {
        (tap, maxFrames, processingFormat) in
        print("tapPrepare tap: \(tap), maxFrames: \(maxFrames)\n processingFormat: \(processingFormat)")
        
        let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
        
        cookie.content!.audioProcessingFormat = AudioStreamBasicDescription(mSampleRate: processingFormat.pointee.mSampleRate,
                                                                mFormatID: processingFormat.pointee.mFormatID,
                                                                   mFormatFlags: processingFormat.pointee.mFormatFlags,
                                                                   mBytesPerPacket: processingFormat.pointee.mBytesPerPacket,
                                                                   mFramesPerPacket: processingFormat.pointee.mFramesPerPacket,
                                                                   mBytesPerFrame: processingFormat.pointee.mBytesPerFrame,
                                                                   mChannelsPerFrame: processingFormat.pointee.mChannelsPerFrame,
                                                                   mBitsPerChannel: processingFormat.pointee.mBitsPerChannel,
                                                                   mReserved: processingFormat.pointee.mReserved)
    }
    
    let tapUnprepare: MTAudioProcessingTapUnprepareCallback = {
        (tap) in
        print("tapUnprepare \(tap)")

    }
    
    // tapProcess not called !!
    let tapProcess: MTAudioProcessingTapProcessCallback = {
        (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in
        print("tapProcess \(tap)\n \(numberFrames)\n \(flags)\n \(bufferListInOut)\n \(numberFramesOut)\n \(flagsOut)\n")
        
        let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut)
        if noErr != status {
            print("get audio: \(status)")
        }
        
        let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
        guard let cookieContent = cookie.content else {
            print("Tap callback: cookie content was deallocated!")
            return
        }
            // process audio here...
    }
    
    let tapFinalize: MTAudioProcessingTapFinalizeCallback = {
        (tap) in
        print("tapFinalize \(tap)")
        // release cookie
        Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).release()
    }
}

Answer 1

You need to create an AVPlayer

player = AVPlayer(playerItem: playerItem)

and then at some point start it playing:

player.play()

Then the prepare and process callbacks will be called.