FFMpeg (libav) hls demux with mediacodec surface rendering is playing too fast

Question

I am demuxing a m3u8 with libav and getting packets to decode them + play, the decoder is mediacodec with a surface set for direct rendering. This works so far but the video plays too fast, extremely fast; tried to play with pts/dts I sent to decoder without succes at the minute.

I saw other implementations that are doing this but they aren't using direct rendering with mediacodec then they're maintaing like a queue of already decoded images to paint. As my implementation is intended to run on a android without too much resources I would need to save as much processing as I can then direct rendering is a must.

Here I put a sample code of what I did:

int MediaParserFFmpeg::init(const std::string &filePath)
{
    m_formatCtx = avformat_alloc_context();
    int ret = 0;
    
    ret = avformat_open_input(&m_formatCtx, m_mediaPath.c_str(), iFormat, nullptr);
    if (ret < 0)
    {
        return MEDIA_ERROR_INIT_FAILED;
    }

    ret = avformat_find_stream_info(m_formatCtx, nullptr);
    if (ret < 0)
    {
        return MEDIA_ERROR_INIT_FAILED;
    }

    m_duration = m_formatCtx->duration;
    const AVCodec *vcodec = nullptr;
    m_vStreamIdx = av_find_best_stream(m_formatCtx,
                                        AVMEDIA_TYPE_VIDEO,
                                        -1,
                                        -1,
                                        &vcodec,
                                        0);


    AVCodecParameters *codecParam = m_formatCtx->streams[m_vStreamIdx]->codecpar;
    m_width = codecParam->width;
    m_height = codecParam->height;
    m_vCodec = codecParam->codec_id;
    AVRational fpsRatio = m_formatCtx->streams[m_vStreamIdx]->avg_frame_rate;
    m_frameRate = fpsRatio.num / (float)fpsRatio.den;

    if (codecParam->codec_id == AV_CODEC_ID_H264)
    {
        // if stored in ANNEXB type convert to NALU
        m_isNal = codecParam->extradata_size > 0 && *(codecParam->extradata) == 1;
        if (m_isNal)
        {
            const AVBitStreamFilter *bsfFilter = av_bsf_get_by_name("h264_mp4toannexb");
            if (!bsfFilter)
            {
                return MEDIA_ERROR_INIT_FAILED;
            }
            av_bsf_alloc(bsfFilter, &m_bsfcContext);
            m_bsfcContext->par_in = codecParam;
            av_bsf_init(m_bsfcContext);

            m_bsfPacket = av_packet_alloc();
        }
    }

    m_packet = av_packet_alloc();
}

int MediaParserFFmpeg::getNextFrame(MediaType *mediaType, AVEncodedFrame **outFrame)
{
    int ret = 0;
    ret = av_read_frame(m_formatCtx, m_packet);

    if (ret < 0 && ret != AVERROR_EOF)
    {
        av_packet_unref(m_packet);
        return MEDIA_ERROR_PARSER_PARSE_FAILED;
    }

    if (ret == AVERROR_EOF)
    {
        av_packet_unref(m_packet);
        return MEDIA_INFO_PARSER_END_OF_STREAM;
    }

    if (m_packet->data && m_packet->size)
    {
        if (m_vCodec == VIDEOCODEC_H264 && m_isNal)
        {
            int res = av_bsf_send_packet(m_bsfcContext, m_packet);
            if (res != 0)
            {
                MEDIA_LOG() << "ZError:" << "Error adding NAL to H264 stream";
            }

            if (res == 0)
            {
                *outFrame = new EncodedVideoFrame(m_bsfPacket->data,
                                                    m_bsfPacket->size,
                                                    getTimeStamp(m_bsfPacket, m_formatCtx->streams[m_vStreamIdx]),
                                                    m_width,
                                                    m_height,
                                                    true);
                *mediaType = MEDIA_TYPE_VIDEO;
            }
            av_packet_unref(m_bsfPacket);

            res = av_bsf_receive_packet(m_bsfcContext, m_bsfPacket);
        }
    }
    av_packet_unref(m_packet);
    return MEDIA_SUCCESS;
}

int64_t MediaParserFFmpeg::getTimeStamp(AVPacket *packet, AVStream *stream)
{
    if (packet->pts != (qint64)AV_NOPTS_VALUE)
    {
        return (1000000 * (packet->pts * ((float)stream->time_base.num / stream->time_base.den)));
    }
    else if (packet->dts != (qint64)AV_NOPTS_VALUE)
    {
        return (1000000 * (packet->dts * ((float)stream->time_base.num / stream->time_base.den)));
    }
    else
    {
        return 0;
    }
}

Then I have a thread that's calling getNextFrame() getting a new EncodedVideoFrame and pushing it to MediaCodec decoder in this way:

AMediaCodec_queueInputBuffer(m_mediaCodec, index, 0, encodedFrame->m_frameSize, encodedFrame->m_timeStamp, 0);

As I said, it works but produces a video that got played extremely fast; already tried to play with pts and dts without luck ( not sure if I was doing something wrong there anyway ).

Any thoughs on how to solve this ?

Answer 1

Thanks to this guy + a guy in my office (kudos my friends) I got the solution, as they pointed, you need to create a wait based on system clock or based in audio clock to wait the duration of each frame; as otherwise it would play as faster as he is able.

Here you go the code to solve this, based on system clock.

void VideoDecoderMediaCodec::preRender(const int64_t presentationTimeUsec)
{

    qDebug() <<"Oh Oh, preRender() pts:"<<presentationTimeUsec<<" mPrevMonoUsec:"<<m_prevClockUsec<<" mPrevPresentUsec:"<<m_prevPtsUsec;
    int64_t OneMillion = 1000000;

    if (m_prevClockUsec == 0)
    {
        // Latch current values, then return immediately.
        m_prevClockUsec = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
        m_prevPtsUsec = presentationTimeUsec;
    }
    else
    {
        int64_t frameDelta = 0;
        frameDelta = presentationTimeUsec - m_prevPtsUsec;
        if(frameDelta <0)
        {
            frameDelta = 0;
        }
        else if(frameDelta > 10 * OneMillion)
        {
            // 5 secs cap.
            frameDelta = 5 * OneMillion;
        }

        qDebug() <<"Oh Oh, preRender() frameDelta:"<<frameDelta;

        int64_t desiredUsec = m_prevClockUsec + frameDelta;
        int64_t nowUsec = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();

        qDebug() <<"Oh Oh, preRender() desiredUsec:"<<desiredUsec<<" nowUsec:"<<nowUsec;
        while (nowUsec < (desiredUsec - 100) /*&& mState == RUNNING*/) {
            long sleepTimeUsec = desiredUsec - nowUsec;
            if (sleepTimeUsec > 500000) {
                sleepTimeUsec = 500000;
            }

            qDebug() <<"Oh Oh, preRender() sleepTimeUsec:"<<sleepTimeUsec;
            std::this_thread::sleep_for(std::chrono::microseconds(sleepTimeUsec));
            nowUsec = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
        }

        m_prevClockUsec += frameDelta;
        m_prevPtsUsec += frameDelta;
    }
}

An that must be called before calling AMediaCodec_releaseOutputBuffer

...
AMediaCodecBufferInfo info;
AMediaCodec_dequeueOutputBuffer(m_mediaCodec, &info, BUFFER_TIMEOUT_US);
...
preRender(info.presentationTimeUs);
AMediaCodec_releaseOutputBuffer(m_mediaCodec, status, m_isDirectRender);
...