Use ffmpeg muxer to MPEG2TS , audio stream can't play in vlc palayer

Question

I use libavformat to encapsulate an h264 stream and an aac stream into an mp4 file which is playable. However, when encapsulated into a ts file, it works fine in the Win10 player, but no audio in the vlc player. When encapsulating, the audio stream is printed, but with fprobe, the audio stream is printed with channel=0. What could be the reason for this? And h264 source file is no pts.So I caculate it by myself.

ffprobe print

ffmpeg print

Here is my code.

#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>


static void log_packet(const AVFormatContext* fmt_ctx, const AVPacket* pkt, const char* tag)
{
    AVRational* time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
    printf("%s num=%d  den=%d\n", tag, time_base->num, time_base->den);
    printf("%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
        tag,
        av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
        av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
        av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
        pkt->stream_index);
}

int main()
{
    const char* in_filename_v = "test.h264";
    const char* in_filename_a = "aoutput.aac";
    const char* out_filename = "lol.ts";


    //Video Input AVFormatContext
    AVFormatContext* ifmt_ctx_v = NULL;
    int ret = avformat_open_input(&ifmt_ctx_v, in_filename_v, 0, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not open input_v %s", in_filename_v);
        return -1;
    }

    //Find Video Stream Info
    ret = avformat_find_stream_info(ifmt_ctx_v, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not find input_v stream info");
        return -1;
    }

    //Audio Input AVFormatContext
    AVFormatContext* ifmt_ctx_a = NULL;
    ret = avformat_open_input(&ifmt_ctx_a, in_filename_a, 0, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not open input_a %s", in_filename_a);
        return -1;
    }

    //Find Audio Stream Info
    ret = avformat_find_stream_info(ifmt_ctx_a, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not find input_a stream info");
        return -1;
    }


    printf("===========Input Information==========\n");
    av_dump_format(ifmt_ctx_v, 0, in_filename_v, 0);
    av_dump_format(ifmt_ctx_a, 0, in_filename_a, 0);
    printf("======================================\n");


    //Output AVFormatContext
    AVFormatContext* ofmt_ctx = NULL;
    avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
    if (!ofmt_ctx)
    {
        fprintf(stderr, "cannot alloc OutputFromat context!");
        ret = AVERROR_UNKNOWN;
        return -1;
    }
    AVOutputFormat* ofmt = ofmt_ctx->oformat;

    //Alloc AVSTREAM
    int istream_index_v = 0, istream_index_a = 0, ostream_index_v = 0, ostream_index_a = 0;
    for (int i = 0; i < ifmt_ctx_v->nb_streams; i++)
    {
        AVStream* outstream;
        AVStream* in_stream = ifmt_ctx_v->streams[i];
        AVCodecParameters* in_codecpar = in_stream->codecpar;
        if (in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
            continue;

        outstream = avformat_new_stream(ofmt_ctx, NULL);
        if (!outstream)
        {
            fprintf(stderr, "Failed allocating output stream\n");
            return -1;
        }

        ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
        if (ret < 0)
        {
            fprintf(stderr, "Failed to copy codec parameters\n");
            return -1;
        }
        outstream->codecpar->codec_tag = 0;
        // Remeber video stream id
        istream_index_v = i;
        ostream_index_v = 0;
        break;
    }

    for (int i = 0; i < ifmt_ctx_a->nb_streams; i++)
    {
        AVStream* outstream;
        AVStream* in_stream = ifmt_ctx_a->streams[i];
        AVCodecParameters* in_codecpar = in_stream->codecpar;
        if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO)
            continue;

        outstream = avformat_new_stream(ofmt_ctx, NULL);
        if (!outstream)
        {
            fprintf(stderr, "Failed allocating output stream\n");
            return -1;
        }

        ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
        if (ret < 0)
        {
            fprintf(stderr, "Failed to copy codec parameters\n");
            return -1;
        }
        outstream->codecpar->codec_tag = 0;
        // Remeber audio stream id
        istream_index_a = i;
        ostream_index_a = 1;
        break;
    }

    printf("===========Output Information==========\n");
    av_dump_format(ofmt_ctx, 0, out_filename, 1);
    printf("======================================\n");

    if (!(ofmt->flags & AVFMT_NOFILE))
    {
        ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
        if (ret < 0)
        {
            fprintf(stderr, "Could not open output file '%s'", out_filename);
            return -1;
        }
    }

    //Write file header
    ret = avformat_write_header(ofmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Error occurred when opening output file\n");
        return -1;
    }

    //read and write packet
    AVPacket* pkt = av_packet_alloc();
    if (!pkt)
    {
        fprintf(stderr, "Could not allocate AVPacket\n");
        return -1;
    }
    while (1)
    {
        AVStream* in_stream, * outstream;
        ret = av_read_frame(ifmt_ctx_v, pkt);
        if (ret < 0)
            break;
        in_stream = ifmt_ctx_v->streams[pkt->stream_index];
        if (pkt->stream_index != istream_index_v)
        {
            av_packet_unref(pkt);
            continue;
        }


        pkt->stream_index = ostream_index_v;
        outstream = ofmt_ctx->streams[pkt->stream_index];
        // in log info
        log_packet(ifmt_ctx_v, pkt, "in");

        if (pkt->pts == AV_NOPTS_VALUE)
        {
            AVRational time_base1 = in_stream->time_base;
            // 
            int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
            static int frame_index = 0;
            pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
            pkt->dts = pkt->pts;
            pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
            frame_index++;
        }


        // duration between two frames(us)
        av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
        pkt->pos = -1;
        // out log info
        log_packet(ofmt_ctx, pkt, "out");

        ret = av_interleaved_write_frame(ofmt_ctx, pkt);
        if (ret < 0)
        {
            fprintf(stderr, "Error muxing packet\n");
            break;
        }
    }

    while (1)
    {
        AVStream* in_stream, * outstream;
        ret = av_read_frame(ifmt_ctx_a, pkt);
        if (ret < 0)
            break;
        in_stream = ifmt_ctx_a->streams[pkt->stream_index];
        if (pkt->stream_index != istream_index_a)
        {
            av_packet_unref(pkt);
            continue;
        }

        if (pkt->pts == AV_NOPTS_VALUE)
        {
            AVRational time_base1 = in_stream->time_base;
            // duration between two frames(us)
            int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
            static int frame_index = 0;
            pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
            pkt->dts = pkt->pts;
            pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
            frame_index++;
        }

        // in log info
        log_packet(ifmt_ctx_a, pkt, "in");

        pkt->stream_index = ostream_index_a;
        outstream = ofmt_ctx->streams[pkt->stream_index];


        //change timestamp
        av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
        pkt->pos = -1;
        // out log info
        log_packet(ofmt_ctx, pkt, "out");

        ret = av_interleaved_write_frame(ofmt_ctx, pkt);
        if (ret < 0)
        {
            fprintf(stderr, "Error muxing packet\n");
            break;
        }
    }

    //write file trailer
    av_write_trailer(ofmt_ctx);


    printf("===========Output Information==========\n");
    av_dump_format(ofmt_ctx, 0, out_filename, 1);
    printf("======================================\n");

}

DVB Insepctor video DVB Insepctor audio

Answer 1

Thanks for @aergistal.

• The Reason is that av_interleaved_write_frame has buffer limit.I hadn't thought about this before so I write all video packages firstly and then write all audio packages.In ts files, at front are lots of video packages, followed by lots of audio packages, and finally both packages interleaved.
• Because of MPEG-TS is a stream consists of packages , so in a long time the player can't find audio packages resulting no voice.

Here is my new code that can work.

#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#define EXTRAL 1

static void log_packet(const AVFormatContext* fmt_ctx, const AVPacket* pkt, const char* tag)
{
    AVRational* time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;

    FILE* fp = fopen("fflog.log", "a+");
    char buf[200];
    sprintf(buf, "%s num=%d  den=%d\n", tag, time_base->num, time_base->den);
    for (int i = 0; *(buf + i) != '\0'; i++)
    {
        fwrite(buf + i, 1, 1, fp);
    }
    sprintf(buf, "%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
        tag,
        av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
        av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
        av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
        pkt->stream_index);
    for (int i = 0; *(buf + i) != '\0'; i++)
    {
        fwrite(buf + i, 1, 1, fp);
    }
    fclose(fp);
}

int main()
{
    const char* in_filename_v = "test.h264";
    const char* in_filename_a = "aoutput.aac";
    const char* out_filename = "lol.ts";


    AVFormatContext* ifmt_ctx_v = NULL;
    int ret = avformat_open_input(&ifmt_ctx_v, in_filename_v, 0, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not open input_v %s", in_filename_v);
        return -1;
    }

    ret = avformat_find_stream_info(ifmt_ctx_v, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not find input_v stream info");
        return -1;
    }

    AVFormatContext* ifmt_ctx_a = NULL;
    ret = avformat_open_input(&ifmt_ctx_a, in_filename_a, 0, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not open input_a %s", in_filename_a);
        return -1;
    }

    ret = avformat_find_stream_info(ifmt_ctx_a, 0);
    if (ret < 0)
    {
        fprintf(stderr, "Could not find input_a stream info");
        return -1;
    }

#if EXTRAL
    printf("===========Input Information==========\n");
    av_dump_format(ifmt_ctx_v, 0, in_filename_v, 0);
    av_dump_format(ifmt_ctx_a, 0, in_filename_a, 0);
    printf("======================================\n");
#endif

    AVFormatContext* ofmt_ctx = NULL;
    avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
    if (!ofmt_ctx)
    {
        fprintf(stderr, "cannot alloc OutputFromat context!");
        ret = AVERROR_UNKNOWN;
        return -1;
    }
    AVOutputFormat* ofmt = ofmt_ctx->oformat;

    int istream_index_v = 0, istream_index_a = 0, ostream_index_v = 0, ostream_index_a = 0;
    for (int i = 0; i < ifmt_ctx_v->nb_streams; i++)
    {
        AVStream* outstream;
        AVStream* in_stream = ifmt_ctx_v->streams[i];
        AVCodecParameters* in_codecpar = in_stream->codecpar;
        if (in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
            continue;

        outstream = avformat_new_stream(ofmt_ctx, NULL);
        if (!outstream)
        {
            fprintf(stderr, "Failed allocating output stream\n");
            return -1;
        }

        ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
        if (ret < 0)
        {
            fprintf(stderr, "Failed to copy codec parameters\n");
            return -1;
        }
        outstream->codecpar->codec_tag = 0;
        istream_index_v = i;
        ostream_index_v = 0;
        break;
    }

    for (int i = 0; i < ifmt_ctx_a->nb_streams; i++)
    {
        AVStream* outstream;
        AVStream* in_stream = ifmt_ctx_a->streams[i];
        AVCodecParameters* in_codecpar = in_stream->codecpar;
        if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO)
            continue;
        outstream = avformat_new_stream(ofmt_ctx, NULL);
        if (!outstream)
        {
            fprintf(stderr, "Failed allocating output stream\n");
            return -1;
        }

        ret = avcodec_parameters_copy(outstream->codecpar, in_codecpar);
        if (ret < 0)
        {
            fprintf(stderr, "Failed to copy codec parameters\n");
            return -1;
        }
        outstream->codecpar->codec_tag = 0;
        istream_index_a = i;
        ostream_index_a = 1;
        break;
    }

#if EXTRAL
    printf("===========Output Information==========\n");
    av_dump_format(ofmt_ctx, 0, out_filename, 1);
    printf("======================================\n");
#endif
    if (!(ofmt->flags & AVFMT_NOFILE))
    {
        ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
        if (ret < 0)
        {
            fprintf(stderr, "Could not open output file '%s'", out_filename);
            return -1;
        }
    }

    ret = avformat_write_header(ofmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Error occurred when opening output file\n");
        return -1;
    }

    AVPacket* pkt = av_packet_alloc();
    if (!pkt)
    {
        fprintf(stderr, "Could not allocate AVPacket\n");
        return -1;
    }
    int64_t pts_v = 0, pts_a = 0;
    while (1)
    {
        if (av_compare_ts(pts_a, ifmt_ctx_a->streams[istream_index_a]->time_base, pts_v, ifmt_ctx_v->streams[istream_index_v]->time_base) <= 0)
        {

            AVStream* in_stream, * outstream;
            ret = av_read_frame(ifmt_ctx_a, pkt);
            if (ret < 0)
                break;
            in_stream = ifmt_ctx_a->streams[pkt->stream_index];
            if (pkt->stream_index != istream_index_a)
            {
                av_packet_unref(pkt);
                continue;
            }

            if (pkt->pts == AV_NOPTS_VALUE)
            {
                AVRational time_base1 = in_stream->time_base;
                int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
                static int frame_index = 0;
                pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
                pkt->dts = pkt->pts;
                pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
                frame_index++;
            }
            pts_a = pkt->pts;
            // in log info
            log_packet(ifmt_ctx_a, pkt, "in audio");

            pkt->stream_index = ostream_index_a;
            outstream = ofmt_ctx->streams[pkt->stream_index];


            av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
            pkt->pos = -1;
            // out log info
            log_packet(ofmt_ctx, pkt, "out audio");

            ret = av_interleaved_write_frame(ofmt_ctx, pkt);
            if (ret < 0)
            {
                fprintf(stderr, "Error muxing packet\n");
                return -1;
            }
        }
        else
        {

            AVStream* in_stream, * outstream;
            ret = av_read_frame(ifmt_ctx_v, pkt);
            if (ret < 0)
                break;
            in_stream = ifmt_ctx_v->streams[pkt->stream_index];
            if (pkt->stream_index != istream_index_v)
            {
                av_packet_unref(pkt);
                continue;
            }


            pkt->stream_index = ostream_index_v;
            outstream = ofmt_ctx->streams[pkt->stream_index];


            if (pkt->pts == AV_NOPTS_VALUE)
            {
                AVRational time_base1 = in_stream->time_base;
                int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream->r_frame_rate);
                static int frame_index = 0;
                pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
                pkt->dts = pkt->pts;
                pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
                frame_index++;
            }
            pts_v = pkt->pts;

            // in log info
            log_packet(ifmt_ctx_v, pkt, "in video");

            av_packet_rescale_ts(pkt, in_stream->time_base, outstream->time_base);
            pkt->pos = -1;
            // out log info
            log_packet(ofmt_ctx, pkt, "out video");

            ret = av_interleaved_write_frame(ofmt_ctx, pkt);
            if (ret < 0)
            {
                fprintf(stderr, "Error muxing packet\n");
                return -1;
            }
        }
    }


    ret = av_write_trailer(ofmt_ctx);
    if (ret < 0)
    {
        fprintf(stderr, "Error av_write_trailer\n");
    }
#if EXTRAL
    printf("===========Output Information==========\n");
    av_dump_format(ofmt_ctx, 0, out_filename, 1);
    printf("======================================\n");
#endif
    av_packet_free(&pkt);
    avformat_close_input(&ifmt_ctx_v);
    avformat_close_input(&ifmt_ctx_a);
    if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
        avio_closep(&ofmt_ctx->pb);
    avformat_free_context(ofmt_ctx);
    return 0;
}