FFMpeg multithreading while decoding udp stream

107 Views Asked by At

I have written a code using FFmpeg to decode an mpts UDP stream and display all programs within it simultaneously. In the first code snippet I shared below allows me to display up to 6 programs smoothly, but pixelation issues start after the 6th program.

public ref class FF_Decode {
    public:
        //Constructor, DeConstructor, ..... etc
        .   
        .
        .
        .
        
        void run() {
            .
            .
            .
            .
            //Initialize ffmpeg and take stream info, ... etc
            .
            .
            .
    
            // Read packets from the stream
            AVPacket packet;
            av_init_packet(&packet);
    
            while (av_read_frame(m_pFormatCtx, &packet) >= 0 && alive) {
                if (videoStreamList->Contains(packet.stream_index)) {
                    VideoDecoder^ v_decoder = videoDecoderDictionary[packet.stream_index];
                    if (v_decoder) {
                        int res = avcodec_send_packet(v_decoder->codecContext, &packet);
                        if (res < 0) {
                            // Error handling
                            char errorBuffer[AV_ERROR_MAX_STRING_SIZE];
                            av_strerror(res, errorBuffer, sizeof(errorBuffer));
                            printf("Error sending packet: %s\n", errorBuffer);
    
                            continue;
                        }
    
                        int ret;
                        AVFrame* frame = av_frame_alloc();
    
                        counter = 0;
                        while ((ret = avcodec_receive_frame(v_decoder->codecContext, frame)) >= 0 && alive) {
                            // Process the decoded frame
                            if (frame->data[0] != nullptr) {
    
                                RaiseVideoCallbackEvent(frame, packet.stream_index);
    
                            }
    
    
                            counter++;
                            av_frame_unref(frame);
                        }
    
                        av_frame_free(&frame);
                    }
                }
    
                av_packet_unref(&packet);
            }
            .
            .
            .
            .
        }
    }

To solve this issue, I modified my code as below to processing packets for each program in separate threads. This eliminates the pixelation problem, but I randomly encounter an AccessViolationException error when attempting to copy frame data in the event handler method.

public ref class VideoDecoder {
public:
    //Constructor, DeConstructor, ..... etc

    void job() {

        while (getPacketCount() < 20) {
            Sleep(40);
        }

        while (alive)
        {
            if (getPacketCount() < 1) {
                Sleep(20);
                continue;
            }

            auto start_time = std::chrono::high_resolution_clock::now();

            AVPacket* packet = getFirstPacket();
            //removePacket(packet);
            removeFirstPacket();

            if (packet->pts > 0) {
                int res = avcodec_send_packet(codecContext, packet);
                if (res < 0) {
                    // Error handling
                    char errorBuffer[AV_ERROR_MAX_STRING_SIZE];
                    av_strerror(res, errorBuffer, sizeof(errorBuffer));
                    printf("Error sending packet: %s\n", errorBuffer);
                } else {
                    int counter = 0;
                    AVFrame* frame = av_frame_alloc();
                    while ((res = avcodec_receive_frame(codecContext, frame)) >= 0 && alive) {
                        if (frame->data[0] != nullptr) {
                            RaiseVideoCallbackEvent(frame, packet->stream_index);
                        }

                        counter++;
                        av_frame_unref(frame);
                        //av_frame_free(&frame);
                    }

                }
            }

            av_packet_unref(packet);
            av_packet_free(&packet);
        }
    }

    void add_to_queue(AVPacket* pp) {
        AVPacket* packet = av_packet_alloc();
        if (packet) {
            av_init_packet(packet);

            if (av_packet_ref(packet, pp) >= 0) {
                addPacket(packet);
            } else {
                av_packet_free(&packet);
            }
        }
    }

    AVCodecContext* codecContext;

private:

    bool alive = true;
    Thread^ thread;

private:

    std::vector<AVPacket*>* packetList;

    void addPacket(AVPacket* packet) {
        packetList->push_back(packet);
    }

    AVPacket* getFirstPacket() {
        if (!packetList->empty()) return packetList->front();
        return nullptr;
    }

    void removeFirstPacket() {
        if (!packetList->empty()) {
            packetList->erase(packetList->begin());
        }
    }

    int getPacketCount()
    {
        return static_cast<int>(packetList->size());
    }
};


public ref class FF_Decode {
public:
    //Constructor, DeConstructor, ..... etc
    .   
    .
    .
    .
    
    void run() {
        .
        .
        .
        .
        //Initialize ffmpeg and take stream info, ... etc
        .
        .
        .

        // Read packets from the stream
        AVPacket packet;
        av_init_packet(&packet);

        while (av_read_frame(m_pFormatCtx, &packet) >= 0 && alive) {
            if (videoStreamList->Contains(packet.stream_index)) {
                VideoDecoder^ v_decoder = videoDecoderDictionary[packet.stream_index];
                if (v_decoder) {
                    v_decoder->add_to_queue(&packet);
                }
            }

            av_packet_unref(&packet);
        }
        .
        .
        .
        .
    }
}

Thanks in advance for any help.

0

There are 0 best solutions below