Decode frame produced by ffmpeg lib using webcodecs

507 Views Asked by At

server side extracting frame from video using ffmpeg library. Here is code:

    AVFormatContext *pFormatContext = avformat_alloc_context();
    avformat_open_input(&pFormatContext, "VID_20230219_171331_300.mp4.mov", NULL, NULL);
    avformat_find_stream_info(pFormatContext,  NULL);

    const AVCodec *pCodec = NULL;
    AVCodecParameters *pCodecParameters =  NULL;
    int video_stream_index = -1;

    // loop though all the streams and print its main information
    for (int i = 0; i < pFormatContext->nb_streams; i++)
    {
        AVCodecParameters *pLocalCodecParameters =  NULL;
        pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
        const AVCodec *pLocalCodec = NULL;
        pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);

        if(pLocalCodec==NULL)
            continue;

        // when the stream is a video we store its index, codec parameters and codec
        if(pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO)
        {
            if(video_stream_index == -1)
            {
                video_stream_index = i;
                pCodec = pLocalCodec;
                pCodecParameters = pLocalCodecParameters;

                const char *buffer = reinterpret_cast <const char*> (pFormatContext->streams[i]->codecpar->extradata);
                QFile file("extradata.txt"); //!!!!!!!!!!!!!!!! storing extradata for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, pFormatContext->streams[i]->codecpar->extradata_size);
                file.flush();
                file.close();
            }
        }
    }

    AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
    avcodec_parameters_to_context(pCodecContext, pCodecParameters);
    avcodec_open2(pCodecContext, pCodec, NULL);

    AVFrame *pFrame = av_frame_alloc();
    AVPacket *pPacket = av_packet_alloc();

    int response = 0;
    int how_many_packets_to_process = 800;

    int frame_c = 0, counter = 0;
    while(true)
    {
        frame_c = av_read_frame(pFormatContext, pPacket);

        if(frame_c < 0)
            break;

        if(pPacket->stream_index == video_stream_index)
        {
            const char *buffer = reinterpret_cast <const char*> (pPacket->data);
            response = decode_packet(pPacket, pCodecContext, pFrame);

            if(response < 0)
                break;

            counter++;
            if(--how_many_packets_to_process <= 0) break;

            if(counter > 1)
            {
                QFile file("frame.txt"); // !!!!!!!!!!!!!!!! storing single frame for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, arr.size());
                file.flush();
                file.close();
                break;
            }
        }

        av_packet_unref(pPacket);
    }

As the result i stored 2 txt files: 1 with extradata of codec and 2 with frame pixel data. The next step is to decode this data in browser and display picture. Here is how i try to do it:

<input type="file" id="file-input" />
<h3>Contents of the file:</h3>
<pre id="file-content"></pre>

<input type="file" id="file-input2" />
<h3>Contents of the file:</h3>
<pre id="file-content2"></pre>

<canvas id="myCanvas" width="720" height="1280"
style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.
</canvas>

<script>

let pendingFrames = [];
let extraData = new Uint8Array;

const init = {
  output: handleFrame,
  error: (e) => {
  console.log(e.message);
  },
};

const config = {
  codec: "avc1.64001E",
  codedWidth:  720,
  codedHeight: 1280,
  description: extraData.buffer,
};

function handleFrame(frame) {
  const canvas = document.getElementById("myCanvas");
  const ctx = canvas.getContext("2d");
  ctx.drawImage(frame, 0, 0);
  frame.close();
}

function readSingleFile(e) {
  var file = e.target.files[0];
  if (!file) {
    return;
  }
  var reader = new FileReader();
  reader.onload = function(e) {
    var contents = e.target.result;
    displayContents(contents);
  };
  reader.readAsText(file);
} //end

function displayContents(contents) {
  var element = document.getElementById('file-content');
  element.textContent = contents;
} // end

const readFile = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      let arrayBuffer = new Uint8Array(reader.result);
      console.log(arrayBuffer);

        const decoder = new VideoDecoder(init);
        decoder.configure(config);
        
        const chunk = new EncodedVideoChunk({
          timestamp: 20,
          type: "key",
          data: arrayBuffer,
        });

        decoder.decode(chunk);
  }

  reader.readAsArrayBuffer(file);
} //end

const readFile2 = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      extraData = new Uint8Array(reader.result);
      console.log(extraData);
  }

  reader.readAsArrayBuffer(file);
} //end

document.getElementById('file-input')
  .addEventListener('change', readFile, false);

document.getElementById('file-input2')
  .addEventListener('change', readFile2, false);

</script>

file-input2 - is for extradata and file-input - for frame data.

After i run this code in browser i got following error message: ---> Failed to execute 'configure' on 'VideoDecoder': description is detached. Also i had another type of error messages such as something wrong with codec config (codec description)

Please help to find what im doing wrong =) Thanks in advance.

0

There are 0 best solutions below