I am using MediaCodec to decode a video, for some reason I cannot config the codec with a Surface. So I render the video using EGL. Here is my render logic.
videoDecoder.setStateListener(object : DefDecoderStateListener {
override fun decodeOneFrame(
codec: MediaCodec, // already set the format MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
bufferInfo: MediaCodec.BufferInfo?,
outputBuffers: Array<ByteBuffer>,
index: Int
) {
// renderer draw
val image = codec.getOutputImage(index)!!
yuvDrawer.setImage(image)
mRenderer.notifySwap(bufferInfo?.presentationTimeUs ?: 0)
}
})
My thought is get the YUV buffer from the output Image and render it by the YuvDrawer. But when I run the code, I got wrong result. Is there any problem with my YuvDrawer?
package com.cxp.learningvideo.opengl.drawer
import android.graphics.SurfaceTexture
import android.media.Image
import android.opengl.GLES11Ext
import android.opengl.GLES20
import android.opengl.Matrix
import android.util.Log
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
class YuvDrawer: IDrawer {
private val mVertexCoors = floatArrayOf(
-1f, -1f,
1f, -1f,
-1f, 1f,
1f, 1f
)
private val mTextureCoors = floatArrayOf(
0f, 1f,
1f, 1f,
0f, 0f,
1f, 0f
)
private var mWorldWidth: Int = -1
private var mWorldHeight: Int = -1
private var mVideoWidth: Int = -1
private var mVideoHeight: Int = -1
private var mTextureId: Int = -1
private var mSurfaceTexture: SurfaceTexture? = null
private var mSftCb: (() -> Unit)? = null
private var mProgram: Int = -1
private var mVertexMatrixHandler: Int = -1
private var mVertexPosHandler: Int = -1
private var mTexturePosHandler: Int = -1
private var mTextureYHandler: Int = -1
private var mTextureUHandler: Int = -1
private var mTextureVHandler: Int = -1
private var mAlphaHandler: Int = -1
private lateinit var mVertexBuffer: FloatBuffer
private lateinit var mTextureBuffer: FloatBuffer
private var mMatrix: FloatArray? = null
private var mAlpha = 1f
private val textures = IntArray(3)
private var bufferY: ByteBuffer? = null
private var bufferU: ByteBuffer? = null
private var bufferV: ByteBuffer? = null
private var image: Image? = null
init {
initPos()
}
private fun initPos() {
val bb = ByteBuffer.allocateDirect(mVertexCoors.size * 4)
bb.order(ByteOrder.nativeOrder())
mVertexBuffer = bb.asFloatBuffer()
mVertexBuffer.put(mVertexCoors)
mVertexBuffer.position(0)
val cc = ByteBuffer.allocateDirect(mTextureCoors.size * 4)
cc.order(ByteOrder.nativeOrder())
mTextureBuffer = cc.asFloatBuffer()
mTextureBuffer.put(mTextureCoors)
mTextureBuffer.position(0)
}
private var mWidthRatio = 1f
private var mHeightRatio = 1f
private fun initDefMatrix() {
if (mMatrix != null) return
if (mVideoWidth != -1 && mVideoHeight != -1 &&
mWorldWidth != -1 && mWorldHeight != -1) {
mMatrix = FloatArray(16)
var prjMatrix = FloatArray(16)
val originRatio = mVideoWidth / mVideoHeight.toFloat()
val worldRatio = mWorldWidth / mWorldHeight.toFloat()
if (mWorldWidth > mWorldHeight) {
if (originRatio > worldRatio) {
mHeightRatio = originRatio / worldRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
} else {
mWidthRatio = worldRatio / originRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
}
} else {
if (originRatio > worldRatio) {
mHeightRatio = originRatio / worldRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
} else {
mWidthRatio = worldRatio / originRatio
Matrix.orthoM(
prjMatrix, 0,
-mWidthRatio, mWidthRatio,
-mHeightRatio, mHeightRatio,
3f, 5f
)
}
}
val viewMatrix = FloatArray(16)
Matrix.setLookAtM(
viewMatrix, 0,
0f, 0f, 5.0f,
0f, 0f, 0f,
0f, 1.0f, 0f
)
Matrix.multiplyMM(mMatrix, 0, prjMatrix, 0, viewMatrix, 0)
}
}
fun setImage(image: Image) {
this.image = image
try {
bufferY = this.image!!.planes[0].buffer
Log.d("testt", "setImage: buffer y $bufferY")
bufferU = this.image!!.planes[1].buffer
bufferV = this.image!!.planes[2].buffer
} catch (e: Exception) {
e.printStackTrace()
Log.d("YuvDrawer", "setImage: exception")
}
}
override fun setVideoSize(videoW: Int, videoH: Int) {
mVideoWidth = videoW
mVideoHeight = videoH
}
override fun setWorldSize(worldW: Int, worldH: Int) {
mWorldWidth = worldW
mWorldHeight = worldH
}
override fun setAlpha(alpha: Float) {
mAlpha = alpha
}
fun setSurfaceCreateListener(listener: () -> Unit) {
mSftCb = listener
}
private fun createGLPrg() {
if (mProgram == -1) {
val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, getVertexShader())
val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, getFragmentShader())
mProgram = GLES20.glCreateProgram()
GLES20.glAttachShader(mProgram, vertexShader)
GLES20.glAttachShader(mProgram, fragmentShader)
GLES20.glLinkProgram(mProgram)
mVertexMatrixHandler = GLES20.glGetUniformLocation(mProgram, "uMatrix")
mVertexPosHandler = GLES20.glGetAttribLocation(mProgram, "a_Position")
mTextureYHandler = GLES20.glGetUniformLocation(mProgram, "textureY")
mTextureUHandler = GLES20.glGetUniformLocation(mProgram, "textureU")
mTextureVHandler = GLES20.glGetUniformLocation(mProgram, "textureV")
mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate")
mAlphaHandler = GLES20.glGetAttribLocation(mProgram, "alpha")
}
GLES20.glUseProgram(mProgram)
}
private fun activateTexture() {
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
// y
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth,
mVideoHeight,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferY
)
// u
GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[1])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth / 2,
mVideoHeight / 2,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferU
)
//v
GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[2])
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
0,
GLES20.GL_LUMINANCE,
mVideoWidth / 2,
mVideoHeight / 2,
0,
GLES20.GL_LUMINANCE,
GLES20.GL_UNSIGNED_BYTE,
bufferV
)
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat())
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
// bufferY?.clear()
// bufferU?.clear()
// bufferV?.clear()
}
override fun draw() {
initDefMatrix()
activateTexture()
doDraw()
bufferY = null
bufferU = null
bufferV = null
image?.close()
}
private fun doDraw() {
GLES20.glEnableVertexAttribArray(mVertexPosHandler)
GLES20.glEnableVertexAttribArray(mTexturePosHandler)
GLES20.glUniformMatrix4fv(mVertexMatrixHandler, 1, false, mMatrix, 0)
GLES20.glVertexAttribPointer(mVertexPosHandler, 2, GLES20.GL_FLOAT, false, 0, mVertexBuffer)
GLES20.glVertexAttribPointer(mTexturePosHandler, 2, GLES20.GL_FLOAT, false, 0, mTextureBuffer)
GLES20.glVertexAttrib1f(mAlphaHandler, mAlpha)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
}
override fun setTextureID(id: Int) {
createGLPrg()
GLES20.glUniform1i(mTextureYHandler, 0)
GLES20.glUniform1i(mTextureUHandler, 1)
GLES20.glUniform1i(mTextureVHandler, 2)
GLES20.glGenTextures(3, textures, 0)
for(i in 0..2) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i])
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_REPEAT)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_REPEAT)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST
)
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D,
GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR
)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
}
mSftCb?.invoke()
}
override fun release() {
GLES20.glDisableVertexAttribArray(mVertexPosHandler)
GLES20.glDisableVertexAttribArray(mTexturePosHandler)
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0)
GLES20.glDeleteTextures(1, intArrayOf(mTextureId), 0)
GLES20.glDeleteProgram(mProgram)
}
private fun loadShader(type: Int, shaderCode: String): Int {
val shader = GLES20.glCreateShader(type)
GLES20.glShaderSource(shader, shaderCode)
GLES20.glCompileShader(shader)
return shader
}
fun translate(dx: Float, dy: Float) {
Matrix.translateM(mMatrix, 0, dx*mWidthRatio*2, -dy*mHeightRatio*2, 0f)
}
fun scale(sx: Float, sy: Float) {
Matrix.scaleM(mMatrix, 0, sx, sy, 1f)
mWidthRatio /= sx
mHeightRatio /= sy
}
private fun getVertexShader(): String {
return """#version 300 es
layout(location = 0) in vec4 a_Position;
layout(location = 1) in vec2 aTexture;
out vec2 vTexture;
void main()
{
gl_Position = a_Position;
vTexture = aTexture;
}
"""
}
private fun getFragmentShader(): String {
return """#version 300 es
precision mediump float;
out vec4 FragColor;
in vec2 vTexture;
uniform sampler2D textureY;
uniform sampler2D textureU;
uniform sampler2D textureV;
void main() {
float y,u,v;
vec3 rgb;
y = texture(textureY, vTexture).r;
u = texture(textureU, vTexture).g - 0.5;
v = texture(textureV, vTexture).b - 0.5;
rgb.r = y + 1.540 * v;
rgb.g = y - 0.183*u - 0.459*v;
rgb.b = y + 1.818*u;
FragColor = vec4(rgb, 1.0);
}
"""
}
}
Did this render logic make sense or should I consider another way.
mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aCoordinate") -> mTexturePosHandler = GLES20.glGetAttribLocation(mProgram, "aTexture")