From 1f8c2a43986bafc2390d7ece46e35da640e142de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=AE=E7=94=9F=E8=8B=A5=E6=A2=A6?= <1070753498@qq.com> Date: Thu, 14 Sep 2023 18:37:42 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A8=E6=80=81=E5=8A=A0=E8=BD=BDshader?= =?UTF-8?q?=EF=BC=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ffmpeg/videorender/openglrender.cc | 315 +++++++++++-------- ffmpeg/videorender/openglrender.hpp | 8 +- ffmpeg/videorender/shader/video_nv12.frag | 24 ++ ffmpeg/videorender/shader/video_yuv420p.frag | 26 ++ ffmpeg/videorender/shaders.qrc | 2 + 5 files changed, 249 insertions(+), 126 deletions(-) create mode 100644 ffmpeg/videorender/shader/video_nv12.frag create mode 100644 ffmpeg/videorender/shader/video_yuv420p.frag diff --git a/ffmpeg/videorender/openglrender.cc b/ffmpeg/videorender/openglrender.cc index c646cc3..9fe56d0 100644 --- a/ffmpeg/videorender/openglrender.cc +++ b/ffmpeg/videorender/openglrender.cc @@ -30,12 +30,10 @@ class OpenglRender::OpenglRenderPrivate GLuint textureY; GLuint textureU; GLuint textureV; - GLuint textureUV; GLuint textureRGBA; // sub QScopedPointer subProgramPtr; GLuint textureSub; - bool subChanged = false; const QVector supportFormats = {AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUYV422, AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, @@ -47,7 +45,9 @@ class OpenglRender::OpenglRenderPrivate QScopedPointer frameConverterPtr; QSharedPointer framePtr; + bool frameChanged = false; QSharedPointer subTitleFramePtr; + bool subChanged = false; QColor backgroundColor = Qt::black; }; @@ -67,13 +67,8 @@ OpenglRender::~OpenglRender() } makeCurrent(); glDeleteVertexArrays(1, &d_ptr->vao); - d_ptr->programPtr.reset(); + cleanup(); d_ptr->subProgramPtr.reset(); - glDeleteTextures(1, &d_ptr->textureY); - glDeleteTextures(1, &d_ptr->textureU); - glDeleteTextures(1, &d_ptr->textureV); - glDeleteTextures(1, &d_ptr->textureUV); - glDeleteTextures(1, &d_ptr->textureRGBA); glDeleteTextures(1, &d_ptr->textureSub); doneCurrent(); } @@ -125,28 +120,13 @@ auto OpenglRender::widget() -> QWidget * void OpenglRender::updateFrame(QSharedPointer frame) { QMetaObject::invokeMethod( - this, - [=] { - d_ptr->framePtr = frame; - update(); - }, - Qt::QueuedConnection); + this, [=] { onUpdateFrame(frame); }, Qt::QueuedConnection); } void OpenglRender::updateSubTitleFrame(QSharedPointer frame) { QMetaObject::invokeMethod( - this, - [=] { - if (d_ptr->subTitleFramePtr.isNull() - || d_ptr->subTitleFramePtr->image().size() != frame->image().size()) { - d_ptr->subChanged = true; - } - d_ptr->subTitleFramePtr = frame; - // need update? - //update(); - }, - Qt::QueuedConnection); + this, [=] { onUpdateSubTitleFrame(frame); }, Qt::QueuedConnection); } void OpenglRender::initTexture() @@ -155,8 +135,7 @@ void OpenglRender::initTexture() d_ptr->programPtr->setUniformValue("tex_y", 0); d_ptr->programPtr->setUniformValue("tex_u", 1); d_ptr->programPtr->setUniformValue("tex_v", 2); - d_ptr->programPtr->setUniformValue("tex_uv", 3); - d_ptr->programPtr->setUniformValue("tex_rgba", 4); + d_ptr->programPtr->setUniformValue("tex_rgba", 3); glGenTextures(1, &d_ptr->textureY); glBindTexture(GL_TEXTURE_2D, d_ptr->textureY); @@ -179,13 +158,6 @@ void OpenglRender::initTexture() glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glGenTextures(1, &d_ptr->textureUV); - glBindTexture(GL_TEXTURE_2D, d_ptr->textureUV); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glGenTextures(1, &d_ptr->textureRGBA); glBindTexture(GL_TEXTURE_2D, d_ptr->textureRGBA); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); @@ -238,11 +210,83 @@ auto OpenglRender::fitToScreen(const QSize &size) -> QMatrix4x4 return matrix; } +void OpenglRender::cleanup() +{ + d_ptr->programPtr.reset(new OpenGLShaderProgram(this)); + if (d_ptr->textureY > 0) { + glDeleteTextures(1, &d_ptr->textureY); + } + if (d_ptr->textureU > 0) { + glDeleteTextures(1, &d_ptr->textureU); + } + if (d_ptr->textureV > 0) { + glDeleteTextures(1, &d_ptr->textureV); + } + if (d_ptr->textureRGBA > 0) { + glDeleteTextures(1, &d_ptr->textureRGBA); + } +} + +void OpenglRender::resetShader(int format) +{ + makeCurrent(); + cleanup(); + d_ptr->programPtr->addShaderFromSourceFile(QOpenGLShader::Vertex, ":/shader/video.vert"); + switch (format) { + case AV_PIX_FMT_YUV420P: + d_ptr->programPtr->addShaderFromSourceFile(QOpenGLShader::Fragment, + ":/shader/video_yuv420p.frag"); + break; + case AV_PIX_FMT_NV12: + case AV_PIX_FMT_NV21: + case AV_PIX_FMT_P010LE: + d_ptr->programPtr->addShaderFromSourceFile(QOpenGLShader::Fragment, + ":/shader/video_nv12.frag"); + break; + default: break; + } + glBindVertexArray(d_ptr->vao); + d_ptr->programPtr->link(); + d_ptr->programPtr->bind(); + d_ptr->programPtr->initVertex("aPos", "aTexCord"); + initTexture(); + d_ptr->programPtr->release(); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArray(0); + + doneCurrent(); +} + +void OpenglRender::onUpdateFrame(const QSharedPointer &framePtr) +{ + if (d_ptr->framePtr.isNull() + || d_ptr->framePtr->avFrame()->format != framePtr->avFrame()->format) { + resetShader(framePtr->avFrame()->format); + d_ptr->frameChanged = true; + } else if (d_ptr->framePtr->avFrame()->width != framePtr->avFrame()->width + || d_ptr->framePtr->avFrame()->height != framePtr->avFrame()->height) { + d_ptr->frameChanged = true; + } + d_ptr->framePtr = framePtr; + update(); +} + +void OpenglRender::onUpdateSubTitleFrame(const QSharedPointer &framePtr) +{ + if (d_ptr->subTitleFramePtr.isNull() + || d_ptr->subTitleFramePtr->image().size() != framePtr->image().size()) { + d_ptr->subChanged = true; + } + d_ptr->subTitleFramePtr = framePtr; + // need update? + //update(); +} + void OpenglRender::paintVideoFrame() { auto avFrame = d_ptr->framePtr->avFrame(); auto format = avFrame->format; - //qDebug() << format; + // qDebug() << format; // 绑定纹理 switch (format) { case AV_PIX_FMT_YUV420P: updateYUV420P(); break; @@ -257,7 +301,7 @@ void OpenglRender::paintVideoFrame() case AV_PIX_FMT_BGR8: updateRGB8(GL_UNSIGNED_BYTE_2_3_3_REV); break; case AV_PIX_FMT_RGB8: updateRGB8(GL_UNSIGNED_BYTE_3_3_2); break; case AV_PIX_FMT_NV12: - case AV_PIX_FMT_NV21: updateNV12(); break; + case AV_PIX_FMT_NV21: updateNV12(GL_UNSIGNED_BYTE); break; case AV_PIX_FMT_ARGB: case AV_PIX_FMT_RGBA: case AV_PIX_FMT_ABGR: @@ -267,11 +311,10 @@ void OpenglRender::paintVideoFrame() case AV_PIX_FMT_0BGR: case AV_PIX_FMT_BGR0: updateRGBA(); break; case AV_PIX_FMT_YUV420P10LE: updateYUV420P10LE(); break; - case AV_PIX_FMT_P010LE: updateP010LE(); break; + case AV_PIX_FMT_P010LE: updateNV12(GL_UNSIGNED_SHORT); break; default: break; } d_ptr->programPtr->bind(); // 绑定着色器 - d_ptr->programPtr->setUniformValue("format", format); d_ptr->programPtr->setUniformValue("transform", fitToScreen({avFrame->width, avFrame->height})); setColorSpace(); draw(); @@ -358,15 +401,6 @@ void OpenglRender::initializeGL() glBindVertexArray(d_ptr->vao); // 加载shader脚本程序 - d_ptr->programPtr.reset(new OpenGLShaderProgram(this)); - d_ptr->programPtr->addShaderFromSourceFile(QOpenGLShader::Vertex, ":/shader/video.vert"); - d_ptr->programPtr->addShaderFromSourceFile(QOpenGLShader::Fragment, ":/shader/video.frag"); - d_ptr->programPtr->link(); - d_ptr->programPtr->bind(); - d_ptr->programPtr->initVertex("aPos", "aTexCord"); - initTexture(); - d_ptr->programPtr->release(); - d_ptr->subProgramPtr.reset(new OpenGLShaderProgram(this)); d_ptr->subProgramPtr->addShaderFromSourceFile(QOpenGLShader::Vertex, ":/shader/video.vert"); d_ptr->subProgramPtr->addShaderFromSourceFile(QOpenGLShader::Fragment, ":/shader/sub.frag"); @@ -405,39 +439,76 @@ void OpenglRender::updateYUV420P() // Y glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, d_ptr->textureY); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RED, - frame->width, - frame->height, - 0, - GL_RED, - GL_UNSIGNED_BYTE, - frame->data[0]); + if (d_ptr->frameChanged) { + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + frame->width, + frame->height, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[0]); + } else { + glTexSubImage2D(GL_TEXTURE_2D, + 0, + 0, + 0, + frame->width, + frame->height, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[0]); + } // U glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, d_ptr->textureU); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RED, - frame->width / 2, - frame->height / 2, - 0, - GL_RED, - GL_UNSIGNED_BYTE, - frame->data[1]); + if (d_ptr->frameChanged) { + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + frame->width / 2, + frame->height / 2, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[1]); + } else { + glTexSubImage2D(GL_TEXTURE_2D, + 0, + 0, + 0, + frame->width / 2, + frame->height / 2, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[1]); + } // V glActiveTexture(GL_TEXTURE2); glBindTexture(GL_TEXTURE_2D, d_ptr->textureV); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RED, - frame->width / 2, - frame->height / 2, - 0, - GL_RED, - GL_UNSIGNED_BYTE, - frame->data[2]); + if (d_ptr->frameChanged) { + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + frame->width / 2, + frame->height / 2, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[2]); + } else { + glTexSubImage2D(GL_TEXTURE_2D, + 0, + 0, + 0, + frame->width / 2, + frame->height / 2, + GL_RED, + GL_UNSIGNED_BYTE, + frame->data[2]); + } + d_ptr->frameChanged = false; } void OpenglRender::updateYUYV422() @@ -652,33 +723,58 @@ void OpenglRender::updateRGB8(int dataType) frame->data[0]); } -void OpenglRender::updateNV12() +void OpenglRender::updateNV12(GLenum type) { auto frame = d_ptr->framePtr->avFrame(); // Y glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, d_ptr->textureY); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RED, - frame->width, - frame->height, - 0, - GL_RED, - GL_UNSIGNED_BYTE, - frame->data[0]); + if (d_ptr->frameChanged) { + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + frame->width, + frame->height, + 0, + GL_RED, + type, + frame->data[0]); + } else { + glTexSubImage2D(GL_TEXTURE_2D, + 0, + 0, + 0, + frame->width, + frame->height, + GL_RED, + type, + frame->data[0]); + } // UV - glActiveTexture(GL_TEXTURE3); - glBindTexture(GL_TEXTURE_2D, d_ptr->textureUV); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RG, // GL_RG GL_LUMINANCE_ALPHA - frame->width / 2, - frame->height / 2, - 0, - GL_RG, // GL_RG GL_LUMINANCE_ALPHA - GL_UNSIGNED_BYTE, - frame->data[1]); + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, d_ptr->textureU); + if (d_ptr->frameChanged) { + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, // GL_RG GL_LUMINANCE_ALPHA + frame->width / 2, + frame->height / 2, + 0, + GL_RG, // GL_RG GL_LUMINANCE_ALPHA + type, + frame->data[1]); + } else { + glTexSubImage2D(GL_TEXTURE_2D, + 0, + 0, + 0, + frame->width / 2, + frame->height / 2, + GL_RG, // GL_RG GL_LUMINANCE_ALPHA + type, + frame->data[1]); + } + d_ptr->frameChanged = false; } void OpenglRender::updateRGB() @@ -754,33 +850,4 @@ void OpenglRender::updateYUV420P10LE() frame->data[2]); } -void OpenglRender::updateP010LE() -{ - auto frame = d_ptr->framePtr->avFrame(); - // Y - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, d_ptr->textureY); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RED, - frame->width, - frame->height, - 0, - GL_RED, - GL_UNSIGNED_SHORT, - frame->data[0]); - // UV - glActiveTexture(GL_TEXTURE3); - glBindTexture(GL_TEXTURE_2D, d_ptr->textureUV); - glTexImage2D(GL_TEXTURE_2D, - 0, - GL_LUMINANCE_ALPHA, - frame->width / 2, - frame->height / 2, - 0, - GL_LUMINANCE_ALPHA, - GL_UNSIGNED_SHORT, - frame->data[1]); -} - } // namespace Ffmpeg diff --git a/ffmpeg/videorender/openglrender.hpp b/ffmpeg/videorender/openglrender.hpp index b3f7b42..9d74df0 100644 --- a/ffmpeg/videorender/openglrender.hpp +++ b/ffmpeg/videorender/openglrender.hpp @@ -41,6 +41,11 @@ class FFMPEG_EXPORT OpenglRender : public VideoRender, void initSubTexture(); void setColorSpace(); auto fitToScreen(const QSize &size) -> QMatrix4x4; + void cleanup(); + void resetShader(int format); + + void onUpdateFrame(const QSharedPointer &framePtr); + void onUpdateSubTitleFrame(const QSharedPointer &framePtr); void paintVideoFrame(); void paintSubTitleFrame(); @@ -53,11 +58,10 @@ class FFMPEG_EXPORT OpenglRender : public VideoRender, void updateYUV411P(); void updateUYVY422(); void updateRGB8(int dataType); - void updateNV12(); + void updateNV12(GLenum type); void updateRGB(); void updateRGBA(); void updateYUV420P10LE(); - void updateP010LE(); class OpenglRenderPrivate; QScopedPointer d_ptr; diff --git a/ffmpeg/videorender/shader/video_nv12.frag b/ffmpeg/videorender/shader/video_nv12.frag new file mode 100644 index 0000000..d60d3c8 --- /dev/null +++ b/ffmpeg/videorender/shader/video_nv12.frag @@ -0,0 +1,24 @@ +#version 330 core + +in vec2 TexCord; // 纹理坐标 +out vec4 FragColor; // 输出颜色 + +uniform sampler2D tex_y; +uniform sampler2D tex_u; + +uniform vec3 offset; +uniform mat3 colorConversion; + +void main() +{ + vec3 yuv; + vec3 rgb; + + yuv.x = texture(tex_y, TexCord).r; + yuv.yz = texture(tex_u, TexCord).rg; + + yuv += offset; + rgb = yuv * colorConversion; + + FragColor = vec4(rgb, 1.0); +} diff --git a/ffmpeg/videorender/shader/video_yuv420p.frag b/ffmpeg/videorender/shader/video_yuv420p.frag new file mode 100644 index 0000000..0997028 --- /dev/null +++ b/ffmpeg/videorender/shader/video_yuv420p.frag @@ -0,0 +1,26 @@ +#version 330 core + +in vec2 TexCord; // 纹理坐标 +out vec4 FragColor; // 输出颜色 + +uniform sampler2D tex_y; +uniform sampler2D tex_u; +uniform sampler2D tex_v; + +uniform vec3 offset; +uniform mat3 colorConversion; + +void main() +{ + vec3 yuv; + vec3 rgb; + + yuv.x = texture(tex_y, TexCord).r; + yuv.y = texture(tex_u, TexCord).r; + yuv.z = texture(tex_v, TexCord).r; + + yuv += offset; + rgb = yuv * colorConversion; + + FragColor = vec4(rgb, 1.0); +} diff --git a/ffmpeg/videorender/shaders.qrc b/ffmpeg/videorender/shaders.qrc index 71ab616..5053e9c 100644 --- a/ffmpeg/videorender/shaders.qrc +++ b/ffmpeg/videorender/shaders.qrc @@ -5,5 +5,7 @@ shader/video_vulkan.frag shader/video_vulkan.vert shader/sub.frag + shader/video_nv12.frag + shader/video_yuv420p.frag