From eee06f2917d93aee6b28f4e578dbf2cf3402f1b6 Mon Sep 17 00:00:00 2001 From: chiajung hung Date: Tue, 8 Jul 2014 20:52:00 +0200 Subject: [PATCH] Bug 814524 - Make WebGLContext::TexImage2D avoid readback for video elements, r=jgilbert --- content/canvas/src/WebGLContext.cpp | 62 ++++ content/canvas/src/WebGLContext.h | 23 ++ gfx/gl/GLBlitHelper.cpp | 540 ++++++++++++++++++++-------- gfx/gl/GLBlitHelper.h | 72 +++- gfx/gl/ScopedGLHelpers.cpp | 70 ++++ gfx/gl/ScopedGLHelpers.h | 32 ++ gfx/gl/moz.build | 1 + 7 files changed, 647 insertions(+), 153 deletions(-) diff --git a/content/canvas/src/WebGLContext.cpp b/content/canvas/src/WebGLContext.cpp index 8e628d938aba..f6bf6f009e72 100644 --- a/content/canvas/src/WebGLContext.cpp +++ b/content/canvas/src/WebGLContext.cpp @@ -15,6 +15,7 @@ #include "WebGLVertexArray.h" #include "WebGLQuery.h" +#include "GLBlitHelper.h" #include "AccessCheck.h" #include "nsIConsoleService.h" #include "nsServiceManagerUtils.h" @@ -28,6 +29,7 @@ #include "nsIVariant.h" #include "ImageEncoder.h" +#include "ImageContainer.h" #include "gfxContext.h" #include "gfxPattern.h" @@ -57,6 +59,7 @@ #include "mozilla/Services.h" #include "mozilla/dom/WebGLRenderingContextBinding.h" #include "mozilla/dom/BindingUtils.h" +#include "mozilla/dom/HTMLVideoElement.h" #include "mozilla/dom/ImageData.h" #include "mozilla/ProcessPriorityManager.h" #include "mozilla/EnumeratedArrayCycleCollection.h" @@ -1523,6 +1526,65 @@ WebGLContext::GetSurfaceSnapshot(bool* aPremultAlpha) return dt->Snapshot(); } +bool WebGLContext::TexImageFromVideoElement(GLenum target, GLint level, + GLenum internalformat, GLenum format, GLenum type, + mozilla::dom::Element& elt) +{ + HTMLVideoElement* video = HTMLVideoElement::FromContentOrNull(&elt); + if (!video) { + return false; + } + + uint16_t readyState; + if (NS_SUCCEEDED(video->GetReadyState(&readyState)) && + readyState < nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA) + { + //No frame inside, just return + return false; + } + + // If it doesn't have a principal, just bail + nsCOMPtr principal = video->GetCurrentPrincipal(); + if (!principal) { + return false; + } + + mozilla::layers::ImageContainer* container = video->GetImageContainer(); + if (!container) { + return false; + } + + if (video->GetCORSMode() == CORS_NONE) { + bool subsumes; + nsresult rv = mCanvasElement->NodePrincipal()->Subsumes(principal, &subsumes); + if (NS_FAILED(rv) || !subsumes) { + GenerateWarning("It is forbidden to load a WebGL texture from a cross-domain element that has not been validated with CORS. " + "See https://developer.mozilla.org/en/WebGL/Cross-Domain_Textures"); + return false; + } + } + + gl->MakeCurrent(); + nsRefPtr srcImage = container->LockCurrentImage(); + WebGLTexture* tex = activeBoundTextureForTarget(target); + + const WebGLTexture::ImageInfo& info = tex->ImageInfoAt(target, 0); + bool dimensionsMatch = info.Width() == srcImage->GetSize().width && + info.Height() == srcImage->GetSize().height; + if (!dimensionsMatch) { + // we need to allocation + gl->fTexImage2D(target, level, internalformat, srcImage->GetSize().width, srcImage->GetSize().height, 0, format, type, nullptr); + } + bool ok = gl->BlitHelper()->BlitImageToTexture(srcImage.get(), srcImage->GetSize(), tex->GLName(), target, mPixelStoreFlipY); + if (ok) { + tex->SetImageInfo(target, level, srcImage->GetSize().width, srcImage->GetSize().height, format, type, WebGLImageDataStatus::InitializedImageData); + tex->Bind(target); + } + srcImage = nullptr; + container->UnlockCurrentImage(); + return ok; +} + // // XPCOM goop // diff --git a/content/canvas/src/WebGLContext.h b/content/canvas/src/WebGLContext.h index 0f2f9564c149..1c18b385a03d 100644 --- a/content/canvas/src/WebGLContext.h +++ b/content/canvas/src/WebGLContext.h @@ -80,6 +80,7 @@ class WebGLVertexArray; namespace dom { class ImageData; +class Element; struct WebGLContextAttributes; template struct Nullable; @@ -460,6 +461,10 @@ public: dom::ImageData* pixels, ErrorResult& rv); // Allow whatever element types the bindings are willing to pass // us in TexImage2D + bool TexImageFromVideoElement(GLenum target, GLint level, + GLenum internalformat, GLenum format, GLenum type, + mozilla::dom::Element& image); + template void TexImage2D(GLenum target, GLint level, GLenum internalformat, GLenum format, GLenum type, @@ -467,6 +472,17 @@ public: { if (IsContextLost()) return; + + WebGLTexture* tex = activeBoundTextureForTarget(target); + + if (!tex) + return ErrorInvalidOperation("no texture is bound to this target"); + + // Trying to handle the video by GPU directly first + if (TexImageFromVideoElement(target, level, internalformat, format, type, elt)) { + return; + } + RefPtr data; WebGLTexelFormat srcFormat; nsLayoutUtils::SurfaceFromElementResult res = SurfaceFromElement(elt); @@ -482,6 +498,7 @@ public: 0, format, type, data->GetData(), byteLength, -1, srcFormat, mPixelStorePremultiplyAlpha); } + void TexParameterf(GLenum target, GLenum pname, GLfloat param) { TexParameter_base(target, pname, nullptr, ¶m); } @@ -507,6 +524,12 @@ public: { if (IsContextLost()) return; + + // Trying to handle the video by GPU directly first + if (TexImageFromVideoElement(target, level, format, format, type, elt)) { + return; + } + RefPtr data; WebGLTexelFormat srcFormat; nsLayoutUtils::SurfaceFromElementResult res = SurfaceFromElement(elt); diff --git a/gfx/gl/GLBlitHelper.cpp b/gfx/gl/GLBlitHelper.cpp index 3f54e9c335c4..fe75c1e0e573 100644 --- a/gfx/gl/GLBlitHelper.cpp +++ b/gfx/gl/GLBlitHelper.cpp @@ -8,6 +8,16 @@ #include "GLContext.h" #include "ScopedGLHelpers.h" #include "mozilla/Preferences.h" +#include "ImageContainer.h" +#include "HeapCopyOfStackArray.h" + +#ifdef MOZ_WIDGET_GONK +#include "GrallocImages.h" +#include "GLLibraryEGL.h" +#endif + +using mozilla::layers::PlanarYCbCrImage; +using mozilla::layers::PlanarYCbCrData; namespace mozilla { namespace gl { @@ -31,14 +41,14 @@ RenderbufferStorageBySamples(GLContext* aGL, GLsizei aSamples, GLuint CreateTexture(GLContext* aGL, GLenum aInternalFormat, GLenum aFormat, - GLenum aType, const gfx::IntSize& aSize) + GLenum aType, const gfx::IntSize& aSize, bool linear) { GLuint tex = 0; aGL->fGenTextures(1, &tex); ScopedBindTexture autoTex(aGL, tex); - aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MIN_FILTER, LOCAL_GL_LINEAR); - aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MAG_FILTER, LOCAL_GL_LINEAR); + aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MIN_FILTER, linear ? LOCAL_GL_LINEAR : LOCAL_GL_NEAREST); + aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_MAG_FILTER, linear ? LOCAL_GL_LINEAR : LOCAL_GL_NEAREST); aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); aGL->fTexParameteri(LOCAL_GL_TEXTURE_2D, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); @@ -54,7 +64,6 @@ CreateTexture(GLContext* aGL, GLenum aInternalFormat, GLenum aFormat, return tex; } - GLuint CreateTextureForOffscreen(GLContext* aGL, const GLFormats& aFormats, const gfx::IntSize& aSize) @@ -129,79 +138,167 @@ GLBlitHelper::GLBlitHelper(GLContext* gl) , mTex2DRectBlit_FragShader(0) , mTex2DBlit_Program(0) , mTex2DRectBlit_Program(0) + , mYFlipLoc(-1) + , mTexExternalBlit_FragShader(0) + , mTexYUVPlanarBlit_FragShader(0) + , mTexExternalBlit_Program(0) + , mTexYUVPlanarBlit_Program(0) + , mFBO(0) + , mSrcTexY(0) + , mSrcTexCb(0) + , mSrcTexCr(0) + , mSrcTexEGL(0) + , mYTexScaleLoc(-1) + , mCbCrTexScaleLoc(-1) + , mTexWidth(0) + , mTexHeight(0) + , mCurYScale(1.0f) + , mCurCbCrScale(1.0f) { } GLBlitHelper::~GLBlitHelper() { DeleteTexBlitProgram(); + + GLuint tex[] = { + mSrcTexY, + mSrcTexCb, + mSrcTexCr, + mSrcTexEGL, + }; + + mSrcTexY = mSrcTexCb = mSrcTexCr = mSrcTexEGL = 0; + mGL->fDeleteTextures(ArrayLength(tex), tex); + + if (mFBO) { + mGL->fDeleteFramebuffers(1, &mFBO); + } + mFBO = 0; } // Allowed to be destructive of state we restore in functions below. bool -GLBlitHelper::InitTexQuadProgram(GLenum target) +GLBlitHelper::InitTexQuadProgram(BlitType target) { const char kTexBlit_VertShaderSource[] = "\ - attribute vec2 aPosition; \n\ - \n\ - varying vec2 vTexCoord; \n\ - \n\ - void main(void) { \n\ - vTexCoord = aPosition; \n\ - vec2 vertPos = aPosition * 2.0 - 1.0; \n\ - gl_Position = vec4(vertPos, 0.0, 1.0); \n\ - } \n\ + attribute vec2 aPosition; \n\ + \n\ + uniform float uYflip; \n\ + varying vec2 vTexCoord; \n\ + \n\ + void main(void) \n\ + { \n\ + vTexCoord = aPosition; \n\ + vTexCoord.y = abs(vTexCoord.y - uYflip); \n\ + vec2 vertPos = aPosition * 2.0 - 1.0; \n\ + gl_Position = vec4(vertPos, 0.0, 1.0); \n\ + } \n\ "; const char kTex2DBlit_FragShaderSource[] = "\ - #ifdef GL_FRAGMENT_PRECISION_HIGH \n\ - precision highp float; \n\ - #else \n\ - precision mediump float; \n\ - #endif \n\ - \n\ - uniform sampler2D uTexUnit; \n\ - \n\ - varying vec2 vTexCoord; \n\ - \n\ - void main(void) { \n\ - gl_FragColor = texture2D(uTexUnit, vTexCoord); \n\ - } \n\ + #ifdef GL_FRAGMENT_PRECISION_HIGH \n\ + precision highp float; \n\ + #else \n\ + prevision mediump float; \n\ + #endif \n\ + uniform sampler2D uTexUnit; \n\ + \n\ + varying vec2 vTexCoord; \n\ + \n\ + void main(void) \n\ + { \n\ + gl_FragColor = texture2D(uTexUnit, vTexCoord); \n\ + } \n\ "; const char kTex2DRectBlit_FragShaderSource[] = "\ - #ifdef GL_FRAGMENT_PRECISION_HIGH \n\ - precision highp float; \n\ - #else \n\ - precision mediump float; \n\ - #endif \n\ - \n\ - uniform sampler2D uTexUnit; \n\ - uniform vec2 uTexCoordMult; \n\ - \n\ - varying vec2 vTexCoord; \n\ - \n\ - void main(void) { \n\ - gl_FragColor = texture2DRect(uTexUnit, \n\ - vTexCoord * uTexCoordMult); \n\ - } \n\ + #ifdef GL_FRAGMENT_PRECISION_HIGH \n\ + precision highp float; \n\ + #else \n\ + precision mediump float; \n\ + #endif \n\ + \n\ + uniform sampler2D uTexUnit; \n\ + uniform vec2 uTexCoordMult; \n\ + \n\ + varying vec2 vTexCoord; \n\ + \n\ + void main(void) \n\ + { \n\ + gl_FragColor = texture2DRect(uTexUnit, \n\ + vTexCoord * uTexCoordMult); \n\ + } \n\ + "; +#ifdef MOZ_WIDGET_GONK + const char kTexExternalBlit_FragShaderSource[] = "\ + #extension GL_OES_EGL_image_external : require \n\ + #ifdef GL_FRAGMENT_PRECISION_HIGH \n\ + precision highp float; \n\ + #else \n\ + precision mediump float; \n\ + #endif \n\ + varying vec2 vTexCoord; \n\ + uniform samplerExternalOES uTexUnit; \n\ + \n\ + void main() \n\ + { \n\ + gl_FragColor = texture2D(uTexUnit, vTexCoord); \n\ + } \n\ + "; +#endif + const char kTexYUVPlanarBlit_FragShaderSource[] = "\ + varying vec2 vTexCoord; \n\ + uniform sampler2D uYTexture; \n\ + uniform sampler2D uCbTexture; \n\ + uniform sampler2D uCrTexture; \n\ + uniform vec2 uYTexScale; \n\ + uniform vec2 uCbCrTexScale; \n\ + void main() \n\ + { \n\ + float y = texture2D(uYTexture, vTexCoord * uYTexScale).r; \n\ + float cb = texture2D(uCbTexture, vTexCoord * uCbCrTexScale).r; \n\ + float cr = texture2D(uCrTexture, vTexCoord * uCbCrTexScale).r; \n\ + y = (y - 0.0625) * 1.164; \n\ + cb = cb - 0.504; \n\ + cr = cr - 0.5; \n\ + gl_FragColor.r = floor((y + cr * 1.596) * 256.0)/256.0; \n\ + gl_FragColor.g = floor((y - 0.813 * cr - 0.391 * cb) * 256.0)/256.0; \n\ + gl_FragColor.b = floor((y + cb * 2.018) * 256.0) /256.0; \n\ + gl_FragColor.a = 1.0; \n\ + } \n\ "; - MOZ_ASSERT(target == LOCAL_GL_TEXTURE_2D || - target == LOCAL_GL_TEXTURE_RECTANGLE_ARB); bool success = false; GLuint *programPtr; GLuint *fragShaderPtr; const char* fragShaderSource; - if (target == LOCAL_GL_TEXTURE_2D) { + switch (target) { + case BlitTex2D: programPtr = &mTex2DBlit_Program; fragShaderPtr = &mTex2DBlit_FragShader; fragShaderSource = kTex2DBlit_FragShaderSource; - } else { + break; + case BlitTexRect: programPtr = &mTex2DRectBlit_Program; fragShaderPtr = &mTex2DRectBlit_FragShader; fragShaderSource = kTex2DRectBlit_FragShaderSource; + break; +#ifdef MOZ_WIDGET_GONK + case ConvertGralloc: + programPtr = &mTexExternalBlit_Program; + fragShaderPtr = &mTexExternalBlit_FragShader; + fragShaderSource = kTexExternalBlit_FragShaderSource; + break; +#endif + case ConvertPlanarYCbCr: + programPtr = &mTexYUVPlanarBlit_Program; + fragShaderPtr = &mTexYUVPlanarBlit_FragShader; + fragShaderSource = kTexYUVPlanarBlit_FragShaderSource; + break; + default: + return false; } GLuint& program = *programPtr; @@ -228,15 +325,14 @@ GLBlitHelper::InitTexQuadProgram(GLenum target) 0.0f, 1.0f, 1.0f, 1.0f }; + HeapCopyOfStackArray vertsOnHeap(verts); MOZ_ASSERT(!mTexBlit_Buffer); mGL->fGenBuffers(1, &mTexBlit_Buffer); mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, mTexBlit_Buffer); - const size_t vertsSize = sizeof(verts); // Make sure we have a sane size. - MOZ_ASSERT(vertsSize >= 3 * sizeof(GLfloat)); - mGL->fBufferData(LOCAL_GL_ARRAY_BUFFER, vertsSize, verts, LOCAL_GL_STATIC_DRAW); + mGL->fBufferData(LOCAL_GL_ARRAY_BUFFER, vertsOnHeap.ByteLength(), vertsOnHeap.Data(), LOCAL_GL_STATIC_DRAW); } if (!mTexBlit_VertShader) { @@ -319,20 +415,46 @@ GLBlitHelper::InitTexQuadProgram(GLenum target) break; } - MOZ_ASSERT(mGL->fGetAttribLocation(program, "aPosition") == 0); - GLint texUnitLoc = mGL->fGetUniformLocation(program, "uTexUnit"); - MOZ_ASSERT(texUnitLoc != -1, "uniform not found"); - - // Set uniforms here: + // Cache and set attribute and uniform mGL->fUseProgram(program); - mGL->fUniform1i(texUnitLoc, 0); + switch (target) { + case BlitTex2D: + case BlitTexRect: + case ConvertGralloc: { +#ifdef MOZ_WIDGET_GONK + GLint texUnitLoc = mGL->fGetUniformLocation(program, "uTexUnit"); + MOZ_ASSERT(texUnitLoc != -1, "uniform uTexUnit not found"); + mGL->fUniform1i(texUnitLoc, 0); + break; +#endif + } + case ConvertPlanarYCbCr: { + GLint texY = mGL->fGetUniformLocation(program, "uYTexture"); + GLint texCb = mGL->fGetUniformLocation(program, "uCbTexture"); + GLint texCr = mGL->fGetUniformLocation(program, "uCrTexture"); + mYTexScaleLoc = mGL->fGetUniformLocation(program, "uYTexScale"); + mCbCrTexScaleLoc= mGL->fGetUniformLocation(program, "uCbCrTexScale"); + DebugOnly hasUniformLocations = texY != -1 && + texCb != -1 && + texCr != -1 && + mYTexScaleLoc != -1 && + mCbCrTexScaleLoc != -1; + MOZ_ASSERT(hasUniformLocations, "uniforms not found"); + + mGL->fUniform1i(texY, Channel_Y); + mGL->fUniform1i(texCb, Channel_Cb); + mGL->fUniform1i(texCr, Channel_Cr); + break; + } + } + MOZ_ASSERT(mGL->fGetAttribLocation(program, "aPosition") == 0); + mYFlipLoc = mGL->fGetUniformLocation(program, "uYflip"); + MOZ_ASSERT(mYFlipLoc != -1, "uniform: uYflip not found"); success = true; } while (false); if (!success) { - NS_ERROR("Creating program for texture blit failed!"); - // Clean up: DeleteTexBlitProgram(); return false; @@ -351,13 +473,13 @@ GLBlitHelper::InitTexQuadProgram(GLenum target) } bool -GLBlitHelper::UseTexQuadProgram(GLenum target, const gfx::IntSize& srcSize) +GLBlitHelper::UseTexQuadProgram(BlitType target, const gfx::IntSize& srcSize) { if (!InitTexQuadProgram(target)) { return false; } - if (target == LOCAL_GL_TEXTURE_RECTANGLE_ARB) { + if (target == BlitTexRect) { GLint texCoordMultLoc = mGL->fGetUniformLocation(mTex2DRectBlit_Program, "uTexCoordMult"); MOZ_ASSERT(texCoordMultLoc != -1, "uniform not found"); mGL->fUniform2f(texCoordMultLoc, srcSize.width, srcSize.height); @@ -393,6 +515,22 @@ GLBlitHelper::DeleteTexBlitProgram() mGL->fDeleteProgram(mTex2DRectBlit_Program); mTex2DRectBlit_Program = 0; } + if (mTexExternalBlit_FragShader) { + mGL->fDeleteShader(mTexExternalBlit_FragShader); + mTexExternalBlit_FragShader = 0; + } + if (mTexYUVPlanarBlit_FragShader) { + mGL->fDeleteShader(mTexYUVPlanarBlit_FragShader); + mTexYUVPlanarBlit_FragShader = 0; + } + if (mTexExternalBlit_Program) { + mGL->fDeleteProgram(mTexExternalBlit_Program); + mTexExternalBlit_Program = 0; + } + if (mTexYUVPlanarBlit_Program) { + mGL->fDeleteProgram(mTexYUVPlanarBlit_Program); + mTexYUVPlanarBlit_Program = 0; + } } void @@ -441,6 +579,182 @@ GLBlitHelper::BlitFramebufferToFramebuffer(GLuint srcFB, GLuint destFB, mGL->fDeleteTextures(1, &tex); } +void +GLBlitHelper::BindAndUploadYUVTexture(Channel which, uint32_t width, uint32_t height, void* data, bool needsAllocation) +{ + MOZ_ASSERT(which < Channel_Max, "Invalid channel!"); + GLuint* srcTexArr[3] = {&mSrcTexY, &mSrcTexCb, &mSrcTexCr}; + GLuint& tex = *srcTexArr[which]; + if (!tex) { + MOZ_ASSERT(needsAllocation); + tex = CreateTexture(mGL, LOCAL_GL_LUMINANCE, LOCAL_GL_LUMINANCE, LOCAL_GL_UNSIGNED_BYTE, + gfx::IntSize(width, height), false); + } + mGL->fActiveTexture(LOCAL_GL_TEXTURE0 + which); + + mGL->fBindTexture(LOCAL_GL_TEXTURE_2D, tex); + if (!needsAllocation) { + mGL->fTexSubImage2D(LOCAL_GL_TEXTURE_2D, + 0, + 0, + 0, + width, + height, + LOCAL_GL_LUMINANCE, + LOCAL_GL_UNSIGNED_BYTE, + data); + } else { + mGL->fTexImage2D(LOCAL_GL_TEXTURE_2D, + 0, + LOCAL_GL_LUMINANCE, + width, + height, + 0, + LOCAL_GL_LUMINANCE, + LOCAL_GL_UNSIGNED_BYTE, + data); + } +} + +#ifdef MOZ_WIDGET_GONK +void +GLBlitHelper::BindAndUploadExternalTexture(EGLImage image) { + MOZ_ASSERT(image != EGL_NO_IMAGE, "Bad EGLImage"); + + if (!mSrcTexEGL) { + mGL->fGenTextures(1, &mSrcTexEGL); + mGL->fBindTexture(LOCAL_GL_TEXTURE_EXTERNAL_OES, mSrcTexEGL); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL_OES, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL_OES, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL_OES, LOCAL_GL_TEXTURE_MAG_FILTER, LOCAL_GL_NEAREST); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_EXTERNAL_OES, LOCAL_GL_TEXTURE_MIN_FILTER, LOCAL_GL_NEAREST); + } else { + mGL->fBindTexture(LOCAL_GL_TEXTURE_EXTERNAL_OES, mSrcTexEGL); + } + mGL->fEGLImageTargetTexture2D(LOCAL_GL_TEXTURE_EXTERNAL_OES, image); +} + +bool +GLBlitHelper::BlitGrallocImage(layers::GrallocImage* grallocImage, bool yFlip) { + ScopedBindTextureUnit boundTU(mGL, LOCAL_GL_TEXTURE0); + mGL->fClear(LOCAL_GL_COLOR_BUFFER_BIT); + + EGLint attrs[] = { + LOCAL_EGL_IMAGE_PRESERVED, LOCAL_EGL_TRUE, + LOCAL_EGL_NONE, LOCAL_EGL_NONE + }; + EGLImage image = sEGLLibrary.fCreateImage(sEGLLibrary.Display(), + EGL_NO_CONTEXT, + LOCAL_EGL_NATIVE_BUFFER_ANDROID, + grallocImage->GetNativeBuffer(), attrs); + if (image == EGL_NO_IMAGE) + return false; + + int oldBinding = 0; + mGL->fGetIntegerv(LOCAL_GL_TEXTURE_BINDING_EXTERNAL_OES, &oldBinding); + + BindAndUploadExternalTexture(image); + + mGL->fUniform1f(mYFlipLoc, yFlip ? (float)1.0f : (float)0.0f); + + mGL->fDrawArrays(LOCAL_GL_TRIANGLE_STRIP, 0, 4); + + sEGLLibrary.fDestroyImage(sEGLLibrary.Display(), image); + mGL->fBindTexture(LOCAL_GL_TEXTURE_EXTERNAL_OES, oldBinding); + return true; +} +#endif + +bool +GLBlitHelper::BlitPlanarYCbCrImage(layers::PlanarYCbCrImage* yuvImage, bool yFlip) +{ + ScopedBindTextureUnit boundTU(mGL, LOCAL_GL_TEXTURE0); + const PlanarYCbCrData* yuvData = yuvImage->GetData(); + + bool needsAllocation = false; + if (mTexWidth != yuvData->mYStride || mTexHeight != yuvData->mYSize.height) { + mTexWidth = yuvData->mYStride; + mTexHeight = yuvData->mYSize.height; + needsAllocation = true; + } + + GLint oldTex[3]; + for (int i = 0; i < 3; i++) { + mGL->fActiveTexture(LOCAL_GL_TEXTURE0 + i); + mGL->fGetIntegerv(LOCAL_GL_TEXTURE_BINDING_2D, &oldTex[i]); + } + BindAndUploadYUVTexture(Channel_Y, yuvData->mYStride, yuvData->mYSize.height, yuvData->mYChannel, needsAllocation); + BindAndUploadYUVTexture(Channel_Cb, yuvData->mCbCrStride, yuvData->mCbCrSize.height, yuvData->mCbChannel, needsAllocation); + BindAndUploadYUVTexture(Channel_Cr, yuvData->mCbCrStride, yuvData->mCbCrSize.height, yuvData->mCrChannel, needsAllocation); + + mGL->fUniform1f(mYFlipLoc, yFlip ? (float)1.0 : (float)0.0); + + if (needsAllocation) { + mGL->fUniform2f(mYTexScaleLoc, (float)yuvData->mYSize.width/yuvData->mYStride, 1.0f); + mGL->fUniform2f(mCbCrTexScaleLoc, (float)yuvData->mCbCrSize.width/yuvData->mCbCrStride, 1.0f); + } + + mGL->fDrawArrays(LOCAL_GL_TRIANGLE_STRIP, 0, 4); + for (int i = 0; i < 3; i++) { + mGL->fActiveTexture(LOCAL_GL_TEXTURE0 + i); + mGL->fBindTexture(LOCAL_GL_TEXTURE_2D, oldTex[i]); + } + return true; +} + +bool +GLBlitHelper::BlitImageToTexture(layers::Image* srcImage, const gfx::IntSize& destSize, GLuint destTex, GLenum destTarget, bool yFlip, GLuint xoffset, GLuint yoffset, GLuint cropWidth, GLuint cropHeight) +{ + ScopedGLDrawState autoStates(mGL); + + BlitType type; + switch (srcImage->GetFormat()) + { + case ImageFormat::PLANAR_YCBCR: + type = ConvertPlanarYCbCr; + break; + case ImageFormat::GRALLOC_PLANAR_YCBCR: +#ifdef MOZ_WIDGET_GONK + type = ConvertGralloc; + break; +#endif + default: + return false; + } + + bool init = InitTexQuadProgram(type); + if (!init) { + return false; + } + + if (!mFBO) { + mGL->fGenFramebuffers(1, &mFBO); + } + + ScopedBindFramebuffer boundFB(mGL, mFBO); + mGL->fFramebufferTexture2D(LOCAL_GL_FRAMEBUFFER, LOCAL_GL_COLOR_ATTACHMENT0, destTarget, destTex, 0); + mGL->fColorMask(LOCAL_GL_TRUE, LOCAL_GL_TRUE, LOCAL_GL_TRUE, LOCAL_GL_TRUE); + mGL->fViewport(0, 0, destSize.width, destSize.height); + if (xoffset != 0 && yoffset != 0 && cropWidth != 0 && cropHeight != 0) { + mGL->fEnable(LOCAL_GL_SCISSOR_TEST); + mGL->fScissor(xoffset, yoffset, (GLsizei)cropWidth, (GLsizei)cropHeight); + } + +#ifdef MOZ_WIDGET_GONK + if (type == ConvertGralloc) { + layers::GrallocImage* grallocImage = static_cast(srcImage); + return BlitGrallocImage(grallocImage, yFlip); + } +#endif + if (type == ConvertPlanarYCbCr) { + mGL->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT, 1); + PlanarYCbCrImage* yuvImage = static_cast(srcImage); + return BlitPlanarYCbCrImage(yuvImage, yFlip); + } + + return false; +} + void GLBlitHelper::BlitTextureToFramebuffer(GLuint srcTex, GLuint destFB, const gfx::IntSize& srcSize, @@ -459,106 +773,32 @@ GLBlitHelper::BlitTextureToFramebuffer(GLuint srcTex, GLuint destFB, return; } + BlitType type; + switch (srcTarget) + { + case LOCAL_GL_TEXTURE_2D: + type = BlitTex2D; + break; + case LOCAL_GL_TEXTURE_RECTANGLE_ARB: + type = BlitTexRect; + break; + default: + printf_stderr("Fatal Error: Failed to prepare to blit texture->framebuffer.\n"); + MOZ_CRASH(); + break; + } - ScopedBindFramebuffer boundFB(mGL, destFB); - // UseTexQuadProgram initializes a shader that reads - // from texture unit 0. - ScopedBindTextureUnit boundTU(mGL, LOCAL_GL_TEXTURE0); - ScopedBindTexture boundTex(mGL, srcTex, srcTarget); - - GLuint boundProgram = 0; - mGL->GetUIntegerv(LOCAL_GL_CURRENT_PROGRAM, &boundProgram); - - GLuint boundBuffer = 0; - mGL->GetUIntegerv(LOCAL_GL_ARRAY_BUFFER_BINDING, &boundBuffer); - - /* - * mGL->fGetVertexAttribiv takes: - * VERTEX_ATTRIB_ARRAY_ENABLED - * VERTEX_ATTRIB_ARRAY_SIZE, - * VERTEX_ATTRIB_ARRAY_STRIDE, - * VERTEX_ATTRIB_ARRAY_TYPE, - * VERTEX_ATTRIB_ARRAY_NORMALIZED, - * VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, - * CURRENT_VERTEX_ATTRIB - * - * CURRENT_VERTEX_ATTRIB is vertex shader state. \o/ - * Others appear to be vertex array state, - * or alternatively in the internal vertex array state - * for a buffer object. - */ - - GLint attrib0_enabled = 0; - GLint attrib0_size = 0; - GLint attrib0_stride = 0; - GLint attrib0_type = 0; - GLint attrib0_normalized = 0; - GLint attrib0_bufferBinding = 0; - void* attrib0_pointer = nullptr; - - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_ENABLED, &attrib0_enabled); - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_SIZE, &attrib0_size); - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_STRIDE, &attrib0_stride); - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_TYPE, &attrib0_type); - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_NORMALIZED, &attrib0_normalized); - mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, &attrib0_bufferBinding); - mGL->fGetVertexAttribPointerv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_POINTER, &attrib0_pointer); - // Note that uniform values are program state, so we don't need to rebind those. - - ScopedGLState blend (mGL, LOCAL_GL_BLEND, false); - ScopedGLState cullFace (mGL, LOCAL_GL_CULL_FACE, false); - ScopedGLState depthTest (mGL, LOCAL_GL_DEPTH_TEST, false); - ScopedGLState dither (mGL, LOCAL_GL_DITHER, false); - ScopedGLState polyOffsFill(mGL, LOCAL_GL_POLYGON_OFFSET_FILL, false); - ScopedGLState sampleAToC (mGL, LOCAL_GL_SAMPLE_ALPHA_TO_COVERAGE, false); - ScopedGLState sampleCover (mGL, LOCAL_GL_SAMPLE_COVERAGE, false); - ScopedGLState scissor (mGL, LOCAL_GL_SCISSOR_TEST, false); - ScopedGLState stencil (mGL, LOCAL_GL_STENCIL_TEST, false); - - realGLboolean colorMask[4]; - mGL->fGetBooleanv(LOCAL_GL_COLOR_WRITEMASK, colorMask); - mGL->fColorMask(LOCAL_GL_TRUE, - LOCAL_GL_TRUE, - LOCAL_GL_TRUE, - LOCAL_GL_TRUE); - - GLint viewport[4]; - mGL->fGetIntegerv(LOCAL_GL_VIEWPORT, viewport); - mGL->fViewport(0, 0, destSize.width, destSize.height); + ScopedGLDrawState autoStates(mGL); // Does destructive things to (only!) what we just saved above. - bool good = UseTexQuadProgram(srcTarget, srcSize); + bool good = UseTexQuadProgram(type, srcSize); if (!good) { // We're up against the wall, so bail. // This should really be MOZ_CRASH(why) or MOZ_RUNTIME_ASSERT(good). - printf_stderr("[%s:%d] Fatal Error: Failed to prepare to blit texture->framebuffer.\n", - __FILE__, __LINE__); + printf_stderr("[%s:%d] Fatal Error: Failed to prepare to blit texture->framebuffer.\n"); MOZ_CRASH(); } mGL->fDrawArrays(LOCAL_GL_TRIANGLE_STRIP, 0, 4); - - mGL->fViewport(viewport[0], viewport[1], - viewport[2], viewport[3]); - - mGL->fColorMask(colorMask[0], - colorMask[1], - colorMask[2], - colorMask[3]); - - if (attrib0_enabled) - mGL->fEnableVertexAttribArray(0); - - mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, attrib0_bufferBinding); - mGL->fVertexAttribPointer(0, - attrib0_size, - attrib0_type, - attrib0_normalized, - attrib0_stride, - attrib0_pointer); - - mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, boundBuffer); - - mGL->fUseProgram(boundProgram); } void diff --git a/gfx/gl/GLBlitHelper.h b/gfx/gl/GLBlitHelper.h index 987c4fdb586a..1f9c6be25860 100644 --- a/gfx/gl/GLBlitHelper.h +++ b/gfx/gl/GLBlitHelper.h @@ -14,6 +14,13 @@ #include "mozilla/gfx/Point.h" namespace mozilla { + +namespace layers { +class Image; +class PlanarYCbCrImage; +class GrallocImage; +} + namespace gl { class GLContext; @@ -38,7 +45,7 @@ GLuint CreateTextureForOffscreen(GLContext* aGL, const GLFormats& aFormats, * GL_TEXTURE_WRAP_T = GL_CLAMP_TO_EDGE */ GLuint CreateTexture(GLContext* aGL, GLenum aInternalFormat, GLenum aFormat, - GLenum aType, const gfx::IntSize& aSize); + GLenum aType, const gfx::IntSize& aSize, bool linear = true); /** * Helper function to create, potentially, multisample render buffers suitable @@ -63,6 +70,34 @@ void CreateRenderbuffersForOffscreen(GLContext* aGL, const GLFormats& aFormats, /** Buffer blitting helper */ class GLBlitHelper MOZ_FINAL { + enum Channel + { + Channel_Y = 0, + Channel_Cb, + Channel_Cr, + Channel_Max, + }; + + /** + * BlitTex2D is used to copy blit the content of a GL_TEXTURE_2D object, + * BlitTexRect is used to copy blit the content of a GL_TEXTURE_RECT object, + * The difference between BlitTex2D and BlitTexRect is the texture type, which affect + * the fragment shader a bit. + * + * ConvertGralloc is used to color convert copy blit the GrallocImage into a + * normal RGB texture by egl_image_external extension + * ConvertPlnarYcbCr is used to color convert copy blit the PlanarYCbCrImage + * into a normal RGB texture by create textures of each color channel, and + * convert it in GPU. + * Convert type is created for canvas. + */ + enum BlitType + { + BlitTex2D, + BlitTexRect, + ConvertGralloc, + ConvertPlanarYCbCr, + }; // The GLContext is the sole owner of the GLBlitHelper. GLContext* mGL; @@ -73,12 +108,40 @@ class GLBlitHelper MOZ_FINAL GLuint mTex2DBlit_Program; GLuint mTex2DRectBlit_Program; + GLint mYFlipLoc; + + // Data for image blit path + GLuint mTexExternalBlit_FragShader; + GLuint mTexYUVPlanarBlit_FragShader; + GLuint mTexExternalBlit_Program; + GLuint mTexYUVPlanarBlit_Program; + GLuint mFBO; + GLuint mSrcTexY; + GLuint mSrcTexCb; + GLuint mSrcTexCr; + GLuint mSrcTexEGL; + GLint mYTexScaleLoc; + GLint mCbCrTexScaleLoc; + int mTexWidth; + int mTexHeight; + + // Cache some uniform values + float mCurYScale; + float mCurCbCrScale; + void UseBlitProgram(); void SetBlitFramebufferForDestTexture(GLuint aTexture); - bool UseTexQuadProgram(GLenum target, const gfx::IntSize& srcSize); - bool InitTexQuadProgram(GLenum target = LOCAL_GL_TEXTURE_2D); + bool UseTexQuadProgram(BlitType target, const gfx::IntSize& srcSize); + bool InitTexQuadProgram(BlitType target = BlitTex2D); void DeleteTexBlitProgram(); + void BindAndUploadYUVTexture(Channel which, uint32_t width, uint32_t height, void* data, bool allocation); + +#ifdef MOZ_WIDGET_GONK + void BindAndUploadExternalTexture(EGLImage image); + bool BlitGrallocImage(layers::GrallocImage* grallocImage, bool yFlip = false); +#endif + bool BlitPlanarYCbCrImage(layers::PlanarYCbCrImage* yuvImage, bool yFlip = false); public: @@ -108,6 +171,9 @@ public: const gfx::IntSize& destSize, GLenum srcTarget = LOCAL_GL_TEXTURE_2D, GLenum destTarget = LOCAL_GL_TEXTURE_2D); + bool BlitImageToTexture(layers::Image* srcImage, const gfx::IntSize& destSize, + GLuint destTex, GLenum destTarget, bool yFlip = false, GLuint xoffset = 0, + GLuint yoffset = 0, GLuint width = 0, GLuint height = 0); }; } diff --git a/gfx/gl/ScopedGLHelpers.cpp b/gfx/gl/ScopedGLHelpers.cpp index 5ba94cfab02e..ec7a29de4974 100644 --- a/gfx/gl/ScopedGLHelpers.cpp +++ b/gfx/gl/ScopedGLHelpers.cpp @@ -411,5 +411,75 @@ ScopedVertexAttribPointer::UnwrapImpl() mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, mBoundBuffer); } +ScopedGLDrawState::ScopedGLDrawState(GLContext* aGL) + : blend (aGL, LOCAL_GL_BLEND, false) + , cullFace (aGL, LOCAL_GL_CULL_FACE, false) + , depthTest (aGL, LOCAL_GL_DEPTH_TEST, false) + , dither (aGL, LOCAL_GL_DITHER, false) + , polyOffsFill(aGL, LOCAL_GL_POLYGON_OFFSET_FILL, false) + , sampleAToC (aGL, LOCAL_GL_SAMPLE_ALPHA_TO_COVERAGE, false) + , sampleCover (aGL, LOCAL_GL_SAMPLE_COVERAGE, false) + , scissor (aGL, LOCAL_GL_SCISSOR_TEST, false) + , stencil (aGL, LOCAL_GL_STENCIL_TEST, false) + , mGL(aGL) + , packAlign(4) +{ + mGL->GetUIntegerv(LOCAL_GL_UNPACK_ALIGNMENT, &packAlign); + mGL->GetUIntegerv(LOCAL_GL_CURRENT_PROGRAM, &boundProgram); + mGL->GetUIntegerv(LOCAL_GL_ARRAY_BUFFER_BINDING, &boundBuffer); + mGL->GetUIntegerv(LOCAL_GL_MAX_VERTEX_ATTRIBS, &maxAttrib); + attrib_enabled = new GLint[maxAttrib]; + + for (unsigned int i = 0; i < maxAttrib; i++) { + mGL->fGetVertexAttribiv(i, LOCAL_GL_VERTEX_ATTRIB_ARRAY_ENABLED, &attrib_enabled[i]); + mGL->fDisableVertexAttribArray(i); + } + // Only Attrib0's client side state affected + mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_SIZE, &attrib0_size); + mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_STRIDE, &attrib0_stride); + mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_TYPE, &attrib0_type); + mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_NORMALIZED, &attrib0_normalized); + mGL->fGetVertexAttribiv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, &attrib0_bufferBinding); + mGL->fGetVertexAttribPointerv(0, LOCAL_GL_VERTEX_ATTRIB_ARRAY_POINTER, &attrib0_pointer); + mGL->fGetBooleanv(LOCAL_GL_COLOR_WRITEMASK, colorMask); + mGL->fGetIntegerv(LOCAL_GL_VIEWPORT, viewport); + mGL->fGetIntegerv(LOCAL_GL_SCISSOR_BOX, scissorBox); +} + +ScopedGLDrawState::~ScopedGLDrawState() +{ + mGL->fScissor(scissorBox[0], scissorBox[1], + scissorBox[2], scissorBox[3]); + + mGL->fViewport(viewport[0], viewport[1], + viewport[2], viewport[3]); + + mGL->fColorMask(colorMask[0], + colorMask[1], + colorMask[2], + colorMask[3]); + + mGL->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT, packAlign); + + for (unsigned int i = 0; i < maxAttrib; i++) { + if (attrib_enabled[i]) + mGL->fEnableVertexAttribArray(i); + else + mGL->fDisableVertexAttribArray(i); + } + + + mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, attrib0_bufferBinding); + mGL->fVertexAttribPointer(0, + attrib0_size, + attrib0_type, + attrib0_normalized, + attrib0_stride, + attrib0_pointer); + + mGL->fBindBuffer(LOCAL_GL_ARRAY_BUFFER, boundBuffer); + + mGL->fUseProgram(boundProgram); +} } /* namespace gl */ } /* namespace mozilla */ diff --git a/gfx/gl/ScopedGLHelpers.h b/gfx/gl/ScopedGLHelpers.h index e7f4aa4f5e15..e55529f5236f 100644 --- a/gfx/gl/ScopedGLHelpers.h +++ b/gfx/gl/ScopedGLHelpers.h @@ -304,6 +304,38 @@ protected: void UnwrapImpl(); }; +struct ScopedGLDrawState { + ScopedGLDrawState(GLContext* gl); + ~ScopedGLDrawState(); + + GLuint boundProgram; + GLuint boundBuffer; + + ScopedGLState blend; + ScopedGLState cullFace; + ScopedGLState depthTest; + ScopedGLState dither; + ScopedGLState polyOffsFill; + ScopedGLState sampleAToC; + ScopedGLState sampleCover; + ScopedGLState scissor; + ScopedGLState stencil; + + GLuint maxAttrib; + ScopedDeleteArray attrib_enabled; + GLint attrib0_size; + GLint attrib0_stride; + GLint attrib0_type; + GLint attrib0_normalized; + GLint attrib0_bufferBinding; + void* attrib0_pointer; + + realGLboolean colorMask[4]; + GLint viewport[4]; + GLint scissorBox[4]; + GLContext* const mGL; + GLuint packAlign; +}; } /* namespace gl */ } /* namespace mozilla */ diff --git a/gfx/gl/moz.build b/gfx/gl/moz.build index 6332dcac9572..7bf76db2cb8d 100644 --- a/gfx/gl/moz.build +++ b/gfx/gl/moz.build @@ -29,6 +29,7 @@ EXPORTS += [ 'DecomposeIntoNoRepeatTriangles.h', 'ForceDiscreteGPUHelperCGL.h', 'GfxTexturesReporter.h', + 'GLBlitHelper.h', 'GLBlitTextureImageHelper.h', 'GLConsts.h', 'GLContext.h',