Bug 716439; Loading a mask layer to textures in DX10. r=Bas

gfx/layers/d3d10/ImageLayerD3D10.cpp

@@ -115,6 +115,72 @@ ImageLayerD3D10::GetLayer()
   return this;
 }
 
+/**
+ * Returns a shader resource view for a Cairo or remote image.
+ * Returns nsnull if unsuccessful.
+ * If successful, aHasAlpha will be true iff the resulting texture 
+ * has an alpha component.
+ */
+ID3D10ShaderResourceView*
+ImageLayerD3D10::GetImageSRView(Image* aImage, bool& aHasAlpha)
+{
+  NS_ASSERTION(aImage, "Null image.");
+
+  if (aImage->GetFormat() == Image::REMOTE_IMAGE_BITMAP) {
+    RemoteBitmapImage *remoteImage =
+      static_cast<RemoteBitmapImage*>(aImage);
+      
+    if (!aImage->GetBackendData(LayerManager::LAYERS_D3D10)) {
+      nsAutoPtr<TextureD3D10BackendData> dat = new TextureD3D10BackendData();
+      dat->mTexture = DataToTexture(device(), remoteImage->mData, remoteImage->mStride, remoteImage->mSize);
+
+      if (dat->mTexture) {
+        device()->CreateShaderResourceView(dat->mTexture, NULL, getter_AddRefs(dat->mSRView));
+        aImage->SetBackendData(LayerManager::LAYERS_D3D10, dat.forget());
+      }
+    }
+
+    aHasAlpha = remoteImage->mFormat == RemoteImageData::BGRA32;
+  } else if (aImage->GetFormat() == Image::CAIRO_SURFACE) {
+    CairoImage *cairoImage =
+      static_cast<CairoImage*>(aImage);
+
+    if (!cairoImage->mSurface) {
+      return nsnull;
+    }
+
+    if (!aImage->GetBackendData(LayerManager::LAYERS_D3D10)) {
+      nsAutoPtr<TextureD3D10BackendData> dat = new TextureD3D10BackendData();
+      dat->mTexture = SurfaceToTexture(device(), cairoImage->mSurface, cairoImage->mSize);
+
+      if (dat->mTexture) {
+        device()->CreateShaderResourceView(dat->mTexture, NULL, getter_AddRefs(dat->mSRView));
+        aImage->SetBackendData(LayerManager::LAYERS_D3D10, dat.forget());
+      }
+    }
+
+    aHasAlpha = cairoImage->mSurface->GetContentType() == gfxASurface::CONTENT_COLOR_ALPHA;
+  } else {
+    NS_WARNING("Incorrect image type.");
+    return nsnull;
+  }
+
+  TextureD3D10BackendData *data =
+    static_cast<TextureD3D10BackendData*>(aImage->GetBackendData(LayerManager::LAYERS_D3D10));
+
+  if (!data) {
+    return nsnull;
+  }
+
+  nsRefPtr<ID3D10Device> dev;
+  data->mTexture->GetDevice(getter_AddRefs(dev));
+  if (dev != device()) {
+    return nsnull;
+  }
+
+  return data->mSRView;
+}
+
 void
 ImageLayerD3D10::RenderLayer()
 {
@@ -140,52 +206,8 @@ ImageLayerD3D10::RenderLayer()
   {
     bool hasAlpha = false;
 
-    if (image->GetFormat() == Image::REMOTE_IMAGE_BITMAP) {
-      RemoteBitmapImage *remoteImage =
-        static_cast<RemoteBitmapImage*>(image);
-      
-      if (!image->GetBackendData(LayerManager::LAYERS_D3D10)) {
-        nsAutoPtr<TextureD3D10BackendData> dat = new TextureD3D10BackendData();
-        dat->mTexture = DataToTexture(device(), remoteImage->mData, remoteImage->mStride, remoteImage->mSize);
-
-        if (dat->mTexture) {
-          device()->CreateShaderResourceView(dat->mTexture, NULL, getter_AddRefs(dat->mSRView));
-          image->SetBackendData(LayerManager::LAYERS_D3D10, dat.forget());
-        }
-      }
-
-      hasAlpha = remoteImage->mFormat == RemoteImageData::BGRA32;
-    } else {
-      CairoImage *cairoImage =
-        static_cast<CairoImage*>(image);
-
-      if (!cairoImage->mSurface) {
-        return;
-      }
-
-      if (!image->GetBackendData(LayerManager::LAYERS_D3D10)) {
-        nsAutoPtr<TextureD3D10BackendData> dat = new TextureD3D10BackendData();
-        dat->mTexture = SurfaceToTexture(device(), cairoImage->mSurface, cairoImage->mSize);
-
-        if (dat->mTexture) {
-          device()->CreateShaderResourceView(dat->mTexture, NULL, getter_AddRefs(dat->mSRView));
-          image->SetBackendData(LayerManager::LAYERS_D3D10, dat.forget());
-        }
-      }
-
-      hasAlpha = cairoImage->mSurface->GetContentType() == gfxASurface::CONTENT_COLOR_ALPHA;
-    }
-
-    TextureD3D10BackendData *data =
-      static_cast<TextureD3D10BackendData*>(image->GetBackendData(LayerManager::LAYERS_D3D10));
-
-    if (!data) {
-      return;
-    }
-
-    nsRefPtr<ID3D10Device> dev;
-    data->mTexture->GetDevice(getter_AddRefs(dev));
-    if (dev != device()) {
+    nsRefPtr<ID3D10ShaderResourceView> srView = GetImageSRView(image, hasAlpha);
+    if (!srView) {
       return;
     }
     
@@ -203,7 +225,7 @@ ImageLayerD3D10::RenderLayer()
       }
     }
 
-    effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(data->mSRView);
+    effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView);
 
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
@@ -358,5 +380,29 @@ void ImageLayerD3D10::AllocateTexturesYCbCr(PlanarYCbCrImage *aImage)
   aImage->SetBackendData(LayerManager::LAYERS_D3D10, backendData.forget());
 }
 
+already_AddRefed<ID3D10ShaderResourceView>
+ImageLayerD3D10::GetAsTexture(gfxIntSize* aSize)
+{
+  if (!GetContainer()) {
+    return nsnull;
+  }
+
+  AutoLockImage autoLock(GetContainer());
+
+  Image *image = autoLock.GetImage();
+  if (!image) {
+    return nsnull;
+  }
+
+  if (image->GetFormat() != Image::CAIRO_SURFACE) {
+    return nsnull;
+  }
+  
+  *aSize = image->GetSize();
+  bool dontCare;
+  nsRefPtr<ID3D10ShaderResourceView> result = GetImageSRView(image, dontCare);
+  return result.forget();
+}
+
 } /* layers */
 } /* mozilla */

gfx/layers/d3d10/ImageLayerD3D10.h

@@ -62,6 +62,11 @@ public:
   virtual void RenderLayer();
 
   void AllocateTexturesYCbCr(PlanarYCbCrImage *aImage);
+
+  virtual already_AddRefed<ID3D10ShaderResourceView> GetAsTexture(gfxIntSize* aSize);
+
+private:
+ ID3D10ShaderResourceView* GetImageSRView(Image* aImage, bool& aHasAlpha);
 };
 
 struct PlanarYCbCrD3D10BackendData : public ImageBackendData

gfx/layers/d3d10/LayerManagerD3D10.cpp

@@ -842,6 +842,37 @@ LayerD3D10::LayerD3D10(LayerManagerD3D10 *aManager)
 {
 }
 
+bool LayerD3D10::LoadMaskTexture()
+{
+  if (Layer* maskLayer = GetLayer()->GetMaskLayer()) {
+    gfxIntSize size;
+    nsRefPtr<ID3D10ShaderResourceView> maskSRV =
+      static_cast<LayerD3D10*>(maskLayer->ImplData())->GetAsTexture(&size);
+  
+    if (!maskSRV) {
+      return false;
+    }
+
+    gfxMatrix maskTransform;
+    bool maskIs2D = maskLayer->GetTransform().CanDraw2D(&maskTransform);
+    NS_ASSERTION(maskIs2D, "How did we end up with a 3D transform here?!");
+    gfxRect bounds = gfxRect(maskTransform.GetTranslation(), size);
+
+    effect()->GetVariableByName("vMaskQuad")->AsVector()->SetFloatVector(
+      ShaderConstantRectD3D10(
+        (float)bounds.x,
+        (float)bounds.y,
+        (float)bounds.width,
+        (float)bounds.height)
+      );
+
+    effect()->GetVariableByName("tMask")->AsShaderResource()->SetResource(maskSRV);
+    return true;
+  }
+
+  return false; 
+}
+
 WindowLayer::WindowLayer(LayerManagerD3D10* aManager)
   : ThebesLayer(aManager, nsnull)
 {

gfx/layers/d3d10/LayerManagerD3D10.h

@@ -291,6 +291,20 @@ public:
    */
   Nv3DVUtils *GetNv3DVUtils()  { return mD3DManager->GetNv3DVUtils(); }
 
+  /*
+   * Returns a shader resource view of a texture containing the contents of this
+   * layer. Will try to return an existing texture if possible, or a temporary
+   * one if not. It is the callee's responsibility to release the shader
+   * resource view. Will return null if a texture could not be constructed.
+   * The texture will not be transformed, i.e., it will be in the same coord
+   * space as this.
+   * Any layer that can be used as a mask layer should override this method.
+   * If aSize is non-null, it will contain the size of the texture.
+   */
+  virtual already_AddRefed<ID3D10ShaderResourceView> GetAsTexture(gfxIntSize* aSize)
+  {
+    return nsnull;
+  }
 
   void SetEffectTransformAndOpacity()
   {
@@ -302,6 +316,14 @@ public:
   }
 
 protected:
+  /*
+   * Finds a texture for this layer's mask layer (if it has one) and sets it
+   * as an input to the shaders.
+   * Returns true if a texture is loaded, false if there was no mask layer, or
+   * a texture for the mask layer could not be loaded.
+   */
+  bool LoadMaskTexture();
+
   LayerManagerD3D10 *mD3DManager;
 };