Bug 648484, part 3 and part 3.1: Allow passing a "backend hint" to GetLayerManager() to request a non-default layer manager backend. Deal with failure to hook up shadow layers. sr=roc

dom/ipc/TabChild.cpp

@@ -460,11 +460,13 @@ TabChild::DestroyWindow()
 void
 TabChild::ActorDestroy(ActorDestroyReason why)
 {
-  // The messageManager relays messages via the TabChild which
-  // no longer exists.
-  static_cast<nsFrameMessageManager*>
-    (mTabChildGlobal->mMessageManager.get())->Disconnect();
-  mTabChildGlobal->mMessageManager = nsnull;
+  if (mTabChildGlobal) {
+    // The messageManager relays messages via the TabChild which
+    // no longer exists.
+    static_cast<nsFrameMessageManager*>
+      (mTabChildGlobal->mMessageManager.get())->Disconnect();
+    mTabChildGlobal->mMessageManager = nsnull;
+  }
 }
 
 TabChild::~TabChild()
@@ -477,11 +479,13 @@ TabChild::~TabChild()
       DestroyCx();
     }
     
-    nsEventListenerManager* elm = mTabChildGlobal->GetListenerManager(PR_FALSE);
-    if (elm) {
-      elm->Disconnect();
+    if (mTabChildGlobal) {
+      nsEventListenerManager* elm = mTabChildGlobal->GetListenerManager(PR_FALSE);
+      if (elm) {
+        elm->Disconnect();
+      }
+      mTabChildGlobal->mTabChild = nsnull;
     }
-    mTabChildGlobal->mTabChild = nsnull;
 }
 
 bool
@@ -496,7 +500,7 @@ TabChild::RecvLoadURL(const nsCString& uri)
         NS_WARNING("mWebNav->LoadURI failed. Eating exception, what else can I do?");
     }
 
-    return NS_SUCCEEDED(rv);
+    return true;
 }
 
 bool
@@ -511,7 +515,11 @@ TabChild::RecvShow(const nsIntSize& size)
     }
 
     if (!InitWidget(size)) {
-        return false;
+        // We can fail to initialize our widget if the <browser
+        // remote> has already been destroyed, and we couldn't hook
+        // into the parent-process's layer system.  That's not a fatal
+        // error.
+        return true;
     }
 
     baseWindow->InitWindow(0, mWidget,
@@ -537,6 +545,9 @@ bool
 TabChild::RecvMove(const nsIntSize& size)
 {
     printf("[TabChild] RESIZE to (w,h)= (%ud, %ud)\n", size.width, size.height);
+    if (!mRemoteFrame) {
+        return true;
+    }
 
     mWidget->Resize(0, 0, size.width, size.height,
                     PR_TRUE);
@@ -775,7 +786,9 @@ bool
 TabChild::RecvLoadRemoteScript(const nsString& aURL)
 {
   if (!mCx && !InitTabChildGlobal())
-    return false;
+    // This can happen if we're half-destroyed.  It's not a fatal
+    // error.
+    return true;
 
   LoadFrameScriptInternal(aURL);
   return true;
@@ -825,10 +838,12 @@ public:
 bool
 TabChild::RecvDestroy()
 {
-  // Let the frame scripts know the child is being closed
-  nsContentUtils::AddScriptRunner(
-    new UnloadScriptEvent(this, mTabChildGlobal)
-  );
+  if (mTabChildGlobal) {
+    // Let the frame scripts know the child is being closed
+    nsContentUtils::AddScriptRunner(
+      new UnloadScriptEvent(this, mTabChildGlobal)
+    );
+  }
 
   // XXX what other code in ~TabChild() should we be running here?
   DestroyWindow();
@@ -946,7 +961,8 @@ TabChild::InitWidget(const nsIntSize& size)
 
     NS_ABORT_IF_FALSE(0 == remoteFrame->ManagedPLayersChild().Length(),
                       "shouldn't have a shadow manager yet");
-    PLayersChild* shadowManager = remoteFrame->SendPLayersConstructor();
+    LayerManager::LayersBackend be;
+    PLayersChild* shadowManager = remoteFrame->SendPLayersConstructor(&be);
     if (!shadowManager) {
       NS_WARNING("failed to construct LayersChild");
       // This results in |remoteFrame| being deleted.
@@ -954,14 +970,11 @@ TabChild::InitWidget(const nsIntSize& size)
       return false;
     }
 
-    LayerManager* lm = mWidget->GetLayerManager();
-    NS_ABORT_IF_FALSE(LayerManager::LAYERS_BASIC == lm->GetBackendType(),
-                      "content processes should only be using BasicLayers");
-
-    BasicShadowLayerManager* bslm = static_cast<BasicShadowLayerManager*>(lm);
-    NS_ABORT_IF_FALSE(!bslm->HasShadowManager(),
-                      "PuppetWidget shouldn't have shadow manager yet");
-    bslm->SetShadowManager(shadowManager);
+    ShadowLayerForwarder* lf =
+        mWidget->GetLayerManager(shadowManager, be)->AsShadowForwarder();
+    NS_ABORT_IF_FALSE(lf && lf->HasShadowManager(),
+                      "PuppetWidget should have shadow manager");
+    lf->SetParentBackendType(be);
 
     mRemoteFrame = remoteFrame;
     return true;

dom/ipc/TabParent.cpp

@@ -744,8 +744,7 @@ PRenderFrameParent*
 TabParent::AllocPRenderFrame()
 {
   nsRefPtr<nsFrameLoader> frameLoader = GetFrameLoader();
-  NS_WARN_IF_FALSE(frameLoader, "'message sent to unknown actor ID' coming up");
-  return frameLoader ? new RenderFrameParent(frameLoader) : nsnull;
+  return new RenderFrameParent(frameLoader);
 }
 
 bool

gfx/layers/ipc/PLayers.ipdl

@@ -54,7 +54,6 @@ using mozilla::GraphicsFilterType;
 using mozilla::layers::FrameMetrics;
 using mozilla::layers::SurfaceDescriptorX11;
 using mozilla::null_t;
-using mozilla::LayersBackend;
 
 /**
  * The layers protocol is spoken between thread contexts that manage
@@ -256,9 +255,6 @@ parent:
   sync Update(Edit[] cset)
     returns (EditReply[] reply);
 
-  sync GetParentType()
-    returns (LayersBackend backend);
-
   async __delete__();
 };
 

gfx/layers/ipc/ShadowLayers.cpp

@@ -379,18 +379,6 @@ ShadowLayerForwarder::EndTransaction(InfallibleTArray<EditReply>* aReplies)
   return PR_TRUE;
 }
 
-LayersBackend
-ShadowLayerForwarder::GetParentBackendType()
-{
-  if (mParentBackend == LayerManager::LAYERS_NONE) {
-    LayersBackend backend;
-    if (mShadowManager->SendGetParentType(&backend)) {
-      mParentBackend = backend;
-    }
-  }
-  return mParentBackend;
-}
-
 static gfxASurface::gfxImageFormat
 OptimalFormatFor(gfxASurface::gfxContentType aContent)
 {

gfx/layers/ipc/ShadowLayers.h

@@ -245,6 +245,11 @@ public:
     mShadowManager = aShadowManager;
   }
 
+  void SetParentBackendType(LayersBackend aBackendType)
+  {
+    mParentBackend = aBackendType;
+  }
+
   /**
    * True if this is forwarding to a ShadowLayerManager.
    */
@@ -325,7 +330,10 @@ public:
    */
   PLayerChild* ConstructShadowFor(ShadowableLayer* aLayer);
 
-  LayersBackend GetParentBackendType();
+  LayersBackend GetParentBackendType()
+  {
+    return mParentBackend;
+  }
 
   /*
    * No need to use double buffer in system memory with GPU rendering,

ipc/glue/IPCMessageUtils.h

@@ -515,12 +515,12 @@ struct ParamTraits<mozilla::LayersBackend>
 
   static void Write(Message* msg, const paramType& param)
   {
-    if (LayerManager::LAYERS_NONE < param &&
+    if (LayerManager::LAYERS_NONE <= param &&
         param < LayerManager::LAYERS_LAST) {
       WriteParam(msg, int32(param));
       return;
     }
-    NS_RUNTIMEABORT("surface type not reached");
+    NS_RUNTIMEABORT("backend type not reached");
   }
 
   static bool Read(const Message* msg, void** iter, paramType* result)
@@ -529,7 +529,7 @@ struct ParamTraits<mozilla::LayersBackend>
     if (!ReadParam(msg, iter, &type))
       return false;
 
-    if (LayerManager::LAYERS_NONE < type &&
+    if (LayerManager::LAYERS_NONE <= type &&
         type < LayerManager::LAYERS_LAST) {
       *result = paramType(type);
       return true;

layout/ipc/PRenderFrame.ipdl

@@ -41,6 +41,8 @@
 include protocol PBrowser;
 include protocol PLayers;
 
+using mozilla::LayersBackend;
+
 namespace mozilla {
 namespace layout {
 
@@ -59,7 +61,9 @@ sync protocol PRenderFrame
     manages PLayers;
 
 parent:
-    async PLayers();
+    sync PLayers()
+      returns (LayersBackend backend);
+
     async __delete__();
 
 state EMPTY:

layout/ipc/RenderFrameChild.cpp

@@ -66,7 +66,7 @@ RenderFrameChild::Destroy()
 }
 
 PLayersChild*
-RenderFrameChild::AllocPLayers()
+RenderFrameChild::AllocPLayers(LayerManager::LayersBackend* aBackendType)
 {
   return new ShadowLayersChild();
 }

layout/ipc/RenderFrameChild.h

@@ -56,7 +56,7 @@ public:
 
 protected:
   NS_OVERRIDE
-  virtual PLayersChild* AllocPLayers();
+  virtual PLayersChild* AllocPLayers(LayerManager::LayersBackend* aBackendType);
   NS_OVERRIDE
   virtual bool DeallocPLayers(PLayersChild* aLayers);
 };

layout/ipc/RenderFrameParent.cpp

@@ -565,10 +565,11 @@ BuildBackgroundPatternFor(ContainerLayer* aContainer,
 RenderFrameParent::RenderFrameParent(nsFrameLoader* aFrameLoader)
   : mFrameLoader(aFrameLoader)
 {
-  NS_ABORT_IF_FALSE(aFrameLoader, "Need a frameloader here");
-  mContentViews[FrameMetrics::ROOT_SCROLL_ID] =
-    new nsContentView(aFrameLoader->GetOwnerContent(),
-                      FrameMetrics::ROOT_SCROLL_ID);
+  if (aFrameLoader) {
+    mContentViews[FrameMetrics::ROOT_SCROLL_ID] =
+      new nsContentView(aFrameLoader->GetOwnerContent(),
+                        FrameMetrics::ROOT_SCROLL_ID);
+  }
 }
 
 RenderFrameParent::~RenderFrameParent()
@@ -701,7 +702,7 @@ RenderFrameParent::OwnerContentChanged(nsIContent* aContent)
 void
 RenderFrameParent::ActorDestroy(ActorDestroyReason why)
 {
-  if (mFrameLoader->GetCurrentRemoteFrame() == this) {
+  if (mFrameLoader && mFrameLoader->GetCurrentRemoteFrame() == this) {
     // XXX this might cause some weird issues ... we'll just not
     // redraw the part of the window covered by this until the "next"
     // remote frame has a layer-tree transaction.  For
@@ -714,10 +715,15 @@ RenderFrameParent::ActorDestroy(ActorDestroyReason why)
 }
 
 PLayersParent*
-RenderFrameParent::AllocPLayers()
+RenderFrameParent::AllocPLayers(LayerManager::LayersBackend* aBackendType)
 {
+  if (!mFrameLoader) {
+    *aBackendType = LayerManager::LAYERS_NONE;
+    return nsnull;
+  }
+
   LayerManager* lm = GetLayerManager();
-  switch (lm->GetBackendType()) {
+  switch (*aBackendType = lm->GetBackendType()) {
   case LayerManager::LAYERS_BASIC: {
     BasicShadowLayerManager* bslm = static_cast<BasicShadowLayerManager*>(lm);
     return new ShadowLayersParent(bslm);
@@ -734,6 +740,7 @@ RenderFrameParent::AllocPLayers()
 #endif //MOZ_ENABLE_D3D9_LAYER
   default: {
     NS_WARNING("shadow layers no sprechen D3D backend yet");
+    *aBackendType = LayerManager::LAYERS_NONE;
     return nsnull;
   }
   }

layout/ipc/RenderFrameParent.h

@@ -99,7 +99,7 @@ public:
 protected:
   NS_OVERRIDE void ActorDestroy(ActorDestroyReason why);
 
-  NS_OVERRIDE virtual PLayersParent* AllocPLayers();
+  NS_OVERRIDE virtual PLayersParent* AllocPLayers(LayerManager::LayersBackend* aBackendType);
   NS_OVERRIDE virtual bool DeallocPLayers(PLayersParent* aLayers);
 
 private:

widget/public/nsIWidget.h

@@ -47,6 +47,7 @@
 #include "nsStringGlue.h"
 
 #include "prthread.h"
+#include "Layers.h"
 #include "nsEvent.h"
 #include "nsCOMPtr.h"
 #include "nsITheme.h"
@@ -71,12 +72,12 @@ class   nsIContent;
 class   ViewWrapper;
 
 namespace mozilla {
-namespace layers {
-class LayerManager;
-}
 namespace dom {
 class PBrowserChild;
 }
+namespace layers {
+class PLayersChild;
+}
 }
 
 /**
@@ -117,8 +118,8 @@ typedef nsEventStatus (* EVENT_CALLBACK)(nsGUIEvent *event);
 #endif
 
 #define NS_IWIDGET_IID \
-  { 0xac809e35, 0x632c, 0x448d, \
-    { 0x9e, 0x34, 0x11, 0x62, 0x32, 0x60, 0x5e, 0xe6 } }
+  { 0xf43254ce, 0xd315, 0x458b, \
+    { 0xba, 0x72, 0xa8, 0xdf, 0x21, 0xcf, 0xa7, 0x2a } }
 
 /*
  * Window shadow styles
@@ -274,6 +275,8 @@ class nsIWidget : public nsISupports {
 
   public:
     typedef mozilla::layers::LayerManager LayerManager;
+    typedef LayerManager::LayersBackend LayersBackend;
+    typedef mozilla::layers::PLayersChild PLayersChild;
 
     // Used in UpdateThemeGeometries.
     struct ThemeGeometry {
@@ -894,6 +897,12 @@ class nsIWidget : public nsISupports {
 
     virtual nsIToolkit* GetToolkit() = 0;    
 
+    enum LayerManagerPersistence
+    {
+      LAYER_MANAGER_CURRENT = 0,
+      LAYER_MANAGER_PERSISTENT
+    };
+
     /**
      * Return the widget's LayerManager. The layer tree for that
      * LayerManager is what gets rendered to the widget.
@@ -903,17 +912,25 @@ class nsIWidget : public nsISupports {
      */
     inline LayerManager* GetLayerManager(bool* aAllowRetaining = nsnull)
     {
-        return GetLayerManager(LAYER_MANAGER_CURRENT, aAllowRetaining);
+        return GetLayerManager(nsnull, LayerManager::LAYERS_NONE,
+                               LAYER_MANAGER_CURRENT, aAllowRetaining);
     }
 
-
-    enum LayerManagerPersistence
+    inline LayerManager* GetLayerManager(LayerManagerPersistence aPersistence,
+                                         bool* aAllowRetaining = nsnull)
     {
-      LAYER_MANAGER_CURRENT = 0,
-      LAYER_MANAGER_PERSISTENT
-    };
+        return GetLayerManager(nsnull, LayerManager::LAYERS_NONE,
+                               aPersistence, aAllowRetaining);
+    }
 
-    virtual LayerManager *GetLayerManager(LayerManagerPersistence aPersistence,
+    /**
+     * Like GetLayerManager(), but prefers creating a layer manager of
+     * type |aBackendHint| instead of what would normally be created.
+     * LAYERS_NONE means "no hint".
+     */
+    virtual LayerManager* GetLayerManager(PLayersChild* aShadowManager,
+                                          LayersBackend aBackendHint,
+                                          LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT,
                                           bool* aAllowRetaining = nsnull) = 0;
 
     /**

widget/src/windows/nsWindow.cpp

@@ -3173,8 +3173,11 @@ GetLayerManagerPrefs(LayerManagerPrefs* aManagerPrefs)
     aManagerPrefs->mDisableAcceleration || safeMode;
 }
 
-mozilla::layers::LayerManager*
-nsWindow::GetLayerManager(LayerManagerPersistence aPersistence, bool* aAllowRetaining)
+LayerManager*
+nsWindow::GetLayerManager(PLayersChild* aShadowManager,
+                          LayersBackend aBackendHint,
+                          LayerManagerPersistence aPersistence,
+                          bool* aAllowRetaining)
 {
   if (aAllowRetaining) {
     *aAllowRetaining = true;

widget/src/windows/nsWindow.h

@@ -159,7 +159,10 @@ public:
                                               PRBool aDoCapture, PRBool aConsumeRollupEvent);
   NS_IMETHOD              GetAttention(PRInt32 aCycleCount);
   virtual PRBool          HasPendingInputEvent();
-  virtual LayerManager*   GetLayerManager(LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT, bool* aAllowRetaining = nsnull);
+  virtual LayerManager*   GetLayerManager(PLayersChild* aShadowManager = nsnull,
+                                          LayersBackend aBackendHint = LayerManager::LAYERS_NONE,
+                                          LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT,
+                                          bool* aAllowRetaining = nsnull);
   gfxASurface             *GetThebesSurface();
   NS_IMETHOD              OnDefaultButtonLoaded(const nsIntRect &aButtonRect);
   NS_IMETHOD              OverrideSystemMouseScrollSpeed(PRInt32 aOriginalDelta, PRBool aIsHorizontal, PRInt32 &aOverriddenDelta);

widget/src/xpwidgets/PuppetWidget.cpp

@@ -330,10 +330,14 @@ PuppetWidget::DispatchEvent(nsGUIEvent* event, nsEventStatus& aStatus)
 }
 
 LayerManager*
-PuppetWidget::GetLayerManager(LayerManagerPersistence, bool* aAllowRetaining)
+PuppetWidget::GetLayerManager(PLayersChild* aShadowManager,
+                              LayersBackend aBackendHint,
+                              LayerManagerPersistence aPersistence,
+                              bool* aAllowRetaining)
 {
   if (!mLayerManager) {
     mLayerManager = new BasicShadowLayerManager(this);
+    mLayerManager->AsShadowForwarder()->SetShadowManager(aShadowManager);
   }
   if (aAllowRetaining) {
     *aAllowRetaining = true;

widget/src/xpwidgets/PuppetWidget.h

@@ -164,8 +164,11 @@ public:
   //
 
 //NS_IMETHOD              CaptureMouse(PRBool aCapture);
-  virtual LayerManager*     GetLayerManager(LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT,
-                                            bool* aAllowRetaining = nsnull);
+  virtual LayerManager*
+  GetLayerManager(PLayersChild* aShadowManager = nsnull,
+                  LayersBackend aBackendHint = LayerManager::LAYERS_NONE,
+                  LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT,
+                  bool* aAllowRetaining = nsnull);
 //  virtual nsDeviceContext* GetDeviceContext();
   virtual gfxASurface*      GetThebesSurface();
 

widget/src/xpwidgets/nsBaseWidget.cpp

@@ -852,7 +852,9 @@ nsBaseWidget::GetShouldAccelerate()
   return mUseAcceleratedRendering;
 }
 
-LayerManager* nsBaseWidget::GetLayerManager(LayerManagerPersistence,
+LayerManager* nsBaseWidget::GetLayerManager(PLayersChild* aShadowManager,
+                                            LayersBackend aBackendHint,
+                                            LayerManagerPersistence aPersistence,
                                             bool* aAllowRetaining)
 {
   if (!mLayerManager) {
@@ -860,8 +862,7 @@ LayerManager* nsBaseWidget::GetLayerManager(LayerManagerPersistence,
     mUseAcceleratedRendering = GetShouldAccelerate();
 
     if (mUseAcceleratedRendering) {
-      nsRefPtr<LayerManagerOGL> layerManager =
-        new mozilla::layers::LayerManagerOGL(this);
+      nsRefPtr<LayerManagerOGL> layerManager = new LayerManagerOGL(this);
       /**
        * XXX - On several OSes initialization is expected to fail for now.
        * If we'd get a none-basic layer manager they'd crash. This is ok though

widget/src/xpwidgets/nsBaseWidget.h

@@ -115,7 +115,9 @@ public:
   NS_IMETHOD              MakeFullScreen(PRBool aFullScreen);
   virtual nsDeviceContext* GetDeviceContext();
   virtual nsIToolkit*     GetToolkit();
-  virtual LayerManager*   GetLayerManager(LayerManagerPersistence aPersistence,
+  virtual LayerManager*   GetLayerManager(PLayersChild* aShadowManager,
+                                          LayersBackend aBackendHint,
+                                          LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT,
                                           bool* aAllowRetaining = nsnull);
   using nsIWidget::GetLayerManager;