Bug 1350408 - Gracefully handle failures in WebRenderLayerManager::Initialize to allow reinit. r=kats

dom/ipc/TabChild.cpp

@@ -3018,15 +3018,16 @@ TabChild::ReinitRendering()
   SendEnsureLayersConnected(&options);
   mCompositorOptions = Some(options);
 
+  bool success = false;
   RefPtr<CompositorBridgeChild> cb = CompositorBridgeChild::Get();
   if (gfxVars::UseWebRender()) {
-    RefPtr<LayerManager> lm = mPuppetWidget->RecreateLayerManager(nullptr);
-    MOZ_ASSERT(lm->AsWebRenderLayerManager());
-    lm->AsWebRenderLayerManager()->Initialize(cb,
-                                              wr::AsPipelineId(mLayersId),
-                                              &mTextureFactoryIdentifier);
+    success = mPuppetWidget->RecreateLayerManager([&] (LayerManager* aLayerManager) -> bool {
+      MOZ_ASSERT(aLayerManager->AsWebRenderLayerManager());
+      return aLayerManager->AsWebRenderLayerManager()->Initialize(cb,
+                                                                  wr::AsPipelineId(mLayersId),
+                                                                  &mTextureFactoryIdentifier);
+    });
   } else {
-    bool success = false;
     nsTArray<LayersBackend> ignored;
     PLayerTransactionChild* shadowManager = cb->SendPLayerTransactionConstructor(ignored, LayersId());
     if (shadowManager &&
@@ -3040,9 +3041,17 @@ TabChild::ReinitRendering()
       return;
     }
 
-    RefPtr<LayerManager> lm = mPuppetWidget->RecreateLayerManager(shadowManager);
-    ShadowLayerForwarder* lf = lm->AsShadowForwarder();
-    lf->IdentifyTextureHost(mTextureFactoryIdentifier);
+    success = mPuppetWidget->RecreateLayerManager([&] (LayerManager* aLayerManager) -> bool {
+      ShadowLayerForwarder* lf = aLayerManager->AsShadowForwarder();
+      lf->SetShadowManager(shadowManager);
+      lf->IdentifyTextureHost(mTextureFactoryIdentifier);
+      return true;
+    });
+  }
+
+  if (!success) {
+    NS_WARNING("failed to recreate layer manager");
+    return;
   }
 
   ImageBridgeChild::IdentifyCompositorTextureHost(mTextureFactoryIdentifier);

gfx/layers/wr/WebRenderLayerManager.cpp

@@ -47,7 +47,7 @@ WebRenderLayerManager::AsKnowsCompositor()
   return mWrChild;
 }
 
-void
+bool
 WebRenderLayerManager::Initialize(PCompositorBridgeChild* aCBChild,
                                   wr::PipelineId aLayersId,
                                   TextureFactoryIdentifier* aTextureFactoryIdentifier)
@@ -62,12 +62,21 @@ WebRenderLayerManager::Initialize(PCompositorBridgeChild* aCBChild,
                                                                             size,
                                                                             &textureFactoryIdentifier,
                                                                             &id_namespace);
-  MOZ_ASSERT(bridge);
+  if (!bridge) {
+    // This should only fail if we attempt to access a layer we don't have
+    // permission for, or more likely, the GPU process crashed again during
+    // reinitialization. We can expect to be notified again to reinitialize
+    // (which may or may not be using WebRender).
+    gfxCriticalNote << "Failed to create WebRenderBridgeChild.";
+    return false;
+  }
+
   mWrChild = static_cast<WebRenderBridgeChild*>(bridge);
   WrBridge()->SendCreate(size.ToUnknownSize());
   WrBridge()->IdentifyTextureHost(textureFactoryIdentifier);
   WrBridge()->SetNamespace(id_namespace);
   *aTextureFactoryIdentifier = textureFactoryIdentifier;
+  return true;
 }
 
 void
@@ -86,9 +95,12 @@ WebRenderLayerManager::DoDestroy(bool aIsSync)
   mWidget->CleanupWebRenderWindowOverlay(WrBridge());
 
   LayerManager::Destroy();
-  DiscardImages();
-  DiscardCompositorAnimations();
-  WrBridge()->Destroy(aIsSync);
+
+  if (WrBridge()) {
+    DiscardImages();
+    DiscardCompositorAnimations();
+    WrBridge()->Destroy(aIsSync);
+  }
 
   if (mTransactionIdAllocator) {
     // Make sure to notify the refresh driver just in case it's waiting on a
@@ -698,7 +710,9 @@ WebRenderLayerManager::Hold(Layer* aLayer)
 void
 WebRenderLayerManager::SetLayerObserverEpoch(uint64_t aLayerObserverEpoch)
 {
-  WrBridge()->SendSetLayerObserverEpoch(aLayerObserverEpoch);
+  if (WrBridge()->IPCOpen()) {
+    WrBridge()->SendSetLayerObserverEpoch(aLayerObserverEpoch);
+  }
 }
 
 void

gfx/layers/wr/WebRenderLayerManager.h

@@ -35,7 +35,7 @@ class WebRenderLayerManager final : public LayerManager
 
 public:
   explicit WebRenderLayerManager(nsIWidget* aWidget);
-  void Initialize(PCompositorBridgeChild* aCBChild, wr::PipelineId aLayersId, TextureFactoryIdentifier* aTextureFactoryIdentifier);
+  bool Initialize(PCompositorBridgeChild* aCBChild, wr::PipelineId aLayersId, TextureFactoryIdentifier* aTextureFactoryIdentifier);
 
   virtual void Destroy() override;
 

widget/PuppetWidget.cpp

@@ -592,24 +592,28 @@ PuppetWidget::GetLayerManager(PLayerTransactionChild* aShadowManager,
   return mLayerManager;
 }
 
-LayerManager*
-PuppetWidget::RecreateLayerManager(PLayerTransactionChild* aShadowManager)
+bool
+PuppetWidget::RecreateLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc)
 {
-  // Force the old LM to self destruct, otherwise if the reference dangles we
-  // could fail to revoke the most recent transaction.
-  DestroyLayerManager();
-
+  RefPtr<LayerManager> lm;
   MOZ_ASSERT(mTabChild);
   if (gfxVars::UseWebRender()) {
-    MOZ_ASSERT(!aShadowManager);
-    mLayerManager = new WebRenderLayerManager(this);
+    lm = new WebRenderLayerManager(this);
   } else {
-    mLayerManager = new ClientLayerManager(this);
+    lm = new ClientLayerManager(this);
   }
-  if (ShadowLayerForwarder* lf = mLayerManager->AsShadowForwarder()) {
-    lf->SetShadowManager(aShadowManager);
+
+  if (!aInitializeFunc(lm)) {
+    return false;
   }
-  return mLayerManager;
+
+  // Force the old LM to self destruct, otherwise if the reference dangles we
+  // could fail to revoke the most recent transaction. We only want to replace
+  // it if we successfully create its successor because a partially initialized
+  // layer manager is worse than a fully initialized but shutdown layer manager.
+  DestroyLayerManager();
+  mLayerManager = lm.forget();
+  return true;
 }
 
 nsresult

widget/PuppetWidget.h

@@ -182,8 +182,11 @@ public:
                   LayersBackend aBackendHint = mozilla::layers::LayersBackend::LAYERS_NONE,
                   LayerManagerPersistence aPersistence = LAYER_MANAGER_CURRENT) override;
 
-  // This is used after a compositor reset.
-  LayerManager* RecreateLayerManager(PLayerTransactionChild* aShadowManager);
+  // This is used after a compositor reset. The lambda aInitializeFunc is used
+  // to perform any caller-required initialization for the newly created layer
+  // manager; in the event of a failure, return false and it will destroy the
+  // new layer manager without changing the state of the widget.
+  bool RecreateLayerManager(const std::function<bool(LayerManager*)>& aInitializeFunc);
 
   virtual void SetInputContext(const InputContext& aContext,
                                const InputContextAction& aAction) override;