Make PLayerTransaction's constructor async. (bug 1350634, ipc_r=billm, r=mattwoodrow, r=kats)

PLayerTransaction's constructor was previously synchronous so we could
return a TextureFactoryIdentifier. This is quite reliably available
already in the case of opening a tab, due to RenderFrameParent knowing
which compositor it is attached to, so we can make the constructor
asynchronous.

In the top-level widget case, we add a new synchronous message to find
the TextureFactoryIdentifier.

dom/ipc/TabChild.cpp

@@ -35,6 +35,7 @@
 #include "mozilla/layers/IAPZCTreeManager.h"
 #include "mozilla/layers/ImageBridgeChild.h"
 #include "mozilla/layers/InputAPZContext.h"
+#include "mozilla/layers/PLayerTransactionChild.h"
 #include "mozilla/layers/ShadowLayers.h"
 #include "mozilla/layers/WebRenderLayerManager.h"
 #include "mozilla/layout/RenderFrameChild.h"
@@ -2713,11 +2714,9 @@ TabChild::InitRenderingState(const TextureFactoryIdentifier& aTextureFactoryIden
     if (lf) {
       nsTArray<LayersBackend> backends;
       backends.AppendElement(mTextureFactoryIdentifier.mParentBackend);
-      bool success;
       PLayerTransactionChild* shadowManager =
-          compositorChild->SendPLayerTransactionConstructor(backends,
-                                                            aLayersId, &mTextureFactoryIdentifier, &success);
-      if (shadowManager && success) {
+          compositorChild->SendPLayerTransactionConstructor(backends, aLayersId);
+      if (shadowManager) {
         lf->SetShadowManager(shadowManager);
         lf->IdentifyTextureHost(mTextureFactoryIdentifier);
         ImageBridgeChild::IdentifyCompositorTextureHost(mTextureFactoryIdentifier);
@@ -3100,20 +3099,20 @@ TabChild::ReinitRendering()
                                               wr::AsPipelineId(mLayersId),
                                               &mTextureFactoryIdentifier);
   } else {
-    bool success;
+    bool success = false;
     nsTArray<LayersBackend> ignored;
-    PLayerTransactionChild* shadowManager =
-      cb->SendPLayerTransactionConstructor(ignored, LayersId(), &mTextureFactoryIdentifier, &success);
+    PLayerTransactionChild* shadowManager = cb->SendPLayerTransactionConstructor(ignored, LayersId());
+    if (shadowManager &&
+        shadowManager->SendGetTextureFactoryIdentifier(&mTextureFactoryIdentifier) &&
+        mTextureFactoryIdentifier.mParentBackend != LayersBackend::LAYERS_NONE)
+    {
+      success = true;
+    }
     if (!success) {
       NS_WARNING("failed to re-allocate layer transaction");
       return;
     }
 
-    if (!shadowManager) {
-      NS_WARNING("failed to re-construct LayersChild");
-      return;
-    }
-
     RefPtr<LayerManager> lm = mPuppetWidget->RecreateLayerManager(shadowManager);
     ShadowLayerForwarder* lf = lm->AsShadowForwarder();
     lf->IdentifyTextureHost(mTextureFactoryIdentifier);

gfx/layers/ipc/CompositorBridgeChild.cpp

@@ -330,10 +330,7 @@ CompositorBridgeChild::CompositorIsInGPUProcess()
 }
 
 PLayerTransactionChild*
-CompositorBridgeChild::AllocPLayerTransactionChild(const nsTArray<LayersBackend>& aBackendHints,
-                                                   const uint64_t& aId,
-                                                   TextureFactoryIdentifier*,
-                                                   bool*)
+CompositorBridgeChild::AllocPLayerTransactionChild(const nsTArray<LayersBackend>& aBackendHints, const uint64_t& aId)
 {
   LayerTransactionChild* c = new LayerTransactionChild(aId);
   c->AddIPDLReference();

gfx/layers/ipc/CompositorBridgeChild.h

@@ -249,9 +249,7 @@ private:
 
   virtual PLayerTransactionChild*
     AllocPLayerTransactionChild(const nsTArray<LayersBackend>& aBackendHints,
-                                const uint64_t& aId,
-                                TextureFactoryIdentifier* aTextureFactoryIdentifier,
-                                bool* aSuccess) override;
+                                const uint64_t& aId) override;
 
   virtual bool DeallocPLayerTransactionChild(PLayerTransactionChild *aChild) override;
 

gfx/layers/ipc/CompositorBridgeParent.cpp

@@ -1443,9 +1443,7 @@ CompositorBridgeParent::NewCompositor(const nsTArray<LayersBackend>& aBackendHin
 
 PLayerTransactionParent*
 CompositorBridgeParent::AllocPLayerTransactionParent(const nsTArray<LayersBackend>& aBackendHints,
-                                                     const uint64_t& aId,
-                                                     TextureFactoryIdentifier* aTextureFactoryIdentifier,
-                                               bool *aSuccess)
+                                                     const uint64_t& aId)
 {
   MOZ_ASSERT(aId == 0);
 
@@ -1453,16 +1451,13 @@ CompositorBridgeParent::AllocPLayerTransactionParent(const nsTArray<LayersBacken
 
   if (!mLayerManager) {
     NS_WARNING("Failed to initialise Compositor");
-    *aSuccess = false;
     LayerTransactionParent* p = new LayerTransactionParent(nullptr, this, 0);
     p->AddIPDLReference();
     return p;
   }
 
   mCompositionManager = new AsyncCompositionManager(this, mLayerManager);
-  *aSuccess = true;
 
-  *aTextureFactoryIdentifier = mLayerManager->GetTextureFactoryIdentifier();
   LayerTransactionParent* p = new LayerTransactionParent(mLayerManager, this, 0);
   p->AddIPDLReference();
   return p;

gfx/layers/ipc/CompositorBridgeParent.h

@@ -511,9 +511,7 @@ protected:
 
   virtual PLayerTransactionParent*
     AllocPLayerTransactionParent(const nsTArray<LayersBackend>& aBackendHints,
-                                 const uint64_t& aId,
-                                 TextureFactoryIdentifier* aTextureFactoryIdentifier,
-                                 bool* aSuccess) override;
+                                 const uint64_t& aId) override;
   virtual bool DeallocPLayerTransactionParent(PLayerTransactionParent* aLayers) override;
   virtual void ScheduleTask(already_AddRefed<CancelableRunnable>, int);
 

gfx/layers/ipc/CrossProcessCompositorBridgeParent.cpp

@@ -62,9 +62,7 @@ CrossProcessCompositorBridgeParent::ActorDestroy(ActorDestroyReason aWhy)
 PLayerTransactionParent*
 CrossProcessCompositorBridgeParent::AllocPLayerTransactionParent(
   const nsTArray<LayersBackend>&,
-  const uint64_t& aId,
-  TextureFactoryIdentifier* aTextureFactoryIdentifier,
-  bool *aSuccess)
+  const uint64_t& aId)
 {
   MOZ_ASSERT(aId != 0);
 
@@ -85,8 +83,6 @@ CrossProcessCompositorBridgeParent::AllocPLayerTransactionParent(
   if (state && state->mLayerManager) {
     state->mCrossProcessParent = this;
     HostLayerManager* lm = state->mLayerManager;
-    *aTextureFactoryIdentifier = lm->GetCompositor()->GetTextureFactoryIdentifier();
-    *aSuccess = true;
     LayerTransactionParent* p = new LayerTransactionParent(lm, this, aId);
     p->AddIPDLReference();
     sIndirectLayerTrees[aId].mLayerTree = p;
@@ -97,7 +93,6 @@ CrossProcessCompositorBridgeParent::AllocPLayerTransactionParent(
   }
 
   NS_WARNING("Created child without a matching parent?");
-  *aSuccess = false;
   LayerTransactionParent* p = new LayerTransactionParent(nullptr, this, aId);
   p->AddIPDLReference();
   return p;

gfx/layers/ipc/CrossProcessCompositorBridgeParent.h

@@ -92,9 +92,7 @@ public:
 
   virtual PLayerTransactionParent*
     AllocPLayerTransactionParent(const nsTArray<LayersBackend>& aBackendHints,
-                                 const uint64_t& aId,
-                                 TextureFactoryIdentifier* aTextureFactoryIdentifier,
-                                 bool *aSuccess) override;
+                                 const uint64_t& aId) override;
 
   virtual bool DeallocPLayerTransactionParent(PLayerTransactionParent* aLayers) override;
 

gfx/layers/ipc/LayerTransactionParent.cpp

@@ -1053,5 +1053,17 @@ LayerTransactionParent::RecvRecordPaintTimes(const PaintTiming& aTiming)
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+LayerTransactionParent::RecvGetTextureFactoryIdentifier(TextureFactoryIdentifier* aIdentifier)
+{
+  if (!mLayerManager) {
+    // Default constructor sets mParentBackend to LAYERS_NONE.
+    return IPC_OK();
+  }
+
+  *aIdentifier = mLayerManager->GetTextureFactoryIdentifier();
+  return IPC_OK();
+}
+
 } // namespace layers
 } // namespace mozilla

gfx/layers/ipc/LayerTransactionParent.h

@@ -141,6 +141,7 @@ protected:
   virtual mozilla::ipc::IPCResult RecvSetConfirmedTargetAPZC(const uint64_t& aBlockId,
                                                              nsTArray<ScrollableLayerGuid>&& aTargets) override;
   virtual mozilla::ipc::IPCResult RecvRecordPaintTimes(const PaintTiming& aTiming) override;
+  virtual mozilla::ipc::IPCResult RecvGetTextureFactoryIdentifier(TextureFactoryIdentifier* aIdentifier) override;
 
   bool SetLayerAttributes(const OpSetLayerAttributes& aOp);
 

gfx/layers/ipc/PCompositorBridge.ipdl

@@ -222,8 +222,7 @@ parent:
   // layersBackendHints is an ordered list of preffered backends where
   // layersBackendHints[0] is the best backend. If any hints are LayersBackend::LAYERS_NONE
   // that hint is ignored.
-  sync PLayerTransaction(LayersBackend[] layersBackendHints, uint64_t id)
-    returns (TextureFactoryIdentifier textureFactoryIdentifier, bool success);
+  async PLayerTransaction(LayersBackend[] layersBackendHints, uint64_t id);
 
   // Notify the compositor that a region of the screen has been invalidated.
   async NotifyRegionInvalidated(nsIntRegion region);

gfx/layers/ipc/PLayerTransaction.ipdl

@@ -18,6 +18,7 @@ using struct mozilla::void_t from "ipc/IPCMessageUtils.h";
 using class mozilla::layers::APZTestData from "mozilla/layers/APZTestData.h";
 using mozilla::layers::FrameMetrics::ViewID from "FrameMetrics.h";
 using struct mozilla::layers::ScrollableLayerGuid from "FrameMetrics.h";
+using struct mozilla::layers::TextureFactoryIdentifier from "mozilla/layers/CompositorTypes.h";
 using mozilla::layers::LayersBackend from "mozilla/layers/LayersTypes.h";
 using mozilla::layers::LayerHandle from "mozilla/layers/LayersTypes.h";
 using mozilla::layers::CompositableHandle from "mozilla/layers/LayersTypes.h";
@@ -118,6 +119,9 @@ parent:
   // Query a named property from the last frame
   sync RequestProperty(nsString property) returns (float value);
 
+  // Return the TextureFactoryIdentifier for this compositor.
+  sync GetTextureFactoryIdentifier() returns (TextureFactoryIdentifier aIdentifier);
+
   // Tell the compositor to notify APZ that a layer has been confirmed for an
   // input event.
   async SetConfirmedTargetAPZC(uint64_t aInputBlockId, ScrollableLayerGuid[] aTargets);

ipc/ipdl/sync-messages.ini

@@ -1001,8 +1001,6 @@ description =
 description =
 [PCompositorBridge::StopFrameTimeRecording]
 description =
-[PCompositorBridge::PLayerTransaction]
-description =
 [PCompositorBridge::SyncWithCompositor]
 description =
 [PCompositorBridge::PWebRenderBridge]
@@ -1036,6 +1034,8 @@ description =
 description =
 [PLayerTransaction::RequestProperty]
 description =
+[PLayerTransaction::GetTextureFactoryIdentifier]
+description = bug 1350634
 [PUiCompositorController::Pause]
 description =
 [PUiCompositorController::Resume]

widget/nsBaseWidget.cpp

@@ -12,6 +12,7 @@
 
 #include "mozilla/layers/CompositorBridgeChild.h"
 #include "mozilla/layers/CompositorBridgeParent.h"
+#include "mozilla/layers/PLayerTransactionChild.h"
 #include "mozilla/layers/ImageBridgeChild.h"
 #include "LiveResizeListener.h"
 #include "nsBaseWidget.h"
@@ -1346,8 +1347,13 @@ void nsBaseWidget::CreateCompositor(int aWidth, int aHeight)
 
     bool success = false;
     if (!backendHints.IsEmpty()) {
-      shadowManager = mCompositorBridgeChild->SendPLayerTransactionConstructor(
-        backendHints, 0, &textureFactoryIdentifier, &success);
+      shadowManager =
+        mCompositorBridgeChild->SendPLayerTransactionConstructor(backendHints, 0);
+      if (shadowManager->SendGetTextureFactoryIdentifier(&textureFactoryIdentifier) &&
+          textureFactoryIdentifier.mParentBackend != LayersBackend::LAYERS_NONE)
+      {
+        success = true;
+      }
     }
 
     if (!success) {