Bug 1641722: Deactivate remote canvas 2D when device creation or stream read failure occurs. r=jrmuizel,chutten

This also adds telemetry probes to track:
* number of times remote canvas 2D is activated
* number of times remote canvas 2D is deactivated due to device creation failure
* number of times remote canvas 2D is deactivated due to a stream read error.

Differential Revision: https://phabricator.services.mozilla.com/D81032

gfx/ipc/GPUParent.cpp

@@ -209,8 +209,11 @@ mozilla::ipc::IPCResult GPUParent::RecvInit(
   if (gfxConfig::IsEnabled(Feature::D3D11_COMPOSITING)) {
     if (DeviceManagerDx::Get()->CreateCompositorDevices() &&
         gfxVars::RemoteCanvasEnabled()) {
-      MOZ_ALWAYS_TRUE(DeviceManagerDx::Get()->CreateCanvasDevice());
-      MOZ_ALWAYS_TRUE(Factory::EnsureDWriteFactory());
+      if (DeviceManagerDx::Get()->CreateCanvasDevice()) {
+        MOZ_ALWAYS_TRUE(Factory::EnsureDWriteFactory());
+      } else {
+        gfxWarning() << "Failed to create canvas device.";
+      }
     }
   }
   if (gfxVars::UseWebRender()) {

gfx/layers/CanvasDrawEventRecorder.cpp

@@ -175,7 +175,7 @@ bool CanvasEventRingBuffer::WaitForAndRecalculateAvailableData() {
   uint32_t maxToRead = kStreamSize - bufPos;
   mAvailable = std::min(maxToRead, WaitForBytesToRead());
   if (!mAvailable) {
-    mGood = false;
+    SetIsBad();
     mBufPos = nullptr;
     return false;
   }
@@ -231,6 +231,7 @@ void CanvasEventRingBuffer::CheckAndSignalReader() {
   do {
     switch (mRead->state) {
       case State::Processing:
+      case State::Failed:
         return;
       case State::AboutToWait:
         // The reader is making a decision about whether to wait. So, we must
@@ -394,7 +395,7 @@ bool CanvasEventRingBuffer::WaitForReadCount(uint32_t aReadCount,
   mWrite->state = State::Waiting;
 
   // Wait unless we detect the reading side has closed.
-  while (!mWriterServices->ReaderClosed()) {
+  while (!mWriterServices->ReaderClosed() && mRead->state != State::Failed) {
     if (mWriterSemaphore->Wait(Some(aTimeout))) {
       MOZ_ASSERT(mOurCount - mRead->count <= requiredDifference);
       return true;

gfx/layers/CanvasDrawEventRecorder.h

@@ -89,7 +89,10 @@ class CanvasEventRingBuffer final : public gfx::EventRingBuffer {
 
   bool good() const final { return mGood; }
 
-  void SetIsBad() final { mGood = false; }
+  void SetIsBad() final {
+    mGood = false;
+    mRead->state = State::Failed;
+  }
 
   void write(const char* const aData, const size_t aSize) final;
 
@@ -178,7 +181,8 @@ class CanvasEventRingBuffer final : public gfx::EventRingBuffer {
      */
     AboutToWait,
     Waiting,
-    Stopped
+    Stopped,
+    Failed,
   };
 
   struct ReadFooter {

gfx/layers/client/TextureClient.cpp

@@ -372,6 +372,10 @@ TextureData* TextureData::Create(TextureForwarder* aAllocator,
       return new RecordedTextureData(canvasChild.forget(), aSize, aFormat,
                                      textureType);
     }
+
+    // We don't have a CanvasChild, but are supposed to be remote.
+    // Fall back to software.
+    textureType = TextureType::Unknown;
   }
 
   switch (textureType) {

gfx/layers/composite/TextureHost.cpp

@@ -13,6 +13,7 @@
 #include "mozilla/gfx/gfxVars.h"
 #include "mozilla/ipc/Shmem.h"  // for Shmem
 #include "mozilla/layers/AsyncImagePipelineManager.h"
+#include "mozilla/layers/BufferTexture.h"
 #include "mozilla/layers/CompositableTransactionParent.h"  // for CompositableParentManager
 #include "mozilla/layers/CompositorBridgeParent.h"
 #include "mozilla/layers/Compositor.h"         // for Compositor
@@ -167,6 +168,25 @@ void TextureHost::SetLastFwdTransactionId(uint64_t aTransactionId) {
   mFwdTransactionId = aTransactionId;
 }
 
+already_AddRefed<TextureHost> CreateDummyBufferTextureHost(
+    mozilla::layers::LayersBackend aBackend,
+    mozilla::layers::TextureFlags aFlags) {
+  // Ensure that the host will delete the memory.
+  aFlags &= ~TextureFlags::DEALLOCATE_CLIENT;
+  UniquePtr<TextureData> textureData(BufferTextureData::Create(
+      gfx::IntSize(1, 1), gfx::SurfaceFormat::B8G8R8A8, gfx::BackendType::SKIA,
+      aBackend, aFlags, TextureAllocationFlags::ALLOC_DEFAULT, nullptr));
+  SurfaceDescriptor surfDesc;
+  textureData->Serialize(surfDesc);
+  const SurfaceDescriptorBuffer& bufferDesc =
+      surfDesc.get_SurfaceDescriptorBuffer();
+  const MemoryOrShmem& data = bufferDesc.data();
+  RefPtr<TextureHost> host =
+      new MemoryTextureHost(reinterpret_cast<uint8_t*>(data.get_uintptr_t()),
+                            bufferDesc.desc(), aFlags);
+  return host.forget();
+}
+
 already_AddRefed<TextureHost> TextureHost::Create(
     const SurfaceDescriptor& aDesc, const ReadLockDescriptor& aReadLock,
     ISurfaceAllocator* aDeallocator, LayersBackend aBackend,
@@ -227,8 +247,10 @@ already_AddRefed<TextureHost> TextureHost::Create(
           aDeallocator->AsCompositorBridgeParentBase()
               ->LookupSurfaceDescriptorForClientDrawTarget(desc.drawTarget());
       if (!realDesc) {
-        NS_WARNING("Failed to get descriptor for recorded texture.");
-        return nullptr;
+        gfxCriticalNote << "Failed to get descriptor for recorded texture.";
+        // Create a dummy to prevent any crashes due to missing IPDL actors.
+        result = CreateDummyBufferTextureHost(aBackend, aFlags);
+        break;
       }
 
       result = TextureHost::Create(*realDesc, aReadLock, aDeallocator, aBackend,

gfx/layers/ipc/CanvasChild.cpp

@@ -115,16 +115,27 @@ CanvasChild::CanvasChild(Endpoint<PCanvasChild>&& aEndpoint) {
 
 CanvasChild::~CanvasChild() = default;
 
-ipc::IPCResult CanvasChild::RecvNotifyDeviceChanged() {
-  nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
+static void NotifyCanvasDeviceReset() {
+  nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
   if (obs) {
     obs->NotifyObservers(nullptr, "canvas-device-reset", nullptr);
   }
+}
 
+ipc::IPCResult CanvasChild::RecvNotifyDeviceChanged() {
+  NotifyCanvasDeviceReset();
   mRecorder->RecordEvent(RecordedDeviceChangeAcknowledged());
   return IPC_OK();
 }
 
+/* static */ bool CanvasChild::mDeactivated = false;
+
+ipc::IPCResult CanvasChild::RecvDeactivate() {
+  mDeactivated = true;
+  NotifyCanvasDeviceReset();
+  return IPC_OK();
+}
+
 void CanvasChild::EnsureRecorder(TextureType aTextureType) {
   if (!mRecorder) {
     MOZ_ASSERT(mTextureType == TextureType::Unknown);
@@ -213,6 +224,11 @@ void CanvasChild::EndTransaction() {
 }
 
 bool CanvasChild::ShouldBeCleanedUp() const {
+  // Always return true if we've been deactivated.
+  if (Deactivated()) {
+    return true;
+  }
+
   // We can only be cleaned up if nothing else references our recorder.
   if (mRecorder && !mRecorder->hasOneRef()) {
     return false;

gfx/layers/ipc/CanvasChild.h

@@ -29,8 +29,15 @@ class CanvasChild final : public PCanvasChild {
 
   explicit CanvasChild(Endpoint<PCanvasChild>&& aEndpoint);
 
+  /**
+   * @returns true if remote canvas has been deactivated due to failure.
+   */
+  static bool Deactivated() { return mDeactivated; }
+
   ipc::IPCResult RecvNotifyDeviceChanged();
 
+  ipc::IPCResult RecvDeactivate();
+
   /**
    * Ensures that the DrawEventRecorder has been created.
    *
@@ -127,6 +134,8 @@ class CanvasChild final : public PCanvasChild {
 
   static const uint32_t kCacheDataSurfaceThreshold = 10;
 
+  static bool mDeactivated;
+
   RefPtr<CanvasDrawEventRecorder> mRecorder;
   TextureType mTextureType = TextureType::Unknown;
   uint32_t mLastWriteLockCheckpoint = 0;

gfx/layers/ipc/CanvasThread.cpp

@@ -80,6 +80,7 @@ void CanvasThreadHolder::StaticRelease(
 
   auto lockedCanvasThreadHolder = sCanvasThreadHolder.Lock();
   if (lockedCanvasThreadHolder.ref()->mRefCnt == 1) {
+    lockedCanvasThreadHolder.ref()->mCanvasThread->Shutdown();
     lockedCanvasThreadHolder.ref() = nullptr;
   }
 }

gfx/layers/ipc/CanvasTranslator.cpp

@@ -11,6 +11,7 @@
 #include "mozilla/gfx/Logging.h"
 #include "mozilla/layers/TextureClient.h"
 #include "mozilla/SyncRunnable.h"
+#include "mozilla/Telemetry.h"
 #include "nsTHashtable.h"
 #include "RecordedCanvasEventImpl.h"
 
@@ -99,7 +100,10 @@ static void EnsureAllClosed() {
 
 CanvasTranslator::CanvasTranslator(
     already_AddRefed<CanvasThreadHolder> aCanvasThreadHolder)
-    : gfx::InlineTranslator(), mCanvasThreadHolder(aCanvasThreadHolder) {}
+    : gfx::InlineTranslator(), mCanvasThreadHolder(aCanvasThreadHolder) {
+  // Track when remote canvas has been activated.
+  Telemetry::ScalarAdd(Telemetry::ScalarID::GFX_CANVAS_REMOTE_ACTIVATED, 1);
+}
 
 CanvasTranslator::~CanvasTranslator() {
   if (mReferenceTextureData) {
@@ -121,26 +125,30 @@ mozilla::ipc::IPCResult CanvasTranslator::RecvInitTranslator(
     const CrossProcessSemaphoreHandle& aReaderSem,
     const CrossProcessSemaphoreHandle& aWriterSem) {
   mTextureType = aTextureType;
-#if defined(XP_WIN)
-  if (!CheckForFreshCanvasDevice(__LINE__)) {
-    gfxCriticalNote << "GFX: CanvasTranslator failed to get device";
-    return IPC_FAIL(this, "Failed to get canvas device.");
-  }
-#endif
 
+  // We need to initialize the stream first, because it might be used to
+  // communicate other failures back to the writer.
   mStream = MakeUnique<CanvasEventRingBuffer>();
   if (!mStream->InitReader(aReadHandle, aReaderSem, aWriterSem,
                            MakeUnique<RingBufferReaderServices>(this))) {
     return IPC_FAIL(this, "Failed to initialize ring buffer reader.");
   }
 
+#if defined(XP_WIN)
+  if (!CheckForFreshCanvasDevice(__LINE__)) {
+    gfxCriticalNote << "GFX: CanvasTranslator failed to get device";
+    return IPC_OK();
+  }
+#endif
+
   mTranslationTaskQueue = mCanvasThreadHolder->CreateWorkerTaskQueue();
   return RecvResumeTranslation();
 }
 
 ipc::IPCResult CanvasTranslator::RecvResumeTranslation() {
-  if (!IsValid()) {
-    return IPC_FAIL(this, "Canvas Translation failed.");
+  if (mDeactivated) {
+    // The other side might have sent a resume message before we deactivated.
+    return IPC_OK();
   }
 
   MOZ_ALWAYS_SUCCEEDS(mTranslationTaskQueue->Dispatch(
@@ -156,6 +164,13 @@ void CanvasTranslator::StartTranslation() {
         NewRunnableMethod("CanvasTranslator::StartTranslation", this,
                           &CanvasTranslator::StartTranslation)));
   }
+
+  // If the stream has been marked as bad deactivate remote canvas.
+  if (!mStream->good()) {
+    Telemetry::ScalarAdd(
+        Telemetry::ScalarID::GFX_CANVAS_REMOTE_DEACTIVATED_BAD_STREAM, 1);
+    Deactivate();
+  }
 }
 
 void CanvasTranslator::ActorDestroy(ActorDestroyReason why) {
@@ -188,6 +203,32 @@ void CanvasTranslator::FinishShutdown() {
   canvasTranslators.RemoveEntry(this);
 }
 
+void CanvasTranslator::Deactivate() {
+  if (mDeactivated) {
+    return;
+  }
+  mDeactivated = true;
+
+  // We need to tell the other side to deactivate. Make sure the stream is
+  // marked as bad so that the writing side won't wait for space to write.
+  mStream->SetIsBad();
+  mCanvasThreadHolder->DispatchToCanvasThread(
+      NewRunnableMethod("CanvasTranslator::SendDeactivate", this,
+                        &CanvasTranslator::SendDeactivate));
+
+  {
+    // Unlock all of our textures.
+    gfx::AutoSerializeWithMoz2D serializeWithMoz2D(GetBackendType());
+    for (auto const& entry : mTextureDatas) {
+      entry.second->Unlock();
+    }
+  }
+
+  // Also notify anyone waiting for a surface descriptor. This must be done
+  // after mDeactivated is set to true.
+  mSurfaceDescriptorsMonitor.NotifyAll();
+}
+
 bool CanvasTranslator::TranslateRecording() {
   MOZ_ASSERT(CanvasThreadHolder::IsInCanvasWorker());
 
@@ -212,6 +253,11 @@ bool CanvasTranslator::TranslateRecording() {
           return recordedEvent->PlayEvent(this);
         });
 
+    // Check the stream is good here or we will log the issue twice.
+    if (!mStream->good()) {
+      return true;
+    }
+
     if (!success && !HandleExtensionEvent(eventType)) {
       if (mDeviceResetInProgress) {
         // We've notified the recorder of a device change, so we are expecting
@@ -241,7 +287,6 @@ bool CanvasTranslator::TranslateRecording() {
     eventType = mStream->ReadNextEvent();
   }
 
-  mIsValid = false;
   return true;
 }
 
@@ -350,7 +395,15 @@ bool CanvasTranslator::CheckForFreshCanvasDevice(int aLineNumber) {
                                  /*aForceDispatch*/ true);
 
   mDevice = gfx::DeviceManagerDx::Get()->GetCanvasDevice();
-  return mDevice && CreateReferenceTexture();
+  if (!mDevice) {
+    // We don't have a canvas device, we need to deactivate.
+    Telemetry::ScalarAdd(
+        Telemetry::ScalarID::GFX_CANVAS_REMOTE_DEACTIVATED_NO_DEVICE, 1);
+    Deactivate();
+    return false;
+  }
+
+  return CreateReferenceTexture();
 #else
   return false;
 #endif
@@ -425,6 +478,11 @@ UniquePtr<SurfaceDescriptor> CanvasTranslator::WaitForSurfaceDescriptor(
   DescriptorMap::iterator result;
   while ((result = mSurfaceDescriptors.find(aDrawTarget)) ==
          mSurfaceDescriptors.end()) {
+    // If remote canvas has been deactivated just return null.
+    if (mDeactivated) {
+      return nullptr;
+    }
+
     mSurfaceDescriptorsMonitor.Wait();
   }
 

gfx/layers/ipc/CanvasTranslator.h

@@ -98,8 +98,6 @@ class CanvasTranslator final : public gfx::InlineTranslator,
             const CrossProcessSemaphoreHandle& aWriterSem,
             UniquePtr<CanvasEventRingBuffer::ReaderServices> aReaderServices);
 
-  bool IsValid() { return mIsValid; }
-
   /**
    * Translates events until no more are available or the end of a transaction
    * If this returns false the caller of this is responsible for re-calling
@@ -259,6 +257,8 @@ class CanvasTranslator final : public gfx::InlineTranslator,
 
   void FinishShutdown();
 
+  void Deactivate();
+
   TextureData* CreateTextureData(TextureType aTextureType,
                                  const gfx::IntSize& aSize,
                                  gfx::SurfaceFormat aFormat);
@@ -294,7 +294,7 @@ class CanvasTranslator final : public gfx::InlineTranslator,
   DescriptorMap mSurfaceDescriptors;
   Monitor mSurfaceDescriptorsMonitor{
       "CanvasTranslator::mSurfaceDescriptorsMonitor"};
-  bool mIsValid = true;
+  Atomic<bool> mDeactivated{false};
   bool mIsInTransaction = false;
   bool mDeviceResetInProgress = false;
 };

gfx/layers/ipc/CompositorBridgeChild.cpp

@@ -949,6 +949,11 @@ PTextureChild* CompositorBridgeChild::CreateTexture(
 
 already_AddRefed<CanvasChild> CompositorBridgeChild::GetCanvasChild() {
   MOZ_ASSERT(gfx::gfxVars::RemoteCanvasEnabled());
+
+  if (CanvasChild::Deactivated()) {
+    return nullptr;
+  }
+
   if (!mCanvasChild) {
     ipc::Endpoint<PCanvasParent> parentEndpoint;
     ipc::Endpoint<PCanvasChild> childEndpoint;

gfx/layers/ipc/PCanvas.ipdl

@@ -37,6 +37,11 @@ parent:
    * Notify that the canvas device used by the translator has changed.
    */
   async NotifyDeviceChanged();
+
+  /**
+   * Deactivate remote canvas, which will cause fall back to software.
+   */
+  async Deactivate();
 };
 
 } // layers

gfx/thebes/gfxWindowsPlatform.cpp

@@ -40,6 +40,7 @@
 #include "gfxGDIFontList.h"
 #include "gfxGDIFont.h"
 
+#include "mozilla/layers/CanvasChild.h"
 #include "mozilla/layers/CompositorThread.h"
 #include "mozilla/layers/PaintThread.h"
 #include "mozilla/layers/ReadbackManagerD3D11.h"
@@ -539,11 +540,17 @@ mozilla::gfx::BackendType gfxWindowsPlatform::GetContentBackendFor(
 mozilla::gfx::BackendType gfxWindowsPlatform::GetPreferredCanvasBackend() {
   mozilla::gfx::BackendType backend = gfxPlatform::GetPreferredCanvasBackend();
 
-  if (backend == BackendType::DIRECT2D1_1 && gfx::gfxVars::UseWebRender() &&
-      !gfx::gfxVars::UseWebRenderANGLE()) {
-    // We can't have D2D without ANGLE when WebRender is enabled, so fallback to
-    // Skia.
-    return BackendType::SKIA;
+  if (backend == BackendType::DIRECT2D1_1) {
+    if (gfx::gfxVars::UseWebRender() && !gfx::gfxVars::UseWebRenderANGLE()) {
+      // We can't have D2D without ANGLE when WebRender is enabled, so fallback
+      // to Skia.
+      return BackendType::SKIA;
+    }
+
+    // Fall back to software when remote canvas has been deactivated.
+    if (CanvasChild::Deactivated()) {
+      return BackendType::SKIA;
+    }
   }
   return backend;
 }

toolkit/components/telemetry/Scalars.yaml

@@ -3238,6 +3238,55 @@ gfx.omtp:
     record_in_processes:
       - 'content'
 
+gfx.canvas.remote:
+  activated:
+    bug_numbers:
+      - 1641722
+    description: >
+      Number of times remote canvas 2D has been activated.
+    kind: uint
+    expires: "85"
+    notification_emails:
+      - gfx-telemetry-alerts@mozilla.com
+      - bowen@mozilla.com
+    release_channel_collection: opt-out
+    products:
+      - 'firefox'
+    record_in_processes:
+      - 'gpu'
+
+  deactivated_no_device:
+    bug_numbers:
+      - 1641722
+    description: >
+      Number of times remote canvas 2D has been deactivated due to device creation failure.
+    kind: uint
+    expires: "85"
+    notification_emails:
+      - gfx-telemetry-alerts@mozilla.com
+      - bowen@mozilla.com
+    release_channel_collection: opt-out
+    products:
+      - 'firefox'
+    record_in_processes:
+      - 'gpu'
+
+  deactivated_bad_stream:
+    bug_numbers:
+      - 1641722
+    description: >
+      Number of times remote canvas 2D has been deactivated due to a stream read failure.
+    kind: uint
+    expires: "85"
+    notification_emails:
+      - gfx-telemetry-alerts@mozilla.com
+      - bowen@mozilla.com
+    release_channel_collection: opt-out
+    products:
+      - 'firefox'
+    record_in_processes:
+      - 'gpu'
+
 gfx.hdr:
   windows_display_colorspace_bitfield:
     bug_numbers: