Bug 1699224 - Part 5. Unmap shared surfaces if memory pressured during resource updates. r=jrmuizel

Before we start a resource update, we should check if we are virtual
memory pressured (32-bit Windows only). If so, pre-emptively unmap
shared surfaces until the pressure is relieved to try to avoid OOMs
elsewhere. This only applies to the GPU process because the parent
process actively watches its own memory pressure and dispatches a
low-memory event which our expiration tracker is an observer for.

Differential Revision: https://phabricator.services.mozilla.com/D109441

gfx/ipc/GPUParent.cpp

@@ -100,17 +100,17 @@ GPUParent* GPUParent::GetSingleton() {
   return sGPUParent;
 }
 
-/* static */ void GPUParent::MaybeFlushMemory() {
+/* static */ bool GPUParent::MaybeFlushMemory() {
 #if defined(XP_WIN) && !defined(HAVE_64BIT_BUILD)
   MOZ_ASSERT(CompositorThreadHolder::IsInCompositorThread());
   if (!XRE_IsGPUProcess()) {
-    return;
+    return false;
   }
 
   MEMORYSTATUSEX stat;
   stat.dwLength = sizeof(stat);
   if (!GlobalMemoryStatusEx(&stat)) {
-    return;
+    return false;
   }
 
   // We only care about virtual process memory space in the GPU process because
@@ -131,6 +131,9 @@ GPUParent* GPUParent::GetSingleton() {
         }));
   }
   sLowMemory = lowMemory;
+  return lowMemory;
+#else
+  return false;
 #endif
 }
 

gfx/ipc/GPUParent.h

@@ -34,7 +34,7 @@ class GPUParent final : public PGPUParent {
   static void GetGPUProcessName(nsACString& aStr);
 
   // Check for memory pressure and notify the parent process if necessary.
-  static void MaybeFlushMemory();
+  static bool MaybeFlushMemory();
 
   bool Init(base::ProcessId aParentPid, const char* aParentBuildID,
             MessageLoop* aIOLoop, UniquePtr<IPC::Channel> aChannel);

gfx/layers/wr/WebRenderBridgeParent.cpp

@@ -485,8 +485,13 @@ bool WebRenderBridgeParent::UpdateResources(
   wr::ShmSegmentsReader reader(aSmallShmems, aLargeShmems);
   UniquePtr<ScheduleSharedSurfaceRelease> scheduleRelease;
 
-  if (!aResourceUpdates.IsEmpty()) {
-    GPUParent::MaybeFlushMemory();
+  while (GPUParent::MaybeFlushMemory()) {
+    // If the GPU process has memory pressure, preemptively unmap some of our
+    // shared memory images. If we are in the parent process, the expiration
+    // tracker itself will listen for the memory pressure event.
+    if (!SharedSurfacesParent::AgeAndExpireOneGeneration()) {
+      break;
+    }
   }
 
   for (const auto& cmd : aResourceUpdates) {