Bug 1837007 Part 1: Make nsCocoaWindow run its own event loop when any window is in a native fullscreen transition. r=mstange

This code runs its own event loop during a native fullscreen transition in
3 places:

1) In ProcessTransitions() when attempting to enter native fullscreen.
2) In ProcessTransitions() when attempting to exit native fullscreen.
3) In DestroyNativeWindow().

This ensures that queued native fullscreen transitions (not user-generated
ones) will succeed, even if multiple windows are transitioning to/from
native fullscreen or being closed while in fullscreen.

This added code is only run for programmatic fullscreen transitions. It
has no knowledge of user-initiated actions. This means that if a user is
manually toggling fullscreen while programmatic actions are being
proceessed, it's still possible for a fullscreen transition to fail.

Differential Revision: https://phabricator.services.mozilla.com/D180121

widget/cocoa/nsCocoaWindow.h

@@ -10,6 +10,7 @@
 
 #import <Cocoa/Cocoa.h>
 
+#include "mozilla/StaticMutex.h"
 #include "mozilla/RefPtr.h"
 #include "nsBaseWidget.h"
 #include "nsPIWidgetCocoa.h"
@@ -485,6 +486,17 @@ class nsCocoaWindow final : public nsBaseWidget, public nsPIWidgetCocoa {
   NSWindowAnimationBehavior mWindowAnimationBehavior;
 
  private:
+  // This is class state for tracking which nsCocoaWindow, if any, is in the
+  // middle of a native fullscreen transition.
+  static mozilla::StaticDataMutex<nsCocoaWindow*> sWindowInNativeTransition;
+
+  // This function returns true if the caller has been able to claim the sole
+  // permission to start a native transition. It must be followed by a call
+  // to EndOurNativeTransition() when the native transition is complete.
+  bool CanStartNativeTransition();
+  bool WeAreInNativeTransition();
+  void EndOurNativeTransition();
+
   // true if Show() has been called.
   bool mWasShown;
 };

widget/cocoa/nsCocoaWindow.mm

@@ -168,6 +168,44 @@ void nsCocoaWindow::DestroyNativeWindow() {
 
   if (!mWindow) return;
 
+  // Define a helper function for checking our fullscreen window status.
+  bool (^inNativeFullscreen)(void) = ^{
+    return ((mWindow.styleMask & NSFullScreenWindowMask) == NSFullScreenWindowMask);
+  };
+
+  // If we are in native fullscreen, or we are in the middle of a native
+  // fullscreen transition, spin our run loop until both those things
+  // are false. This ensures that we've completed all our native
+  // fullscreen transitions and updated our class state before we close
+  // the window. We need to do this here (rather than in some other
+  // nsCocoaWindow trying to do a native fullscreen transition) because
+  // part of closing our window is clearing out our delegate, and the
+  // delegate callback is the only other way to clear our class state.
+  //
+  // While spinning this run loop, check to see if we are still in native
+  // fullscreen. If we are, request that we leave fullscreen (only once)
+  // and continue to spin the run loop until we're out of fullscreen.
+  //
+  // However, it's possible that we are *already* in a run loop inside of
+  // ProcessTransitions(), waiting on a native fullscreen transition to
+  // complete. In such a case, we can't spin the run loop again, so we
+  // don't even enter this loop if mInProcessTransitions is true.
+  bool haveRequestedFullscreenExit = false;
+  NSRunLoop* localRunLoop = [NSRunLoop currentRunLoop];
+  while (!mInProcessTransitions && (inNativeFullscreen() || WeAreInNativeTransition()) &&
+         [localRunLoop runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]) {
+    // This loop continues to process events until mWindow is fully out
+    // of native fullscreen.
+
+    // Check to see if we should one-time request an exit from fullscreen.
+    // We do this if we are in native fullscreen and no window is in a
+    // native fullscreen transition.
+    if (!haveRequestedFullscreenExit && inNativeFullscreen() && CanStartNativeTransition()) {
+      [mWindow toggleFullScreen:nil];
+      haveRequestedFullscreenExit = true;
+    }
+  }
+
   [mWindow releaseJSObjects];
   // We want to unhook the delegate here because we don't want events
   // sent to it after this object has been destroyed.
@@ -176,6 +214,12 @@ void nsCocoaWindow::DestroyNativeWindow() {
   mWindow = nil;
   [mDelegate autorelease];
 
+  // We've lost access to our delegate. If we are still in a native
+  // fullscreen transition, we have to give up on it now even if it
+  // isn't finished, because the delegate is the only way that the
+  // class state would ever be cleared.
+  EndOurNativeTransition();
+
   NS_OBJC_END_TRY_IGNORE_BLOCK;
 }
 
@@ -1625,6 +1669,7 @@ void nsCocoaWindow::CocoaWindowWillEnterFullscreen(bool aFullscreen) {
 
 void nsCocoaWindow::CocoaWindowDidEnterFullscreen(bool aFullscreen) {
   mHasStartedNativeFullscreen = false;
+  EndOurNativeTransition();
   DispatchOcclusionEvent();
   HandleUpdateFullscreenOnResize();
   FinishCurrentTransitionIfMatching(aFullscreen ? TransitionType::Fullscreen
@@ -1633,6 +1678,7 @@ void nsCocoaWindow::CocoaWindowDidEnterFullscreen(bool aFullscreen) {
 
 void nsCocoaWindow::CocoaWindowDidFailFullscreen(bool aAttemptedFullscreen) {
   mHasStartedNativeFullscreen = false;
+  EndOurNativeTransition();
   DispatchOcclusionEvent();
 
   // If we already updated our fullscreen state due to a resize, we need to update it again.
@@ -1750,6 +1796,17 @@ void nsCocoaWindow::ProcessTransitions() {
     switch (*mTransitionCurrent) {
       case TransitionType::Fullscreen: {
         if (!mInFullScreenMode) {
+          // Check our global state to see if it is safe to start a native
+          // fullscreen transition. While that is false, run our own event loop.
+          // Eventually, the native fullscreen transition will finish, and we can
+          // safely continue.
+          NSRunLoop* localRunLoop = [NSRunLoop currentRunLoop];
+          while (mWindow && !CanStartNativeTransition() &&
+                 [localRunLoop runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]) {
+            // This loop continues to process events until CanStartNativeTransition()
+            // returns true.
+          }
+
           // This triggers an async animation, so continue.
           [mWindow toggleFullScreen:nil];
           continue;
@@ -1776,6 +1833,17 @@ void nsCocoaWindow::ProcessTransitions() {
       case TransitionType::Windowed: {
         if (mInFullScreenMode) {
           if (mInNativeFullScreenMode) {
+            // Check our global state to see if it is safe to start a native
+            // fullscreen transition. While that is false, run our own event loop.
+            // Eventually, the native fullscreen transition will finish, and we can
+            // safely continue.
+            NSRunLoop* localRunLoop = [NSRunLoop currentRunLoop];
+            while (mWindow && !CanStartNativeTransition() &&
+                   [localRunLoop runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]) {
+              // This loop continues to process events until CanStartNativeTransition()
+              // returns true.
+            }
+
             // This triggers an async animation, so continue.
             [mWindow toggleFullScreen:nil];
             continue;
@@ -1873,7 +1941,7 @@ void nsCocoaWindow::FinishCurrentTransitionIfMatching(const TransitionType& aTra
     // nsWindowDelegate transition callbacks, we want to make sure those callbacks are
     // all the way done before we start processing more transitions. To accomplish this,
     // we dispatch our cleanup to happen on the next event loop. Doing this will ensure
-    // that any async native transition methods we call (like toggleFullscreen) will
+    // that any async native transition methods we call (like toggleFullScreen) will
     // succeed.
     NS_DispatchToCurrentThread(NewRunnableMethod("FinishCurrentTransition", this,
                                                  &nsCocoaWindow::FinishCurrentTransition));
@@ -1892,6 +1960,32 @@ bool nsCocoaWindow::HandleUpdateFullscreenOnResize() {
   return true;
 }
 
+/* static */ mozilla::StaticDataMutex<nsCocoaWindow*> nsCocoaWindow::sWindowInNativeTransition(
+    nullptr);
+
+bool nsCocoaWindow::CanStartNativeTransition() {
+  auto window = sWindowInNativeTransition.Lock();
+  if (*window == nullptr) {
+    // Claim it and return true, indicating that the caller has permission to start
+    // the native fullscreen transition.
+    *window = this;
+    return true;
+  }
+  return false;
+}
+
+bool nsCocoaWindow::WeAreInNativeTransition() {
+  auto window = sWindowInNativeTransition.Lock();
+  return (*window == this);
+}
+
+void nsCocoaWindow::EndOurNativeTransition() {
+  auto window = sWindowInNativeTransition.Lock();
+  if (*window == this) {
+    *window = nullptr;
+  }
+}
+
 // Coordinates are desktop pixels
 void nsCocoaWindow::DoResize(double aX, double aY, double aWidth, double aHeight, bool aRepaint,
                              bool aConstrainToCurrentScreen) {