Bug 1702351 - Stop overriding NSApp.appearance if widget.macos.respect-system-appearance is false; instead, only override NSWindow appearance, nsLookAndFeel appearance and nsNativeThemeCocoa appearance. r=spohl

This makes it so that native context menus, the file picker, and the emoji picker
will respect the system appearance even if widget.macos.respect-system-appearance
is false.
The pref widget.macos.respect-system-appearance now only applies to system colors
(nsLookAndFeel), `-moz-default-appearance` rendering (nsNativeThemeCocoa), and to
the effective NSWindow appearance for windows without an lwtheme appearance
override.

Differential Revision: https://phabricator.services.mozilla.com/D110485

widget/cocoa/AppearanceOverride.h

@@ -0,0 +1,22 @@
+/* -*- Mode: c++; tab-width: 2; indent-tabs-mode: nil; -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AppearanceOverride_h
+#define AppearanceOverride_h
+
+#import <Cocoa/Cocoa.h>
+
+// Implements support for the widget.macos.respect-system-appearance pref.
+// Use MOZGlobalAppearance.sharedInstance.effectiveAppearance in all places where you would like the
+// global override to be respected.
+// The effectiveAppearance property can be key-value observed.
+//
+// Once the pref is removed, all occurrences of MOZGlobalAppearance.sharedInstance can be replaced
+// with NSApp.
+@interface MOZGlobalAppearance : NSObject <NSAppearanceCustomization>
+@property(class, readonly) MOZGlobalAppearance* sharedInstance;
+@end
+
+#endif

widget/cocoa/AppearanceOverride.mm

@@ -0,0 +1,103 @@
+/* -*- Mode: c++; tab-width: 2; indent-tabs-mode: nil; -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#import <Cocoa/Cocoa.h>
+
+#include "AppearanceOverride.h"
+
+#include "mozilla/Preferences.h"
+#include "mozilla/StaticPrefs_widget.h"
+
+#include "nsXULAppAPI.h"
+
+#if !defined(MAC_OS_X_VERSION_10_14) || MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_14
+@interface NSApplication (NSApplicationAppearance3)
+@property(strong) NSAppearance* appearance NS_AVAILABLE_MAC(10_14);
+@property(readonly, strong) NSAppearance* effectiveAppearance NS_AVAILABLE_MAC(10_14);
+@end
+#endif
+
+static void RespectSystemAppearancePrefChanged(const char* aPref, void* aUserInfo);
+
+@interface MOZGlobalAppearance ()
+@property BOOL shouldOverrideWithAqua;
+@end
+
+@implementation MOZGlobalAppearance
+
++ (MOZGlobalAppearance*)sharedInstance {
+  static MOZGlobalAppearance* sInstance = nil;
+  if (!sInstance) {
+    sInstance = [[MOZGlobalAppearance alloc] init];
+    if (XRE_IsParentProcess()) {
+      mozilla::Preferences::RegisterCallbackAndCall(
+          &RespectSystemAppearancePrefChanged,
+          nsDependentCString(
+              mozilla::StaticPrefs::GetPrefName_widget_macos_respect_system_appearance()));
+    }
+  }
+  return sInstance;
+}
+
++ (NSSet*)keyPathsForValuesAffectingAppearance {
+  return [NSSet setWithObject:@"shouldOverrideWithAqua"];
+}
+
+- (NSAppearance*)appearance {
+  if (self.shouldOverrideWithAqua) {
+    // Override with aqua.
+    return [NSAppearance appearanceNamed:NSAppearanceNameAqua];
+  }
+  // nil means "no override".
+  return nil;
+}
+
+- (void)setAppearance:(NSAppearance*)aAppearance {
+  // ignored
+}
+
+- (NSApplication*)_app {
+  return NSApp;
+}
+
++ (NSSet*)keyPathsForValuesAffectingEffectiveAppearance {
+  if (@available(macOS 10.14, *)) {
+    // Automatically notify any key-value observers of our effectiveAppearance property whenever the
+    // pref or the NSApp's effectiveAppearance change.
+    return [NSSet setWithObjects:@"shouldOverrideWithAqua", @"_app.effectiveAppearance", nil];
+  }
+  return [NSSet set];
+}
+
+- (NSAppearance*)effectiveAppearance {
+  if (self.shouldOverrideWithAqua) {
+    // Override with aqua.
+    return [NSAppearance appearanceNamed:NSAppearanceNameAqua];
+  }
+  if (@available(macOS 10.14, *)) {
+    // Use the NSApp effectiveAppearance. This is the system appearance.
+    return NSApp.effectiveAppearance;
+  }
+  // Use aqua on pre-10.14.
+  return [NSAppearance appearanceNamed:NSAppearanceNameAqua];
+}
+
+@end
+
+static void RespectSystemAppearancePrefChanged(const char* aPref, void* aUserInfo) {
+  MOZ_RELEASE_ASSERT(XRE_IsParentProcess());
+  MOZ_RELEASE_ASSERT(NS_IsMainThread());
+
+  MOZGlobalAppearance.sharedInstance.shouldOverrideWithAqua =
+      !mozilla::StaticPrefs::widget_macos_respect_system_appearance();
+
+  // Send a notification that ChildView reacts to. This will cause it to call ThemeChanged and
+  // invalidate LookAndFeel colors.
+  [[NSDistributedNotificationCenter defaultCenter]
+      postNotificationName:@"AppleInterfaceThemeChangedNotification"
+                    object:nil
+                  userInfo:nil
+        deliverImmediately:YES];
+}

widget/cocoa/moz.build

@@ -34,6 +34,7 @@ EXPORTS.mozilla.widget += [
 ]
 
 UNIFIED_SOURCES += [
+    "AppearanceOverride.mm",
     "GfxInfo.mm",
     "MOZIconHelper.mm",
     "NativeKeyBindings.mm",

widget/cocoa/nsCocoaWindow.mm

@@ -6,6 +6,7 @@
 
 #include "nsCocoaWindow.h"
 
+#include "AppearanceOverride.h"
 #include "NativeKeyBindings.h"
 #include "ScreenHelperCocoa.h"
 #include "TextInputHandler.h"
@@ -97,6 +98,12 @@ extern CGSConnection _CGSDefaultConnection(void);
 extern CGError CGSSetWindowTransform(CGSConnection cid, CGSWindow wid, CGAffineTransform transform);
 }
 
+#if !defined(MAC_OS_X_VERSION_10_14) || MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_14
+@interface NSWindow (NSWindowAppearanceSource)
+@property(weak) NSObject<NSAppearanceCustomization>* appearanceSource NS_AVAILABLE_MAC(10_14);
+@end
+#endif
+
 #define NS_APPSHELLSERVICE_CONTRACTID "@mozilla.org/appshell/appShellService;1"
 
 NS_IMPL_ISUPPORTS_INHERITED(nsCocoaWindow, Inherited, nsPIWidgetCocoa)
@@ -551,6 +558,11 @@ nsresult nsCocoaWindow::CreateNativeWindow(const NSRect& aRect, nsBorderStyle aB
   [[WindowDataMap sharedWindowDataMap] ensureDataForWindow:mWindow];
   mWindowMadeHere = true;
 
+  if (@available(macOS 10.14, *)) {
+    // When the window's appearance is set to nil (no override), make sure it respects the global
+    // aqua override.
+    mWindow.appearanceSource = MOZGlobalAppearance.sharedInstance;
+  }
   [mWindow setWindowAppearance:mWindowAppearance];
 
   return NS_OK;
@@ -3237,7 +3249,7 @@ const NSAppearanceName NSAppearanceNameDarkAqua = @"NSAppearanceNameDarkAqua";
         self.appearance = [NSAppearance appearanceNamed:NSAppearanceNameDarkAqua];
         break;
       default:
-        // nil means "no override".
+        // nil means "inherit effectiveAppearance from self.appearanceSource".
         self.appearance = nil;
         break;
     }
@@ -3593,6 +3605,8 @@ typedef NS_ENUM(NSInteger, NSTitlebarSeparatorStyle) {
     [[self mainChildView] ensureNextCompositeIsAtomicWithMainThreadPaint];
     NSNumber* revealAmount = (change[NSKeyValueChangeNewKey]);
     [self updateTitlebarShownAmount:[revealAmount doubleValue]];
+  } else {
+    [super observeValueForKeyPath:keyPath ofObject:object change:change context:context];
   }
 }
 

widget/cocoa/nsLookAndFeel.mm

@@ -3,6 +3,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "AppearanceOverride.h"
 #include "nsLookAndFeel.h"
 #include "nsCocoaFeatures.h"
 #include "nsNativeThemeColors.h"
@@ -44,8 +45,6 @@ using NSAppearanceName = NSString*;
 @end
 #endif
 
-static void RegisterRespectSystemAppearancePrefListenerOnce();
-
 nsLookAndFeel::nsLookAndFeel()
     : nsXPLookAndFeel(),
       mUseOverlayScrollbars(-1),
@@ -85,9 +84,7 @@ nsLookAndFeel::nsLookAndFeel()
       mColorSourceListFontSmoothingBg(0),
       mColorSourceListSelectionFontSmoothingBg(0),
       mColorActiveSourceListSelectionFontSmoothingBg(0),
-      mInitialized(false) {
-  RegisterRespectSystemAppearancePrefListenerOnce();
-}
+      mInitialized(false) {}
 
 nsLookAndFeel::~nsLookAndFeel() {}
 
@@ -691,12 +688,13 @@ void nsLookAndFeel::EnsureInit() {
     // NSAppearance.currentAppearance is global state that can be changed at will to influence the
     // behavior of NSColor and probably others.
     // NSAppearance.currentAppearance does not update automatically if the user switches between
-    // Light Mode and Dark Mode, but NSApp.effectiveAppearance does (unless NSApp.appearance is set
-    // to a non-nil value, which overrides the system appearance).
-    NSAppearance.currentAppearance = NSApp.effectiveAppearance;
+    // Light Mode and Dark Mode, but MOZGlobalAppearance.sharedInstance.effectiveAppearance does
+    // (unless the respect-system-appearance pref is set to false, in which case it will always be
+    // Aqua).
+    NSAppearance.currentAppearance = MOZGlobalAppearance.sharedInstance.effectiveAppearance;
 
     // Check if the current appearance is dark.
-    NSAppearanceName aquaOrDarkAqua = [NSApp.effectiveAppearance
+    NSAppearanceName aquaOrDarkAqua = [MOZGlobalAppearance.sharedInstance.effectiveAppearance
         bestMatchFromAppearancesWithNames:@[ NSAppearanceNameAqua, @"NSAppearanceNameDarkAqua" ]];
     appearanceIsDark = [aquaOrDarkAqua isEqualToString:@"NSAppearanceNameDarkAqua"];
   }
@@ -756,38 +754,3 @@ void nsLookAndFeel::EnsureInit() {
 
   NS_OBJC_END_TRY_IGNORE_BLOCK
 }
-
-static void RespectSystemAppearancePrefChanged(const char* aPref, void* UserInfo) {
-  MOZ_RELEASE_ASSERT(XRE_IsParentProcess());
-  MOZ_RELEASE_ASSERT(NS_IsMainThread());
-
-  if (@available(macOS 10.14, *)) {
-    if (StaticPrefs::widget_macos_respect_system_appearance()) {
-      // nil means "no override".
-      NSApp.appearance = nil;
-    } else {
-      // Override with aqua.
-      NSApp.appearance = [NSAppearance appearanceNamed:NSAppearanceNameAqua];
-    }
-  }
-
-  // Send a notification that ChildView reacts to. This will cause it to call ThemeChanged and
-  // invalidate LookAndFeel colors.
-  [[NSDistributedNotificationCenter defaultCenter]
-      postNotificationName:@"AppleInterfaceThemeChangedNotification"
-                    object:nil
-                  userInfo:nil
-        deliverImmediately:YES];
-}
-
-static void RegisterRespectSystemAppearancePrefListenerOnce() {
-  static bool sRegistered = false;
-  if (sRegistered || !XRE_IsParentProcess()) {
-    return;
-  }
-
-  sRegistered = true;
-  Preferences::RegisterCallbackAndCall(
-      &RespectSystemAppearancePrefChanged,
-      nsDependentCString(StaticPrefs::GetPrefName_widget_macos_respect_system_appearance()));
-}

widget/cocoa/nsNativeThemeCocoa.mm

@@ -5,6 +5,7 @@
 
 #include "nsNativeThemeCocoa.h"
 
+#include "AppearanceOverride.h"
 #include "mozilla/gfx/2D.h"
 #include "mozilla/gfx/Helpers.h"
 #include "mozilla/gfx/PathHelpers.h"
@@ -74,12 +75,6 @@ void CUIDraw(CUIRendererRef r, CGRect rect, CGContextRef ctx, CFDictionaryRef op
 }
 @end
 
-#if !defined(MAC_OS_X_VERSION_10_14) || MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_14
-@interface NSApplication (NSApplicationAppearance)
-@property(readonly, strong) NSAppearance* effectiveAppearance NS_AVAILABLE_MAC(10_14);
-@end
-#endif
-
 // This is the window for our MOZCellDrawView. When an NSCell is drawn, some NSCell implementations
 // look at the draw view's window to determine whether the cell should draw with the active look.
 @interface MOZCellDrawWindow : NSWindow
@@ -779,10 +774,11 @@ static void DrawCellWithSnapping(NSCell* cell, CGContextRef cgContext, const HIR
 @end
 
 static id GetAppAppearance() {
-  if (@available(macOS 10.14, *)) {
-    return NSApp.effectiveAppearance;
-  }
-  return [NSAppearance appearanceNamed:NSAppearanceNameAqua];
+  // Use MOZGlobalAppearance.sharedInstance.effectiveAppearance, which is consistent with
+  // what nsLookAndFeel uses for CSS system colors; it's either the system appearance or our Aqua
+  // override. We could also make nsNativeThemeCocoa respect the per-window appearance, but only
+  // once CSS system colors also respect the window appearance.
+  return MOZGlobalAppearance.sharedInstance.effectiveAppearance;
 }
 
 @interface NSObject (NSAppearanceCoreUIRendering)
@@ -2676,11 +2672,8 @@ void nsNativeThemeCocoa::RenderWidget(const WidgetInfo& aWidgetInfo, DrawTarget&
   if (@available(macOS 10.14, *)) {
     // Some of the drawing below uses NSAppearance.currentAppearance behind the scenes.
     // Set it to the appearance we want, to overwrite any state that's still around from earlier
-    // paints. We set it to NSApp.effectiveAppearance, which is consistent with what nsLookAndFeel
-    // uses for CSS system colors; it's either the system appearance or our Aqua override.
-    // We could also make nsNativeThemeCocoa respect the per-window appearance, but only once CSS
-    // system colors also respect the window appearance.
-    NSAppearance.currentAppearance = NSApp.effectiveAppearance;
+    // paints.
+    NSAppearance.currentAppearance = GetAppAppearance();
   }
 
   const Widget widget = aWidgetInfo.Widget();