Bug 1045213 - Make menu separators look correct by having them use foreground vibrancy. r=smichaud

widget/cocoa/VibrancyManager.mm

@@ -140,18 +140,31 @@ HitTestNil(id self, SEL _cmd, NSPoint aPoint)
   return nil;
 }
 
+static BOOL
+AllowsVibrancyYes(id self, SEL _cmd)
+{
+  // Means that the foreground is blended using a vibrant blend mode.
+  return YES;
+}
+
 static Class
-CreateEffectViewClass()
+CreateEffectViewClass(BOOL aForegroundVibrancy)
 {
   // Create a class called EffectView that inherits from NSVisualEffectView
   // and overrides the methods -[NSVisualEffectView drawRect:] and
   // -[NSView hitTest:].
   Class NSVisualEffectViewClass = NSClassFromString(@"NSVisualEffectView");
-  Class EffectViewClass = objc_allocateClassPair(NSVisualEffectViewClass, "EffectView", 0);
+  const char* className = aForegroundVibrancy
+    ? "EffectViewWithForegroundVibrancy" : "EffectViewWithoutForegroundVibrancy";
+  Class EffectViewClass = objc_allocateClassPair(NSVisualEffectViewClass, className, 0);
   class_addMethod(EffectViewClass, @selector(drawRect:), (IMP)DrawRectNothing,
                   "v@:{CGRect={CGPoint=dd}{CGSize=dd}}");
   class_addMethod(EffectViewClass, @selector(hitTest:), (IMP)HitTestNil,
                   "@@:{CGPoint=dd}");
+  if (aForegroundVibrancy) {
+    // Also override the -[NSView allowsVibrancy] method to return YES.
+    class_addMethod(EffectViewClass, @selector(allowsVibrancy), (IMP)AllowsVibrancyYes, "I@:");
+  }
   return EffectViewClass;
 }
 
@@ -178,6 +191,10 @@ enum {
   NSVisualEffectStateActive,
   NSVisualEffectStateInactive
 };
+
+enum {
+  NSVisualEffectMaterialTitlebar = 3
+};
 #endif
 
 static NSUInteger
@@ -195,6 +212,17 @@ VisualEffectStateForVibrancyType(VibrancyType aType)
   }
 }
 
+static BOOL
+HasVibrantForeground(VibrancyType aType)
+{
+  switch (aType) {
+    case VibrancyType::MENU:
+      return YES;
+    default:
+      return NO;
+  }
+}
+
 enum {
   NSVisualEffectMaterialMenuItem = 4
 };
@@ -208,18 +236,31 @@ enum {
 NSView*
 VibrancyManager::CreateEffectView(VibrancyType aType, NSRect aRect)
 {
-  static Class EffectViewClass = CreateEffectViewClass();
+  static Class EffectViewClassWithoutForegroundVibrancy = CreateEffectViewClass(NO);
+  static Class EffectViewClassWithForegroundVibrancy = CreateEffectViewClass(YES);
+
+  Class EffectViewClass = HasVibrantForeground(aType)
+    ? EffectViewClassWithForegroundVibrancy : EffectViewClassWithoutForegroundVibrancy;
   NSView* effectView = [[EffectViewClass alloc] initWithFrame:aRect];
   [effectView performSelector:@selector(setAppearance:)
                    withObject:AppearanceForVibrancyType(aType)];
   [effectView setState:VisualEffectStateForVibrancyType(aType)];
 
-  if (aType == VibrancyType::HIGHLIGHTED_MENUITEM) {
+  if (aType == VibrancyType::MENU) {
+    // NSVisualEffectMaterialTitlebar doesn't match the native menu look
+    // perfectly but comes pretty close. Ideally we'd use a material with
+    // materialTypeName "MacLight", since that's what menus use, but there's
+    // no entry with that material in the internalMaterialType-to-
+    // CGSWindowBackdropViewSpec table which NSVisualEffectView consults when
+    // setting up the effect.
+    [effectView setMaterial:NSVisualEffectMaterialTitlebar];
+  } else if (aType == VibrancyType::HIGHLIGHTED_MENUITEM) {
     [effectView setMaterial:NSVisualEffectMaterialMenuItem];
     if ([effectView respondsToSelector:@selector(setEmphasized:)]) {
       [effectView setEmphasized:YES];
     }
   }
+
   return effectView;
 }