Bug 1511438 Part 2: Enable win32k lockdown on RDD process. r=jimm

This stops the use of some win32k calls during start-up that will fail and in
some cases cause a crash.

It also moves the MITIGATION_DYNAMIC_CODE_DISABLE to be enabled after start-up.
This is required because the hooks to fake the user32 and gdi32 initialization
are applied as the DLLs load and the dynamic code disable blocks that.

browser/app/profile/firefox.js

@@ -1041,10 +1041,11 @@ pref("security.sandbox.windows.log.stackTraceDepth", 0);
 // security/sandbox/win/src/sandboxbroker/sandboxBroker.cpp
 pref("security.sandbox.gpu.level", 0);
 
-// Controls whether we disable win32k for the GMP processes.
+// Controls whether we disable win32k for the processes.
 // true means that win32k system calls are not permitted.
-// Note: win32k is currently _not_ disabled due to intermittent test failures,
-// where the GMP process fails very early. See bug 1449348.
+pref("security.sandbox.rdd.win32k-disable", true);
+// Note: win32k is currently _not_ disabled for GMP due to intermittent test
+// failures, where the GMP process fails very early. See bug 1449348.
 pref("security.sandbox.gmp.win32k-disable", false);
 #endif
 

dom/media/ipc/RDDProcessImpl.h

@@ -29,11 +29,6 @@ class RDDProcessImpl final : public ipc::ProcessChild {
   DISALLOW_COPY_AND_ASSIGN(RDDProcessImpl);
 
   RDDParent mRDD;
-
-#if defined(XP_WIN)
-  // This object initializes and configures COM.
-  mozilla::mscom::MainThreadRuntime mCOMRuntime;
-#endif
 };
 
 }  // namespace mozilla

security/sandbox/win/src/sandboxbroker/sandboxBroker.cpp

@@ -52,6 +52,7 @@ static UniquePtr<nsString> sUserExtensionsDir;
 #endif
 
 // Cached prefs which are needed off main thread.
+static bool sRddWin32kDisable = false;
 static bool sGmpWin32kDisable = false;
 
 static LazyLogModule sSandboxBrokerLog("SandboxBroker");
@@ -129,6 +130,8 @@ void SandboxBroker::GeckoDependentInitialize() {
   ClearOnShutdown(&sLaunchErrors);
 
   // Cache prefs that are needed off main thread.
+  Preferences::AddBoolVarCache(&sRddWin32kDisable,
+                               "security.sandbox.rdd.win32k-disable");
   Preferences::AddBoolVarCache(&sGmpWin32kDisable,
                                "security.sandbox.gmp.win32k-disable");
 }
@@ -725,13 +728,21 @@ bool SandboxBroker::SetSecurityLevelForRDDProcess() {
       sandbox::MITIGATION_BOTTOM_UP_ASLR | sandbox::MITIGATION_HEAP_TERMINATE |
       sandbox::MITIGATION_SEHOP | sandbox::MITIGATION_EXTENSION_POINT_DISABLE |
       sandbox::MITIGATION_DEP_NO_ATL_THUNK | sandbox::MITIGATION_DEP |
-      sandbox::MITIGATION_DYNAMIC_CODE_DISABLE |
       sandbox::MITIGATION_IMAGE_LOAD_PREFER_SYS32;
 
+  if (sRddWin32kDisable) {
+    mitigations |= sandbox::MITIGATION_WIN32K_DISABLE;
+    result =
+        mPolicy->AddRule(sandbox::TargetPolicy::SUBSYS_WIN32K_LOCKDOWN,
+                         sandbox::TargetPolicy::FAKE_USER_GDI_INIT, nullptr);
+    SANDBOX_ENSURE_SUCCESS(result, "Failed to set FAKE_USER_GDI_INIT policy.");
+  }
+
   result = mPolicy->SetProcessMitigations(mitigations);
   SANDBOX_ENSURE_SUCCESS(result, "Invalid flags for SetProcessMitigations.");
 
   mitigations = sandbox::MITIGATION_STRICT_HANDLE_CHECKS |
+                sandbox::MITIGATION_DYNAMIC_CODE_DISABLE |
                 sandbox::MITIGATION_DLL_SEARCH_ORDER;
 
   result = mPolicy->SetDelayedProcessMitigations(mitigations);

toolkit/xre/nsAppRunner.cpp

@@ -4714,7 +4714,7 @@ bool XRE_IsE10sParentProcess() {
 #undef GECKO_PROCESS_TYPE
 
 bool XRE_UseNativeEventProcessing() {
-#ifdef XP_MACOSX
+#if defined(XP_MACOSX) || defined(XP_WIN)
   if (XRE_IsRDDProcess() || XRE_IsSocketProcess()) {
     return false;
   }

widget/windows/nsAppShell.cpp

@@ -323,9 +323,11 @@ nsresult nsAppShell::Init() {
 
   mozilla::ipc::windows::InitUIThread();
 
-  sTaskbarButtonCreatedMsg = ::RegisterWindowMessageW(kTaskbarButtonEventId);
-  NS_ASSERTION(sTaskbarButtonCreatedMsg,
-               "Could not register taskbar button creation message");
+  if (XRE_Win32kCallsAllowed()) {
+    sTaskbarButtonCreatedMsg = ::RegisterWindowMessageW(kTaskbarButtonEventId);
+    NS_ASSERTION(sTaskbarButtonCreatedMsg,
+                 "Could not register taskbar button creation message");
+  }
 
   // The hidden message window is used for interrupting the processing of native
   // events, so that we can process gecko events. Therefore, we only need it if
@@ -354,7 +356,7 @@ nsresult nsAppShell::Init() {
     mEventWnd = CreateWindowW(kWindowClass, L"nsAppShell:EventWindow", 0, 0, 0,
                               10, 10, HWND_MESSAGE, nullptr, module, nullptr);
     NS_ENSURE_STATE(mEventWnd);
-  } else {
+  } else if (XRE_IsContentProcess()) {
     // We're not generally processing native events, but still using GDI and we
     // still have some internal windows, e.g. from calling CoInitializeEx.
     // So we use a class that will do a single event pump where previously we