Bug 1804023 - Part 2: add ability to blocklist DLLs in just the GPU process r=gsvelto

As with the socket process, we can't automated test that the block works in the GPU process, but I manually verified this. I did add an automated test that ensures blocking something in the GPU process doesn't block it in other processes.

Differential Revision: https://phabricator.services.mozilla.com/D167399

browser/app/nsBrowserApp.cpp

@@ -311,13 +311,7 @@ int main(int argc, char* argv[], char* envp[]) {
 #  ifdef HAS_DLL_BLOCKLIST
     uint32_t initFlags =
         gBlocklistInitFlags | eDllBlocklistInitFlagIsChildProcess;
-    // This is too early in launch to call XRE_IsUtilityProcess(), so roll
-    // our own.
-    if (GetGeckoProcessType() == GeckoProcessType_Utility) {
-      initFlags |= eDllBlocklistInitFlagIsUtilityProcess;
-    } else if (GetGeckoProcessType() == GeckoProcessType_Socket) {
-      initFlags |= eDllBlocklistInitFlagIsSocketProcess;
-    }
+    SetDllBlocklistProcessTypeFlags(initFlags, GetGeckoProcessType());
     DllBlocklist_Initialize(initFlags);
 #  endif  // HAS_DLL_BLOCKLIST
 #  if defined(XP_WIN) && defined(MOZ_SANDBOX)

browser/app/winlauncher/DllBlocklistInit.cpp

@@ -142,11 +142,7 @@ static LauncherVoidResultWithLineInfo InitializeDllBlocklistOOPInternal(
     newFlags |= eDllBlocklistInitFlagIsChildProcess;
   }
 
-  if (aProcessType == GeckoProcessType_Utility) {
-    newFlags |= eDllBlocklistInitFlagIsUtilityProcess;
-  } else if (aProcessType == GeckoProcessType_Socket) {
-    newFlags |= eDllBlocklistInitFlagIsSocketProcess;
-  }
+  SetDllBlocklistProcessTypeFlags(newFlags, aProcessType);
 
   LauncherVoidResult writeResult =
       aTransferMgr.Transfer(&gBlocklistInitFlags, &newFlags, sizeof(newFlags));

browser/app/winlauncher/freestanding/DllBlocklist.cpp

@@ -191,6 +191,11 @@ static BlockAction CheckBlockInfo(const DllBlockInfo* aInfo,
     return BlockAction::Allow;
   }
 
+  if ((aInfo->mFlags & DllBlockInfo::GPU_PROCESSES_ONLY) &&
+      !(gBlocklistInitFlags & eDllBlocklistInitFlagIsGPUProcess)) {
+    return BlockAction::Allow;
+  }
+
   if ((aInfo->mFlags & DllBlockInfo::BROWSER_PROCESS_ONLY) &&
       (gBlocklistInitFlags & eDllBlocklistInitFlagIsChildProcess)) {
     return BlockAction::Allow;

ipc/app/MozillaRuntimeMain.cpp

@@ -83,13 +83,7 @@ int main(int argc, char* argv[]) {
 
 #ifdef HAS_DLL_BLOCKLIST
     uint32_t initFlags = eDllBlocklistInitFlagIsChildProcess;
-    // This is too early in launch to call XRE_IsUtilityProcess(), so roll
-    // our own.
-    if (GetGeckoProcessType() == GeckoProcessType_Utility) {
-      initFlags |= eDllBlocklistInitFlagIsUtilityProcess;
-    } else if (GetGeckoProcessType() == GeckoProcessType_Socket) {
-      initFlags |= eDllBlocklistInitFlagIsSocketProcess;
-    }
+    SetDllBlocklistProcessTypeFlags(initFlags, GetGeckoProcessType());
     DllBlocklist_Initialize(initFlags);
 #endif
 

toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.cpp

@@ -470,6 +470,11 @@ static NTSTATUS NTAPI patched_LdrLoadDll(PWCHAR filePath, PULONG flags,
         goto continue_loading;
       }
 
+      if ((info->mFlags & DllBlockInfo::GPU_PROCESSES_ONLY) &&
+          !(sInitFlags & eDllBlocklistInitFlagIsGPUProcess)) {
+        goto continue_loading;
+      }
+
       if ((info->mFlags & DllBlockInfo::BROWSER_PROCESS_ONLY) &&
           (sInitFlags & eDllBlocklistInitFlagIsChildProcess)) {
         goto continue_loading;

toolkit/xre/dllservices/mozglue/WindowsDllBlocklist.h

@@ -12,18 +12,31 @@
 #  include <windows.h>
 #  include "CrashAnnotations.h"
 #  include "mozilla/Attributes.h"
+#  include "mozilla/ProcessType.h"
 #  include "mozilla/Types.h"
 
 #  define HAS_DLL_BLOCKLIST
 
 enum DllBlocklistInitFlags {
   eDllBlocklistInitFlagDefault = 0,
-  eDllBlocklistInitFlagIsChildProcess = 1,
-  eDllBlocklistInitFlagWasBootstrapped = 2,
-  eDllBlocklistInitFlagIsUtilityProcess = 4,
-  eDllBlocklistInitFlagIsSocketProcess = 8
+  eDllBlocklistInitFlagIsChildProcess = 1 << 0,
+  eDllBlocklistInitFlagWasBootstrapped = 1 << 1,
+  eDllBlocklistInitFlagIsUtilityProcess = 1 << 2,
+  eDllBlocklistInitFlagIsSocketProcess = 1 << 3,
+  eDllBlocklistInitFlagIsGPUProcess = 1 << 4,
 };
 
+inline void SetDllBlocklistProcessTypeFlags(uint32_t& aFlags,
+                                            GeckoProcessType aProcessType) {
+  if (aProcessType == GeckoProcessType_Utility) {
+    aFlags |= eDllBlocklistInitFlagIsUtilityProcess;
+  } else if (aProcessType == GeckoProcessType_Socket) {
+    aFlags |= eDllBlocklistInitFlagIsSocketProcess;
+  } else if (aProcessType == GeckoProcessType_GPU) {
+    aFlags |= eDllBlocklistInitFlagIsGPUProcess;
+  }
+}
+
 // Only available from within firefox.exe
 #  if !defined(IMPL_MFBT) && !defined(MOZILLA_INTERNAL_API)
 extern uint32_t gBlocklistInitFlags;

toolkit/xre/dllservices/mozglue/WindowsDllBlocklistDefs.in

@@ -4,14 +4,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file exposes five lists:
-# ALL_PROCESSES, BROWSER_PROCESS, CHILD_PROCESSES, UTILITY_PROCESSES,
-# and SOCKET_PROCESSES
+# This file exposes six lists:
+# ALL_PROCESSES, BROWSER_PROCESS, CHILD_PROCESSES, GPU_PROCESSES,
+# UTILITY_PROCESSES, and SOCKET_PROCESSES
 #
 # In addition, each of those lists supports a special variant for test-only
 # entries:
 # ALL_PROCESSES_TESTS, BROWSER_PROCESS_TESTS, CHILD_PROCESSES_TESTS,
-# UTILITY_PROCESSES_TESTS, and SOCKET_PROCESSES_TESTS
+# GPU_PROCESSES_TESTS, UTILITY_PROCESSES_TESTS, and SOCKET_PROCESSES_TESTS
 #
 # Choose the list that is applicable to the applicable process type(s) for your
 # DLL block.
@@ -330,6 +330,14 @@ CHILD_PROCESSES += [
     DllBlocklistEntry("hmpalert.dll", (3, 8, 8, 889)),
 ]
 
+GPU_PROCESSES += [
+]
+
+GPU_PROCESSES_TESTS += [
+    # DLLs used by TestDllBlocklist* gTests
+    DllBlocklistEntry("testdllblocklist_gpuprocessonly.dll", ALL_VERSIONS),
+]
+
 SOCKET_PROCESSES += [
     # Causes crashes in the socket process, bug 1760668
     DllBlocklistEntry("ipseng64.dll", (17, 2, 6, 25)),

toolkit/xre/dllservices/mozglue/WindowsDllBlocklistInfo.h

@@ -30,9 +30,10 @@ struct DllBlockInfoT {
   // If the USE_TIMESTAMP flag is set, then we use the timestamp from
   // the IMAGE_FILE_HEADER in lieu of a version number.
   //
-  // If the UTILITY_PROCESSES_ONLY or SOCKET_PROCESSES_ONLY flags are
-  // set, the dll will only be blocked for these specific child
-  // processes. Note that these are a subset of CHILD_PROCESSES_ONLY.
+  // If the UTILITY_PROCESSES_ONLY, SOCKET_PROCESSES_ONLY, or
+  // GPU_PROCESSES_ONLY flags are set, the dll will only be blocked
+  // for these specific child processes. Note that these are a subset
+  // of CHILD_PROCESSES_ONLY. These flags cannot be combined.
   enum Flags {
     FLAGS_DEFAULT = 0,
     BLOCK_WIN7_AND_OLDER = 1 << 0,
@@ -43,6 +44,7 @@ struct DllBlockInfoT {
     REDIRECT_TO_NOOP_ENTRYPOINT = 1 << 5,
     UTILITY_PROCESSES_ONLY = 1 << 6,
     SOCKET_PROCESSES_ONLY = 1 << 7,
+    GPU_PROCESSES_ONLY = 1 << 8,
   } mFlags;
 
   bool IsVersionBlocked(const uint64_t aOther) const {

toolkit/xre/dllservices/mozglue/gen_dll_blocklist_defs.py

@@ -52,7 +52,7 @@ DLL_BLOCKLIST_DEFINITIONS_BEGIN_NAMED(gBlockedInprocDlls)
 
 """
 
-# These flag names should match the ones defined in WindowsDllBlocklistCommon.h
+# These flag names should match the ones defined in WindowsDllBlocklistInfo.h
 FLAGS_DEFAULT = "FLAGS_DEFAULT"
 BLOCK_WIN8_AND_OLDER = "BLOCK_WIN8_AND_OLDER"
 BLOCK_WIN7_AND_OLDER = "BLOCK_WIN7_AND_OLDER"
@@ -63,6 +63,7 @@ SUBSTITUTE_LSP_PASSTHROUGH = "SUBSTITUTE_LSP_PASSTHROUGH"
 REDIRECT_TO_NOOP_ENTRYPOINT = "REDIRECT_TO_NOOP_ENTRYPOINT"
 UTILITY_PROCESSES_ONLY = "UTILITY_PROCESSES_ONLY"
 SOCKET_PROCESSES_ONLY = "SOCKET_PROCESSES_ONLY"
+GPU_PROCESSES_ONLY = "GPU_PROCESSES_ONLY"
 
 # Only these flags are available in the input script
 INPUT_ONLY_FLAGS = {
@@ -102,6 +103,7 @@ ALL_DEFINITION_LISTS = (
     "ALL_PROCESSES",
     "BROWSER_PROCESS",
     "CHILD_PROCESSES",
+    "GPU_PROCESSES",
     "UTILITY_PROCESSES",
     "SOCKET_PROCESSES",
 )
@@ -372,6 +374,7 @@ GENERATED_BLOCKLIST_FILES = [
         flagspec={
             "BROWSER_PROCESS": {BROWSER_PROCESS_ONLY},
             "CHILD_PROCESSES": {CHILD_PROCESSES_ONLY},
+            "GPU_PROCESSES": {GPU_PROCESSES_ONLY},
             "UTILITY_PROCESSES": {UTILITY_PROCESSES_ONLY},
             "SOCKET_PROCESSES": {SOCKET_PROCESSES_ONLY},
         },
@@ -382,6 +385,7 @@ GENERATED_BLOCKLIST_FILES = [
         flagspec={
             "BROWSER_PROCESS": {BROWSER_PROCESS_ONLY},
             "CHILD_PROCESSES": {CHILD_PROCESSES_ONLY},
+            "GPU_PROCESSES": {GPU_PROCESSES_ONLY},
             "UTILITY_PROCESSES": {UTILITY_PROCESSES_ONLY},
             "SOCKET_PROCESSES": {SOCKET_PROCESSES_ONLY},
         },

toolkit/xre/dllservices/tests/gtest/TestDLLBlocklist.cpp

@@ -80,6 +80,17 @@ TEST(TestDllBlocklist, AllowDllByVersion)
   EXPECT_TRUE(!!::GetModuleHandleW(kLeafName.get()));
 }
 
+TEST(TestDllBlocklist, GPUProcessOnly_AllowInMainProcess)
+{
+  constexpr auto kLeafName = u"TestDllBlocklist_GPUProcessOnly.dll"_ns;
+  nsString dllPath = GetFullPath(kLeafName);
+
+  nsModuleHandle hDll(::LoadLibraryW(dllPath.get()));
+
+  EXPECT_TRUE(!!hDll);
+  EXPECT_TRUE(!!::GetModuleHandleW(kLeafName.get()));
+}
+
 TEST(TestDllBlocklist, SocketProcessOnly_AllowInMainProcess)
 {
   constexpr auto kLeafName = u"TestDllBlocklist_SocketProcessOnly.dll"_ns;

toolkit/xre/dllservices/tests/gtest/TestDllBlocklist_GPUProcessOnly/TestDllBlocklist_GPUProcessOnly.cpp

@@ -0,0 +1,7 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <windows.h>
+
+BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD aReason, LPVOID) { return TRUE; }

toolkit/xre/dllservices/tests/gtest/TestDllBlocklist_GPUProcessOnly/moz.build

@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIST_INSTALL = False
+
+SharedLibrary("TestDllBlocklist_GPUProcessOnly")
+
+UNIFIED_SOURCES = [
+    "TestDllBlocklist_GPUProcessOnly.cpp",
+]
+
+if CONFIG["COMPILE_ENVIRONMENT"]:
+    TEST_HARNESS_FILES.gtest += ["!TestDllBlocklist_GPUProcessOnly.dll"]

toolkit/xre/dllservices/tests/gtest/moz.build

@@ -20,6 +20,7 @@ LOCAL_INCLUDES += [
 
 TEST_DIRS += [
     "TestDllBlocklist_AllowByVersion",
+    "TestDllBlocklist_GPUProcessOnly",
     "TestDllBlocklist_MatchByName",
     "TestDllBlocklist_MatchByVersion",
     "TestDllBlocklist_NoOpEntryPoint",