Bug 1129492 - Remove X11 access from the Linux content process sandbox. r=gcp,jgilbert

Background: The X11 protocol has a very permissive security model;
clients have essentially full access to the windows of other clients,
and to global resources like input devices.  Previously, our sandbox
policy for content processes needed to allow access to the X server;
this limited its effectiveness against a dedicated attacker.

This patch turns on the `security.sandbox.content.headless` pref added
in bug 1640345, which removes the sandbox policy rules that allowed
making new X11 connections, as well as opening the Xauthority file,
reading hardware info needed by Mesa, etc.  It also runs content
processes in headless mode (whence the name) so they won't connect to a
display server at startup.

This also removes access to the Wayland compositor: the sandbox policy
never allowed that (as of when socket connections became default-deny),
but now content processes won't connect to it at startup.  Wayland is
more capability-oriented so this is less significant for security, but at
a minimum it removes unnecessary attack surface.

Note that if the `webgl.out-of-process` pref is turned off, WebGL
will break unless `security.sandbox.content.headless` is also turned
off.  (Similarly, `widget.non-native-theme.enabled` is needed to render
scrollbars and form controls in content.)  As a result, this patch
adjusts the job definitions used by CI to test in-process WebGL so that
that they will continue to work.

Differential Revision: https://phabricator.services.mozilla.com/D138613

modules/libpref/init/StaticPrefList.yaml

@@ -11691,14 +11691,15 @@
 #endif
 
 #if defined(XP_LINUX) && defined(MOZ_SANDBOX)
-  # Run content processes in headless mode and disallow connections to
-  # the X server.  Experimental; breaks WebGL and Flash, and requires
-  # `widget.non-native-theme.enabled` and `widget.remote-look-and-feel`.
-  # Changing it requires a restart because sandbox policy information dependent
-  # on it is cached.  See bug 1640345 for details.
+  # Run content processes in headless mode and disallow
+  # connections to the X server.  Requires:
+  # * `webgl.out-of-process` (or else WebGL breaks)
+  # * `widget.non-native-theme.enabled` (scrollbars & form controls)
+  # Changing it requires a restart because sandbox policy information
+  # dependent on it is cached.  See bug 1640345 for details.
 - name: security.sandbox.content.headless
   type: bool
-  value: false
+  value: true
   mirror: once
 #endif
 

security/sandbox/common/test/SandboxTestingChildTests.h

@@ -23,7 +23,11 @@
 #    include <sys/syscall.h>
 #    include <sys/un.h>
 #    include "mozilla/ProcInfo_linux.h"
-#  endif  // XP_LINUX
+#    ifdef MOZ_X11
+#      include "X11/Xlib.h"
+#      include "X11UndefineNone.h"
+#    endif  // MOZ_X11
+#  endif    // XP_LINUX
 #  include <sys/socket.h>
 #  include <sys/stat.h>
 #  include <sys/types.h>
@@ -266,7 +270,22 @@ void RunTestsContent(SandboxTestingChild* child) {
                        return fd;
                      });
   }
-#  endif  // XP_LINUX
+
+#    ifdef MOZ_X11
+  // Check that X11 access is blocked (bug 1129492).
+  // This will fail if security.sandbox.content.headless is turned off.
+  if (PR_GetEnv("DISPLAY")) {
+    Display* disp = XOpenDisplay(nullptr);
+
+    child->SendReportTestResults(
+        "x11_access"_ns, !disp,
+        disp ? "XOpenDisplay succeeded"_ns : "XOpenDisplay failed"_ns);
+    if (disp) {
+      XCloseDisplay(disp);
+    }
+  }
+#    endif  // MOZ_X11
+#  endif    // XP_LINUX
 
 #  ifdef XP_MACOSX
   // Test that content processes can not connect to the macOS window server.

taskcluster/ci/test/mochitest.yml

@@ -764,6 +764,7 @@ mochitest-media:
                 default: true
         extra-options:
             - --setpref=webgl.out-of-process=false
+            - --setpref=security.sandbox.content.headless=false
     tier:
         by-test-platform:
             .*ccov.*: 2
@@ -910,6 +911,7 @@ mochitest-webgl1-core:
         mochitest-flavor: plain
         extra-options:
             - --setpref=webgl.out-of-process=false
+            - --setpref=security.sandbox.content.headless=false
 
 mochitest-webgl1-ext:
     description: "Mochitest webgl1-ext run"
@@ -984,6 +986,7 @@ mochitest-webgl1-ext:
         chunked: true
         extra-options:
             - --setpref=webgl.out-of-process=false
+            - --setpref=security.sandbox.content.headless=false
 
 mochitest-webgl2-core:
     description: "Mochitest webgl2-core run"
@@ -1125,6 +1128,7 @@ mochitest-webgl2-ext:
         chunked: true
         extra-options:
             - --setpref=webgl.out-of-process=false
+            - --setpref=security.sandbox.content.headless=false
 
 mochitest-webgl2-deqp:
     description: "Mochitest webgl2-deqp run"
@@ -1146,6 +1150,7 @@ mochitest-webgl2-deqp:
         chunked: true
         extra-options:
             - --setpref=webgl.out-of-process=false
+            - --setpref=security.sandbox.content.headless=false
 
 mochitest-webgpu:
     description: "Mochitest webgpu run"

taskcluster/ci/test/variants.yml

@@ -151,6 +151,7 @@ webgl-ipc:
         mozharness:
             extra-options:
                 - "--setpref=webgl.out-of-process=true"
+                - "--setpref=security.sandbox.content.headless=true"
 
 devtools-no-eft:
     description: "{description} with DevTools EFT disabled"