Bug 1635451 - Attempt to start WebGL even in headless mode. r=jgilbert

Currently we return an error when creating a WebGL context in headless
mode, but our WebGL implementation renders to an offscreen context, so
in theory it could work normally in a headless browser, and in practice
it already does work on some OSes.  This patch removes that check; the
attempt to use GL may fail, in which case we'll return an error to
content.

The main purpose of this patch is to run content processes with headless
mode set in an otherwise non-headless browser, but it should also be
useful for fully headless mode.  Comments in bug NNNNNNN indicate that
this change should be sufficient for headless WebGL on Windows and MacOS,
although it may not have been extensively tested.

Linux is more complicated.  The EGL/X11 backend manages its own
connection to the X server (indirectly via the EGL library); a later
patch in this series allows doing that in GLX mode as well.  Our Wayland
support can't do this yet, but it should be possible.

This patch also modifies the Linux sandbox policy so that content
processes can connect to a local X server (via the file broker) even when
the parent process is in headless mode.

Differential Revision: https://phabricator.services.mozilla.com/D118721

dom/canvas/WebGLContext.cpp

@@ -248,16 +248,6 @@ bool WebGLContext::CreateAndInitGL(
     bool forceEnabled, std::vector<FailureReason>* const out_failReasons) {
   const FuncScope funcScope(*this, "<Create>");
 
-  // Can't use WebGL in headless mode.
-  if (gfxPlatform::IsHeadless()) {
-    FailureReason reason;
-    reason.info =
-        "Can't use WebGL in headless mode (https://bugzil.la/1375585).";
-    out_failReasons->push_back(reason);
-    GenerateWarning("%s", reason.info.BeginReading());
-    return false;
-  }
-
   // WebGL2 is separately blocked:
   if (IsWebGL2() && !forceEnabled) {
     FailureReason reason;

security/sandbox/linux/broker/SandboxBrokerPolicyFactory.cpp

@@ -523,14 +523,20 @@ void SandboxBrokerPolicyFactory::InitContentPolicy() {
     policy->AddPath(SandboxBroker::MAY_CONNECT, bumblebeeSocket);
 
 #if defined(MOZ_WIDGET_GTK) && defined(MOZ_X11)
-    // Allow local X11 connections, for Primus and VirtualGL to contact
-    // the secondary X server. No exception for Wayland.
-    if (mozilla::widget::GdkIsX11Display()) {
+    // Allow local X11 connections, for several purposes:
+    //
+    // * for content processes to use WebGL when the browser is in headless
+    //   mode, by opening the X display if/when needed
+    //
+    // * if Primus or VirtualGL is used, to contact the secondary X server
+    static const bool kIsX11 =
+        !mozilla::widget::GdkIsWaylandDisplay() && PR_GetEnv("DISPLAY");
+    if (kIsX11) {
       policy->AddPrefix(SandboxBroker::MAY_CONNECT, "/tmp/.X11-unix/X");
-    }
-    if (const auto xauth = PR_GetEnv("XAUTHORITY")) {
+      if (auto* const xauth = PR_GetEnv("XAUTHORITY")) {
         policy->AddPath(rdonly, xauth);
       }
+    }
 #endif
   }
 

security/sandbox/linux/launch/SandboxLaunch.cpp

@@ -64,18 +64,25 @@ namespace mozilla {
 //
 // (Longer-term we intend to either proxy or remove X11 access from
 // content processes, at which point this will stop being an issue.)
-static bool IsDisplayLocal() {
+static bool IsGraphicsOkWithoutNetwork() {
   // For X11, check whether the parent's connection is a Unix-domain
   // socket.  This is done instead of trying to parse the display name
   // because an empty hostname (e.g., ":0") will fall back to TCP in
   // case of failure to connect using Unix-domain sockets.
 #ifdef MOZ_X11
   // First, ensure that the parent process's graphics are initialized.
-  Unused << gfxPlatform::GetPlatform();
+  DebugOnly<gfxPlatform*> gfxPlatform = gfxPlatform::GetPlatform();
 
   const auto display = gdk_display_get_default();
-  if (NS_WARN_IF(display == nullptr)) {
-    return false;
+  if (!display) {
+    // In this case, the browser is headless, but WebGL could still
+    // try to use X11.  However, WebGL isn't supported with remote
+    // X11, and in any case these connections are made after sandbox
+    // startup (lazily when WebGL is used), so they aren't being done
+    // directly by the process anyway.  (For local X11, they're
+    // brokered.)
+    MOZ_ASSERT(gfxPlatform->IsHeadless());
+    return true;
   }
   if (mozilla::widget::GdkIsX11Display(display)) {
     const int xSocketFd = ConnectionNumber(GDK_DISPLAY_XDISPLAY(display));
@@ -331,7 +338,8 @@ void SandboxLaunchPrepare(GeckoProcessType aType,
         // local-ness is cached because it won't change.)
         static const bool canCloneNet =
             StaticPrefs::security_sandbox_content_headless_AtStartup() ||
-            (IsDisplayLocal() && !PR_GetEnv("RENDERDOC_CAPTUREOPTS"));
+            (IsGraphicsOkWithoutNetwork() &&
+             !PR_GetEnv("RENDERDOC_CAPTUREOPTS"));
 
         if (canCloneNet) {
           flags |= CLONE_NEWNET;