Bug 1445003 - Detect RenderDoc and adjust the sandbox policy so it can work. r=gcp

RenderDoc, a graphics debugging tool, uses a preload library that
creates a listening socket (Internet-domain) early in startup and
accepts connections from the frontend.  If it's detected (via env vars),
we allow accept/accept4 (but not socket/bind/listen), and remain in
the parent process's network namespace so that other processes can
connect to the socket.

This doesn't change the sandbox policy if not running under RenderDoc.

MozReview-Commit-ID: 964RW4BFh4u

--HG--
extra : rebase_source : d4a954e68431d84fa2e0edea4171421a948794af

security/sandbox/linux/SandboxFilter.cpp

@@ -379,6 +379,7 @@ private:
   SandboxBrokerClient* mBroker;
   ContentProcessSandboxParams mParams;
   bool mAllowSysV;
+  bool mUsingRenderDoc;
 
   bool BelowLevel(int aLevel) const {
     return mParams.mLevel < aLevel;
@@ -744,6 +745,7 @@ public:
     : mBroker(aBroker)
     , mParams(Move(aParams))
     , mAllowSysV(PR_GetEnv("MOZ_SANDBOX_ALLOW_SYSV") != nullptr)
+    , mUsingRenderDoc(PR_GetEnv("RENDERDOC_CAPTUREOPTS") != nullptr)
     { }
 
   ~ContentSandboxPolicy() override = default;
@@ -790,6 +792,12 @@ public:
       const auto trapFn = aHasArgs ? ConnectTrap : ConnectTrapLegacy;
       return Some(AllowBelowLevel(4, Trap(trapFn, mBroker)));
     }
+    case SYS_ACCEPT:
+    case SYS_ACCEPT4:
+      if (mUsingRenderDoc) {
+        return Some(Allow());
+      }
+      return SandboxPolicyCommon::EvaluateSocketCall(aCall, aHasArgs);
     case SYS_RECV:
     case SYS_SEND:
     case SYS_GETSOCKOPT:

security/sandbox/linux/launch/SandboxLaunch.cpp

@@ -277,8 +277,9 @@ SandboxLaunchPrepare(GeckoProcessType aType,
       // Unshare network namespace if allowed by graphics; see
       // function definition above for details.  (The display
       // local-ness is cached because it won't change.)
-      static const bool isDisplayLocal = IsDisplayLocal();
-      if (isDisplayLocal) {
+      static const bool canCloneNet =
+        IsDisplayLocal() && !PR_GetEnv("RENDERDOC_CAPTUREOPTS");
+      if (canCloneNet) {
         flags |= CLONE_NEWNET;
       }
     }