Bug 1743928 - Allow DMA-Buf ioctls for RDD. r=stransky,alwu

In some cases (doing video decode on the CPU and uploading the result
with dmabuf) we'll need to use `DMA_BUF_IOCTL_SYNC` in the RDD process.
This patch allows that ioctl type ('b', used only by dmabuf and Android
binder; the latter doesn't apply on Desktop), for forward compatibility
with any future usage of dmabuf ioctls.

Differential Revision: https://phabricator.services.mozilla.com/D133715

security/sandbox/common/test/SandboxTestingChildTests.h

@@ -258,6 +258,12 @@ void RunTestsRDD(SandboxTestingChild* child) {
     return uname(&uts);
   });
 
+  child->ErrnoValueTest("ioctl_dma_buf"_ns, false, ENOTTY, [] {
+    // Apply the ioctl to the wrong kind of fd; it should fail with
+    // ENOTTY (rather than ENOSYS if it were blocked).
+    return ioctl(0, _IOW('b', 0, uint64_t), nullptr);
+  });
+
 #  endif  // XP_LINUX
 #else     // XP_UNIX
   child->ReportNoTests();

security/sandbox/linux/SandboxFilter.cpp

@@ -1806,9 +1806,13 @@ class RDDSandboxPolicy final : public SandboxPolicyCommon {
         auto shifted_type = request & kIoctlTypeMask;
         static constexpr unsigned long kDrmType =
             static_cast<unsigned long>('d') << _IOC_TYPESHIFT;
+        // Note: 'b' is also the Binder device on Android.
+        static constexpr unsigned long kDmaBufType =
+            static_cast<unsigned long>('b') << _IOC_TYPESHIFT;
 
-        // Allow DRI for VA-API
+        // Allow DRI and DMA-Buf for VA-API
         return If(shifted_type == kDrmType, Allow())
+            .ElseIf(shifted_type == kDmaBufType, Allow())
             .Else(SandboxPolicyCommon::EvaluateSyscall(sysno));
       }