Bug 1582954 - Use posix_fallocate if available to avoid lazy allocation for shared memory. r=jld

WebRender makes extensive use of shared memory buffers, particularly for
images decoded in the content process. These images can be arbitrarily
large, and there being insufficient memory for an allocation must be
handled gracefully.

On Linux, we will currently crash with a SIGBUS signal during image
decoding instead of just displaying the broken image tag. This is
because the pages backing the shared memory are only allocated when we
write to them. This blocks shipping WebRender on Linux.

This patch uses posix_fallocate to force the reservation of the pages,
and allows failing gracefully if they are unavailable.

Differential Revision: https://phabricator.services.mozilla.com/D80650

dom/media/ipc/RemoteDecoderChild.cpp

@@ -133,8 +133,14 @@ RefPtr<MediaDataDecoder::DecodePromise> RemoteDecoderChild::Decode(
                      aValue) {
                // We no longer need the ShmemBuffer as the data has been
                // processed by the parent.
-               for (auto&& mem : mems) {
-                 mRawFramePool.Put(ShmemBuffer(std::move(mem)));
+               if (self->CanSend()) {
+                 for (auto&& mem : mems) {
+                   mRawFramePool.Put(ShmemBuffer(std::move(mem)));
+                 }
+               } else {
+                 for (auto mem : mems) {
+                   self->DeallocShmem(mem);
+                 }
                }
 
                if (aValue.IsReject()) {

ipc/chromium/src/base/eintr_wrapper.h

@@ -43,10 +43,20 @@
       eintr_wrapper_result;                                 \
     })
 
+#  define HANDLE_RV_EINTR(x)                   \
+    ({                                         \
+      decltype(x) eintr_wrapper_result;        \
+      do {                                     \
+        eintr_wrapper_result = (x);            \
+      } while (eintr_wrapper_result == EINTR); \
+      eintr_wrapper_result;                    \
+    })
+
 #else
 
 #  define HANDLE_EINTR(x) (x)
 #  define IGNORE_EINTR(x) (x)
+#  define HANDLE_RV_EINTR(x) (x)
 
 #endif  // OS_POSIX
 

ipc/chromium/src/base/shared_memory_posix.cc

@@ -164,10 +164,41 @@ bool SharedMemory::CreateInternal(size_t size, bool freezeable) {
   }
 
   if (needs_truncate) {
-    if (HANDLE_EINTR(ftruncate(fd.get(), static_cast<off_t>(size))) != 0) {
-      CHROMIUM_LOG(WARNING) << "failed to set shm size: " << strerror(errno);
+#if defined(HAVE_POSIX_FALLOCATE)
+    // Using posix_fallocate will ensure that there's actually space for this
+    // file. Otherwise we end up with a sparse file that can give SIGBUS if we
+    // run out of space while writing to it.
+    int rv =
+        HANDLE_RV_EINTR(posix_fallocate(fd.get(), 0, static_cast<off_t>(size)));
+    if (rv != 0) {
+      if (rv == EOPNOTSUPP || rv == EINVAL || rv == ENODEV) {
+        // Some filesystems have trouble with posix_fallocate. For now, we must
+        // fallback ftruncate and accept the allocation failures like we do
+        // without posix_fallocate.
+        // See https://bugzilla.mozilla.org/show_bug.cgi?id=1618914
+        int fallocate_errno = rv;
+        rv = HANDLE_EINTR(ftruncate(fd.get(), static_cast<off_t>(size)));
+        if (rv != 0) {
+          CHROMIUM_LOG(WARNING) << "fallocate failed to set shm size: "
+                                << strerror(fallocate_errno);
+          CHROMIUM_LOG(WARNING)
+              << "ftruncate failed to set shm size: " << strerror(errno);
+          return false;
+        }
+      } else {
+        CHROMIUM_LOG(WARNING)
+            << "fallocate failed to set shm size: " << strerror(rv);
+        return false;
+      }
+    }
+#else
+    int rv = HANDLE_EINTR(ftruncate(fd.get(), static_cast<off_t>(size)));
+    if (rv != 0) {
+      CHROMIUM_LOG(WARNING)
+          << "ftruncate failed to set shm size: " << strerror(errno);
       return false;
     }
+#endif
   }
 
   mapped_file_ = fd.release();

security/sandbox/linux/SandboxFilter.cpp

@@ -570,6 +570,7 @@ class SandboxPolicyCommon : public SandboxPolicyBase {
         return Allow();
 
       CASES_FOR_ftruncate:
+      case __NR_fallocate:
         return mMayCreateShmem ? Allow() : InvalidSyscall();
 
         // Used by our fd/shm classes