Bug 1744460 part 2 - Update woff2 RLBoxSandboxPool to track minimum sandbox size r=bholley

Depends on D133009

Differential Revision: https://phabricator.services.mozilla.com/D133158

config/external/rlbox_wasm2c_sandbox/moz.yaml

@@ -9,8 +9,8 @@ origin:
   description: rlbox integration for the wasm2c sandboxed code
   url: https://github.com/PLSysSec/rlbox_wasm2c_sandbox
 
-  release: commit 287fca460d8df2673f16d834cca2d240caf28417 (2021-11-19T04:33:46Z).
-  revision: 287fca460d8df2673f16d834cca2d240caf28417
+  release: commit 54e8469095e7929c66aeecdc26f23f502b986218 (2021-12-08T08:12:13Z).
+  revision: 54e8469095e7929c66aeecdc26f23f502b986218
 
   license: MIT
   license-file: LICENSE
@@ -34,3 +34,4 @@ vendoring:
     - LibrarySandbox.md
     - README.md
 
+

gfx/ots/RLBoxWOFF2Host.cpp

@@ -49,19 +49,19 @@ tainted_woff2<BrotliDecoderResult> RLBoxBrotliDecoderDecompressCallback(
   return res;
 }
 
-UniquePtr<RLBoxSandboxDataBase> RLBoxWOFF2SandboxPool::CreateSandboxData() {
+UniquePtr<RLBoxSandboxDataBase> RLBoxWOFF2SandboxPool::CreateSandboxData(uint64_t aSize) {
   // Create woff2 sandbox
   auto sandbox = MakeUnique<rlbox_sandbox_woff2>();
 
-#ifdef MOZ_WASM_SANDBOXING_WOFF2
-  bool createOK = sandbox->create_sandbox(/* infallible = */ false);
+#if defined(MOZ_WASM_SANDBOXING_WOFF2)
+  bool createOK = sandbox->create_sandbox(/* infallible = */ false, aSize);
 #else
   bool createOK = sandbox->create_sandbox();
 #endif
   NS_ENSURE_TRUE(createOK, nullptr);
 
   UniquePtr<RLBoxWOFF2SandboxData> sbxData =
-      MakeUnique<RLBoxWOFF2SandboxData>(std::move(sandbox));
+      MakeUnique<RLBoxWOFF2SandboxData>(aSize, std::move(sandbox));
 
   // Register brotli callback
   sbxData->mDecompressCallback = sbxData->Sandbox()->register_callback(
@@ -80,9 +80,10 @@ void RLBoxWOFF2SandboxPool::Initalize(size_t aDelaySeconds) {
   ClearOnShutdown(&RLBoxWOFF2SandboxPool::sSingleton);
 }
 
-RLBoxWOFF2SandboxData::RLBoxWOFF2SandboxData(
+RLBoxWOFF2SandboxData::RLBoxWOFF2SandboxData(uint64_t aSize,
     mozilla::UniquePtr<rlbox_sandbox_woff2> aSandbox)
-    : mSandbox(std::move(aSandbox)) {
+    : mozilla::RLBoxSandboxDataBase(aSize),
+      mSandbox(std::move(aSandbox)) {
   MOZ_COUNT_CTOR(RLBoxWOFF2SandboxData);
 }
 
@@ -154,7 +155,12 @@ bool RLBoxProcessWOFF2(ots::FontFile* aHeader, ots::OTSStream* aOutput,
   uint32_t expectedSize = ComputeWOFF2FinalSize(aData, aLength);
   NS_ENSURE_TRUE(expectedSize > 0, false);
 
-  auto sandboxPoolData = RLBoxWOFF2SandboxPool::sSingleton->PopOrCreate();
+  // The sandbox should have space for the input, output and misc allocations
+  // To account for misc allocations, we'll set the sandbox size to
+  // input + output + 33% extra
+
+  const uint64_t expectedSandboxSize = static_cast<uint64_t>(1.33 * (aLength + expectedSize));
+  auto sandboxPoolData = RLBoxWOFF2SandboxPool::sSingleton->PopOrCreate(expectedSandboxSize);
   NS_ENSURE_TRUE(sandboxPoolData, false);
 
   const auto* sandboxData =

gfx/ots/RLBoxWOFF2Host.h

@@ -33,7 +33,7 @@ class RLBoxWOFF2SandboxData : public mozilla::RLBoxSandboxDataBase {
   friend class RLBoxWOFF2SandboxPool;
 
  public:
-  RLBoxWOFF2SandboxData(mozilla::UniquePtr<rlbox_sandbox_woff2> aSandbox);
+  RLBoxWOFF2SandboxData(uint64_t aSize, mozilla::UniquePtr<rlbox_sandbox_woff2> aSandbox);
   ~RLBoxWOFF2SandboxData();
 
   rlbox_sandbox_woff2* Sandbox() const { return mSandbox.get(); }

gfx/ots/RLBoxWOFF2Types.h

@@ -29,7 +29,7 @@ class RLBoxWOFF2SandboxPool : public mozilla::RLBoxSandboxPool {
   static void Initalize(size_t aDelaySeconds = 10);
 
  protected:
-  mozilla::UniquePtr<mozilla::RLBoxSandboxDataBase> CreateSandboxData()
+  mozilla::UniquePtr<mozilla::RLBoxSandboxDataBase> CreateSandboxData(uint64_t aSize)
       override;
   ~RLBoxWOFF2SandboxPool() = default;
 };

parser/htmlparser/nsExpatDriver.cpp

@@ -1342,12 +1342,12 @@ nsExpatDriver::ConsumeToken(nsScanner& aScanner, bool& aFlushTokens) {
 }
 
 mozilla::UniquePtr<mozilla::RLBoxSandboxDataBase>
-RLBoxExpatSandboxPool::CreateSandboxData() {
+RLBoxExpatSandboxPool::CreateSandboxData(uint64_t aSize) {
   // Create expat sandbox
   auto sandbox = mozilla::MakeUnique<rlbox_sandbox_expat>();
 
 #ifdef MOZ_WASM_SANDBOXING_EXPAT
-  bool create_ok = sandbox->create_sandbox(/* infallible = */ false);
+  bool create_ok = sandbox->create_sandbox(/* infallible = */ false, aSize);
 #else
   bool create_ok = sandbox->create_sandbox();
 #endif
@@ -1355,7 +1355,7 @@ RLBoxExpatSandboxPool::CreateSandboxData() {
   NS_ENSURE_TRUE(create_ok, nullptr);
 
   mozilla::UniquePtr<RLBoxExpatSandboxData> sbxData =
-      mozilla::MakeUnique<RLBoxExpatSandboxData>();
+      mozilla::MakeUnique<RLBoxExpatSandboxData>(aSize);
 
   // Register callbacks common to both system and non-system principals
   sbxData->mHandleXMLDeclaration =

parser/htmlparser/nsExpatDriver.h

@@ -154,7 +154,10 @@ class RLBoxExpatSandboxData : public mozilla::RLBoxSandboxDataBase {
   friend class nsExpatDriver;
 
  public:
-  MOZ_COUNTED_DEFAULT_CTOR(RLBoxExpatSandboxData);
+  explicit RLBoxExpatSandboxData(uint64_t aSize)
+      : mozilla::RLBoxSandboxDataBase(aSize) {
+    MOZ_COUNT_CTOR(RLBoxExpatSandboxData);
+  }
   ~RLBoxExpatSandboxData();
   rlbox_sandbox_expat* Sandbox() const { return mSandbox.get(); }
   // After getting a sandbox from the pool we need to register the

parser/htmlparser/nsRLBoxExpatDriver.h

@@ -19,8 +19,8 @@ class RLBoxExpatSandboxPool : public mozilla::RLBoxSandboxPool {
   static void Initialize(size_t aDelaySeconds = 10);
 
  protected:
-  mozilla::UniquePtr<mozilla::RLBoxSandboxDataBase> CreateSandboxData()
-      override;
+  mozilla::UniquePtr<mozilla::RLBoxSandboxDataBase> CreateSandboxData(
+      uint64_t aSize) override;
   ~RLBoxExpatSandboxPool() = default;
 };
 

third_party/rlbox_wasm2c_sandbox/include/rlbox_wasm2c_sandbox.hpp

@@ -417,6 +417,35 @@ __attribute__((weak))
     return power;
   }
 
+public:
+
+#define WASM_PAGE_SIZE 65536
+#define WASM_HEAP_MAX_ALLOWED_PAGES 65536
+#define WASM_MAX_HEAP (static_cast<uint64_t>(1) << 32)
+  static uint64_t rlbox_wasm2c_get_adjusted_heap_size(uint64_t heap_size)
+  {
+    if (heap_size == 0){
+      return 0;
+    }
+
+    if(heap_size <= WASM_PAGE_SIZE) {
+      return WASM_PAGE_SIZE;
+    } else if (heap_size >= WASM_MAX_HEAP) {
+      return WASM_MAX_HEAP;
+    }
+
+    return next_power_of_two(static_cast<uint32_t>(heap_size));
+  }
+
+  static uint64_t rlbox_wasm2c_get_heap_page_count(uint64_t heap_size)
+  {
+    const uint64_t pages = heap_size / WASM_PAGE_SIZE;
+    return pages;
+  }
+#undef WASM_MAX_HEAP
+#undef WASM_HEAP_MAX_ALLOWED_PAGES
+#undef WASM_PAGE_SIZE
+
 protected:
 
 #ifndef RLBOX_USE_STATIC_CALLS
@@ -524,23 +553,9 @@ protected:
       sandbox_info.wasm_rt_sys_init();
     });
 
-#define WASM_PAGE_SIZE 65536
-#define WASM_HEAP_MAX_ALLOWED_PAGES 65536
-#define WASM_MAX_HEAP (static_cast<uint64_t>(1) << 32)
-    if (override_max_heap_size != 0){
-      if(override_max_heap_size < WASM_PAGE_SIZE) {
-        override_max_heap_size = WASM_PAGE_SIZE;
-      } else if (override_max_heap_size > WASM_MAX_HEAP) {
-        override_max_heap_size = WASM_MAX_HEAP;
-      } else {
-        override_max_heap_size = next_power_of_two(override_max_heap_size);
-      }
-    }
-    const uint64_t override_max_wasm_pages = override_max_heap_size / WASM_PAGE_SIZE;
+    override_max_heap_size = rlbox_wasm2c_get_adjusted_heap_size(override_max_heap_size);
+    const uint64_t override_max_wasm_pages = rlbox_wasm2c_get_heap_page_count(override_max_heap_size);
     FALLIBLE_DYNAMIC_CHECK(infallible, override_max_wasm_pages <= 65536, "Wasm allows a max heap size of 4GB");
-#undef WASM_MAX_HEAP
-#undef WASM_HEAP_MAX_ALLOWED_PAGES
-#undef WASM_PAGE_SIZE
 
     sandbox = sandbox_info.create_wasm2c_sandbox(static_cast<uint32_t>(override_max_wasm_pages));
     FALLIBLE_DYNAMIC_CHECK(infallible, sandbox != nullptr, "Sandbox could not be created");

xpcom/base/RLBoxSandboxPool.cpp

@@ -8,6 +8,10 @@
 #include "mozilla/ClearOnShutdown.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/RLBoxSandboxPool.h"
+#ifdef MOZ_USING_WASM_SANDBOXING
+#  include "mozilla/rlbox/rlbox_config.h"
+#  include "mozilla/rlbox/rlbox_wasm2c_sandbox.hpp"
+#endif
 
 using namespace mozilla;
 
@@ -63,18 +67,49 @@ void RLBoxSandboxPool::Push(UniquePtr<RLBoxSandboxDataBase> sbxData) {
   }
 }
 
-UniquePtr<RLBoxSandboxPoolData> RLBoxSandboxPool::PopOrCreate() {
+UniquePtr<RLBoxSandboxPoolData> RLBoxSandboxPool::PopOrCreate(
+    uint64_t aMinSize) {
   MutexAutoLock lock(mMutex);
 
   UniquePtr<RLBoxSandboxDataBase> sbxData;
+
   if (!mPool.IsEmpty()) {
-    sbxData = mPool.PopLastElement();
+    const int64_t lastIndex = ReleaseAssertedCast<int64_t>(mPool.Length()) - 1;
+    for (int64_t i = lastIndex; i >= 0; i--) {
+      if (mPool[i]->mSize >= aMinSize) {
+        sbxData = std::move(mPool[i]);
+        mPool.RemoveElementAt(i);
+
+        // If we reuse a sandbox from the pool, reset the timer to clear the
+        // pool
         CancelTimer();
         if (!mPool.IsEmpty()) {
           StartTimer();
         }
-  } else {
-    sbxData = CreateSandboxData();
+        break;
+      }
+    }
+  }
+
+  if (!sbxData) {
+#ifdef MOZ_USING_WASM_SANDBOXING
+    // RLBox's wasm sandboxes have a limited platform dependent capacity. We
+    // track this capacity in this pool. Note the noop sandboxes have no
+    // capacity limit but this design assumes that all sandboxes use the wasm
+    // sandbox limit.
+    const uint64_t defaultCapacityForSandbox =
+        wasm_rt_get_default_max_linear_memory_size();
+    const uint64_t minSandboxCapacity =
+        std::max(aMinSize, defaultCapacityForSandbox);
+    const uint64_t chosenAdjustedCapacity =
+        rlbox::rlbox_wasm2c_sandbox::rlbox_wasm2c_get_adjusted_heap_size(
+            minSandboxCapacity);
+#else
+    // If sandboxing is disabled altogether we just set a limit of 4gb.
+    // This is not actually enforced by the noop sandbox.
+    const uint64_t chosenAdjustedCapacity = static_cast<uint64_t>(1) << 32;
+#endif
+    sbxData = CreateSandboxData(chosenAdjustedCapacity);
     NS_ENSURE_TRUE(sbxData, nullptr);
   }
 

xpcom/base/RLBoxSandboxPool.h

@@ -44,13 +44,15 @@ class RLBoxSandboxPool : public nsITimerCallback, public nsINamed {
         mMutex("RLBoxSandboxPool::mMutex"){};
 
   void Push(UniquePtr<RLBoxSandboxDataBase> sbx);
-  // PopOrCreate() returns a sandbox from the pool if the pool is not empty and
+  // PopOrCreate returns a sandbox from the pool if the pool is not empty and
   // tries to mint a new one otherwise. If creating a new sandbox fails, the
-  // function returns a nullptr.
-  UniquePtr<RLBoxSandboxPoolData> PopOrCreate();
+  // function returns a nullptr. The parameter aMinSize is the minimum size of
+  // the sandbox memory.
+  UniquePtr<RLBoxSandboxPoolData> PopOrCreate(uint64_t aMinSize = 0);
 
  protected:
-  virtual UniquePtr<RLBoxSandboxDataBase> CreateSandboxData() = 0;
+  // CreateSandboxData takes a parameter which is the size of the sandbox memory
+  virtual UniquePtr<RLBoxSandboxDataBase> CreateSandboxData(uint64_t aSize) = 0;
   virtual ~RLBoxSandboxPool() = default;
 
  private:
@@ -68,6 +70,8 @@ class RLBoxSandboxPool : public nsITimerCallback, public nsINamed {
 // (e.g., callbacks).
 class RLBoxSandboxDataBase {
  public:
+  const uint64_t mSize;
+  explicit RLBoxSandboxDataBase(uint64_t aSize) : mSize(aSize) {}
   virtual ~RLBoxSandboxDataBase() = default;
 };