Bug 1741525 - Update rlbox for correct app_ptr support on 32-bit platforms r=bholley

Differential Revision: https://phabricator.services.mozilla.com/D131311

third_party/rlbox/README-mozilla

@@ -1,7 +1,7 @@
 This directory contains the rlbox source from the upstream repo:
 https://github.com/PLSysSec/rlbox_sandboxing_api/
 
-Current version: [commit 543a85c32652df9088302e9130b6b4ff5b0c0584]
+Current version: [commit 7813eba669cdc20beeeacee230bf8cbb7433625c]
 
 UPDATING:
 

third_party/rlbox/include/rlbox_app_pointer.hpp

@@ -27,17 +27,16 @@ private:
   RLBOX_SHARED_LOCK(map_mutex);
 #endif
 
-  T_PointerType get_unused_index()
+  T_PointerType get_unused_index(T_PointerType max_ptr_val)
   {
-    constexpr auto max_val = std::numeric_limits<T_PointerTypeUnsigned>::max();
-    constexpr auto min_val = std::numeric_limits<T_PointerTypeUnsigned>::min();
-    for (T_PointerTypeUnsigned i = counter; i < max_val; i++) {
+    const auto max_val = (T_PointerTypeUnsigned) max_ptr_val;
+    for (T_PointerTypeUnsigned i = counter; i <= max_val; i++) {
       if (pointer_map.find(i) == pointer_map.end()) {
         counter = i + 1;
         return (T_PointerType)i;
       }
     }
-    for (T_PointerTypeUnsigned i = min_val; i < counter; i++) {
+    for (T_PointerTypeUnsigned i = 1; i < counter; i++) {
       if (pointer_map.find(i) == pointer_map.end()) {
         counter = i + 1;
         return (T_PointerType)i;
@@ -55,12 +54,12 @@ public:
     pointer_map[0] = nullptr;
   }
 
-  T_PointerType get_app_pointer_idx(void* ptr)
+  T_PointerType get_app_pointer_idx(void* ptr, T_PointerType max_ptr_val)
   {
 #ifndef RLBOX_SINGLE_THREADED_INVOCATIONS
     RLBOX_ACQUIRE_UNIQUE_GUARD(lock, map_mutex);
 #endif
-    T_PointerType idx = get_unused_index();
+    T_PointerType idx = get_unused_index(max_ptr_val);
     T_PointerTypeUnsigned idx_int = (T_PointerTypeUnsigned)idx;
     pointer_map[idx_int] = ptr;
     return idx;

third_party/rlbox/include/rlbox_sandbox.hpp

@@ -970,7 +970,8 @@ public:
   template<typename T>
   app_pointer<T*, T_Sbx> get_app_pointer(T* ptr)
   {
-    auto idx = app_ptr_map.get_app_pointer_idx((void*)ptr);
+    auto max_ptr = (typename T_Sbx::T_PointerType)(get_total_memory() - 1);
+    auto idx = app_ptr_map.get_app_pointer_idx((void*)ptr, max_ptr);
     auto idx_as_ptr = this->template impl_get_unsandboxed_pointer<T>(idx);
     // Right now we simply assume that any integer can be converted to a valid
     // pointer in the sandbox This may not be true for some sandboxing mechanism