Bug 1458161 - Hook rust OOM handler. r=froydnj

OOM rust crashes are currently not identified as such in crash reports
because rust libstd handles the OOMs and panics itself.

There are unstable ways to hook into this, which unfortunately are under
active changes in rust 1.27, but we're currently on 1.24 and 1.27 is not
released yet. The APIs didn't change between 1.24 and 1.26, so it's
fine-ish to use them as long as we limit their use to those versions.

As long as the Firefox versions we ship (as opposed to downstream) use
the "right" version of rust, we're good to go.

The APIs are in their phase of stabilization, so there shouldn't be too
many variants of the code to support.

--HG--
extra : rebase_source : 08a85aa102b24380b1f6764effffcc909ef3191b

build/moz.configure/rust.configure

@@ -254,6 +254,8 @@ set_config('RUST_TARGET_ENV_NAME', rust_target_env_name)
 # This is used for putting source info into symbol files.
 set_config('RUSTC_COMMIT', depends(rustc_info)(lambda i: i.commit))
 
+set_config('RUSTC_VERSION', depends(rustc_info)(lambda i: str(i.version)))
+
 # Until we remove all the other Rust checks in old-configure.
 add_old_configure_assignment('RUSTC', rustc)
 add_old_configure_assignment('RUST_TARGET', rust_target_triple)

toolkit/library/gtest/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
 
 [dependencies]
 mp4parse-gtest = { path = "../../../../dom/media/gtest" }

toolkit/library/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
 
 [dependencies]
 gkrust-shared = { path = "shared" }

toolkit/library/rust/gkrust-features.mozbuild

@@ -24,3 +24,8 @@ if (CONFIG['OS_ARCH'] == 'Linux' and CONFIG['OS_TARGET'] != 'Android') or CONFIG
     gkrust_features += ['cubeb-remoting']
 
 gkrust_features += ['no-static-ideograph-encoder-tables']
+
+# See details in toolkit/library/rust/shared/lib.rs
+# A string test is not the best thing, but it works well enough here.
+if CONFIG['RUSTC_VERSION'] < "1.27":
+    gkrust_features += ['oom_with_global_alloc']

toolkit/library/rust/shared/Cargo.toml

@@ -38,6 +38,7 @@ cubeb_pulse_rust = ["cubeb-sys", "cubeb-pulse"]
 gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 no-static-ideograph-encoder-tables = ["encoding_c/no-static-ideograph-encoder-tables", "encoding_glue/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = []
 
 [lib]
 path = "lib.rs"

toolkit/library/rust/shared/build.rs

@@ -0,0 +1,8 @@
+fn main() {
+    // This is a rather awful thing to do, but we're only doing it on
+    // versions of rustc, >= 1.24 < 1.27, that are not going to change
+    // the unstable APIs we use from under us (1.26 being a beta as of
+    // writing, and close to release).
+    #[cfg(feature = "oom_with_global_alloc")]
+    println!("cargo:rustc-env=RUSTC_BOOTSTRAP=1");
+}

toolkit/library/rust/shared/lib.rs

@@ -2,6 +2,9 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+#![cfg_attr(feature = "oom_with_global_alloc",
+            feature(global_allocator, alloc, alloc_system, allocator_api))]
+
 #[cfg(feature="servo")]
 extern crate geckoservo;
 
@@ -159,3 +162,59 @@ pub extern "C" fn get_rust_panic_reason(reason: *mut *const c_char, length: *mut
         }
     }
 }
+
+// Wrap the rust system allocator to override the OOM handler, redirecting
+// to Gecko's, which interacts with the crash reporter.
+// This relies on unstable APIs that have not changed between 1.24 and 1.27.
+// In 1.27, the API changed, so we'll need to adapt to those changes before
+// we can ship with 1.27. As of writing, there might still be further changes
+// to those APIs before 1.27 is released, so we wait for those.
+#[cfg(feature = "oom_with_global_alloc")]
+mod global_alloc {
+    extern crate alloc;
+    extern crate alloc_system;
+
+    use self::alloc::allocator::{Alloc, AllocErr, Layout};
+    use self::alloc_system::System;
+
+    pub struct GeckoHeap;
+
+    extern "C" {
+        fn GeckoHandleOOM(size: usize) -> !;
+    }
+
+    unsafe impl<'a> Alloc for &'a GeckoHeap {
+        unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
+            System.alloc(layout)
+        }
+
+        unsafe fn dealloc(&mut self, ptr: *mut u8, layout: Layout) {
+            System.dealloc(ptr, layout)
+        }
+
+        fn oom(&mut self, e: AllocErr) -> ! {
+            match e {
+                AllocErr::Exhausted { request } => unsafe { GeckoHandleOOM(request.size()) },
+                _ => System.oom(e),
+            }
+        }
+
+        unsafe fn realloc(
+            &mut self,
+            ptr: *mut u8,
+            layout: Layout,
+            new_layout: Layout,
+        ) -> Result<*mut u8, AllocErr> {
+            System.realloc(ptr, layout, new_layout)
+        }
+
+        unsafe fn alloc_zeroed(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
+            System.alloc_zeroed(layout)
+        }
+    }
+
+}
+
+#[cfg(feature = "oom_with_global_alloc")]
+#[global_allocator]
+static HEAP: global_alloc::GeckoHeap = global_alloc::GeckoHeap;

toolkit/xre/nsAppRunner.cpp

@@ -231,6 +231,8 @@
 #include "mozilla/CodeCoverageHandler.h"
 #endif
 
+#include "mozilla/mozalloc_oom.h"
+
 extern uint32_t gRestartMode;
 extern void InstallSignalHandlers(const char *ProgramName);
 
@@ -5369,3 +5371,10 @@ XRE_EnableSameExecutableForContentProc() {
     mozilla::ipc::GeckoChildProcessHost::EnableSameExecutableForContentProc();
   }
 }
+
+// Because rust doesn't handle weak symbols, this function wraps the weak
+// malloc_handle_oom for it.
+extern "C" void
+GeckoHandleOOM(size_t size) {
+  mozalloc_handle_oom(size);
+}