Bug 1205708: Check if validation failed before reporting helper thread failure in Odin; r=luke

--HG--
extra : commitid : wOOPXy1fXN
extra : rebase_source : f03aa9184a4cb390c50ffa49fe34852824e7c6ae

js/src/asmjs/AsmJSValidate.cpp

@@ -1304,8 +1304,12 @@ class MOZ_STACK_CLASS ModuleValidator
     }
 
     // Error handling.
+    bool hasAlreadyFailed() const {
+        return !!errorString_;
+    }
+
     bool failOffset(uint32_t offset, const char* str) {
-        MOZ_ASSERT(!errorString_);
+        MOZ_ASSERT(!hasAlreadyFailed());
         MOZ_ASSERT(errorOffset_ == UINT32_MAX);
         MOZ_ASSERT(str);
         errorOffset_ = offset;
@@ -1318,7 +1322,7 @@ class MOZ_STACK_CLASS ModuleValidator
     }
 
     bool failfVAOffset(uint32_t offset, const char* fmt, va_list ap) {
-        MOZ_ASSERT(!errorString_);
+        MOZ_ASSERT(!hasAlreadyFailed());
         MOZ_ASSERT(errorOffset_ == UINT32_MAX);
         MOZ_ASSERT(fmt);
         errorOffset_ = offset;
@@ -6786,10 +6790,14 @@ CheckFunctions(ModuleValidator& m, ScopedJSDeletePtr<ModuleCompileResults>* resu
     if (!CheckFunctionsParallel(m, group, results)) {
         CancelOutstandingJobs(group);
 
-        // If failure was triggered by a helper thread, report error.
-        if (void* maybeFunc = HelperThreadState().maybeAsmJSFailedFunction()) {
-            AsmFunction* func = reinterpret_cast<AsmFunction*>(maybeFunc);
-            return m.failOffset(func->srcBegin(), "allocation failure during compilation");
+        // If a validation error didn't occur on the main thread, either a
+        // syntax error occurred and will be signalled by the regular parser,
+        // or an error occurred on an helper thread.
+        if (!m.hasAlreadyFailed()) {
+            if (void* maybeFunc = HelperThreadState().maybeAsmJSFailedFunction()) {
+                AsmFunction* func = reinterpret_cast<AsmFunction*>(maybeFunc);
+                return m.failOffset(func->srcBegin(), "allocation failure during compilation");
+            }
         }
 
         // Otherwise, the error occurred on the main thread and was already reported.

js/src/jit-test/tests/asm.js/oom-helper-thread-plus-validation-error.js

@@ -0,0 +1,11 @@
+if (typeof oomAfterAllocations !== 'function' || typeof evaluate !== 'function')
+    quit();
+
+oomAfterAllocations(10, 2);
+evaluate(`function mod(stdlib, ffi, heap) {
+    "use asm";
+    function f3(k) {
+        k = k | 0;
+    }
+    function g3(k) {}
+}`);