Bug 1390856 - Fix XDR helper thread error handling. r=nbp

Make sure not to return TranscodeResult_Ok if an OOM happens while
running XDR on a helper thread. Also fix OOM handling when computing
BuildID.

MozReview-Commit-ID: EPYNhFsclVG

--HG--
extra : rebase_source : c21e42a206e993a16c92cacda1c1aada9dd53b8e

js/src/jit-test/tests/xdr/bug1390856.js

@@ -0,0 +1,30 @@
+if (!('oomTest' in this))
+    quit();
+
+if (helperThreadCount() == 0)
+    quit();
+
+
+let THREAD_TYPE_PARSE = 4;
+
+// Test main thread encode/decode OOM
+oomTest(function() {
+    let t = cacheEntry(`function f() { function g() { }; return 3; };`);
+
+    evaluate(t, { sourceIsLazy: true, saveIncrementalBytecode: true });
+    evaluate(t, { sourceIsLazy: true, readBytecode: true });
+});
+
+// Test helper thread decode OOM
+let t = cacheEntry(`function f() { function g() { }; return 3; };`);
+evaluate(t, { sourceIsLazy: true, saveIncrementalBytecode: true });
+for (var i = 1; i < 100; ++i) {
+    try {
+        oomAtAllocation(i, THREAD_TYPE_PARSE);
+        offThreadDecodeScript(t);
+        runOffThreadDecodedScript();
+    }
+    catch (e) {
+        assertEq(e, "out of memory");
+    }
+}

js/src/jsscript.cpp

@@ -2090,10 +2090,12 @@ ScriptSource::xdrEncodeTopLevel(JSContext* cx, HandleScript script)
 
     RootedScript s(cx, script);
     if (!xdrEncoder_->codeScript(&s)) {
-        if (xdrEncoder_->resultCode() == JS::TranscodeResult_Throw)
-            return false;
-        // Encoding failures are reported by the xdrFinalizeEncoder function.
-        return true;
+        // On encoding failure, let failureCase destroy encoder and return true
+        // to avoid failing any currently executing script.
+        if (xdrEncoder_->resultCode() & JS::TranscodeResult_Failure)
+            return true;
+
+        return false;
     }
 
     failureCase.release();
@@ -2110,8 +2112,14 @@ ScriptSource::xdrEncodeFunction(JSContext* cx, HandleFunction fun, HandleScriptS
     });
 
     RootedFunction f(cx, fun);
-    if (!xdrEncoder_->codeFunction(&f, sourceObject))
+    if (!xdrEncoder_->codeFunction(&f, sourceObject)) {
+        // On encoding failure, let failureCase destroy encoder and return true
+        // to avoid failing any currently executing script.
+        if (xdrEncoder_->resultCode() & JS::TranscodeResult_Failure)
+            return true;
+
         return false;
+    }
 
     failureCase.release();
     return true;

js/src/vm/Xdr.cpp

@@ -31,11 +31,14 @@ template<XDRMode mode>
 void
 XDRState<mode>::postProcessContextErrors(JSContext* cx)
 {
-    if (!cx->helperThread() && cx->isExceptionPending()) {
-        MOZ_ASSERT(resultCode_ == JS::TranscodeResult_Ok ||
-                   resultCode_ == JS::TranscodeResult_Throw);
+    // NOTE: This should only be called on transcode failure. Not all failure
+    // paths call XDRState::fail(...), so we should update resultCode_ if it
+    // doesn't hold a specific transcode error.
+
+    if (resultCode_ & JS::TranscodeResult_Failure)
+        MOZ_ASSERT_IF(!cx->helperThread(), !cx->isExceptionPending());
+    else
         resultCode_ = JS::TranscodeResult_Throw;
-    }
 }
 
 template<XDRMode mode>
@@ -80,8 +83,11 @@ static bool
 VersionCheck(XDRState<mode>* xdr)
 {
     JS::BuildIdCharVector buildId;
-    if (!xdr->cx()->buildIdOp() || !xdr->cx()->buildIdOp()(&buildId))
-        return xdr->fail(JS::TranscodeResult_Failure_BadBuildId);
+    MOZ_ASSERT(xdr->cx()->buildIdOp());
+    if (!xdr->cx()->buildIdOp()(&buildId)) {
+        ReportOutOfMemory(xdr->cx());
+        return xdr->fail(JS::TranscodeResult_Throw);
+    }
     MOZ_ASSERT(!buildId.empty());
 
     uint32_t buildIdLength;