diff --git a/js/src/jit-test/tests/xdr/bug1390856.js b/js/src/jit-test/tests/xdr/bug1390856.js new file mode 100644 index 000000000000..6b5cd28f72d8 --- /dev/null +++ b/js/src/jit-test/tests/xdr/bug1390856.js @@ -0,0 +1,30 @@ +if (!('oomTest' in this)) + quit(); + +if (helperThreadCount() == 0) + quit(); + + +let THREAD_TYPE_PARSE = 4; + +// Test main thread encode/decode OOM +oomTest(function() { + let t = cacheEntry(`function f() { function g() { }; return 3; };`); + + evaluate(t, { sourceIsLazy: true, saveIncrementalBytecode: true }); + evaluate(t, { sourceIsLazy: true, readBytecode: true }); +}); + +// Test helper thread decode OOM +let t = cacheEntry(`function f() { function g() { }; return 3; };`); +evaluate(t, { sourceIsLazy: true, saveIncrementalBytecode: true }); +for (var i = 1; i < 100; ++i) { + try { + oomAtAllocation(i, THREAD_TYPE_PARSE); + offThreadDecodeScript(t); + runOffThreadDecodedScript(); + } + catch (e) { + assertEq(e, "out of memory"); + } +} diff --git a/js/src/jsscript.cpp b/js/src/jsscript.cpp index 627624a0ebde..643cbba055a0 100644 --- a/js/src/jsscript.cpp +++ b/js/src/jsscript.cpp @@ -2090,10 +2090,12 @@ ScriptSource::xdrEncodeTopLevel(JSContext* cx, HandleScript script) RootedScript s(cx, script); if (!xdrEncoder_->codeScript(&s)) { - if (xdrEncoder_->resultCode() == JS::TranscodeResult_Throw) - return false; - // Encoding failures are reported by the xdrFinalizeEncoder function. - return true; + // On encoding failure, let failureCase destroy encoder and return true + // to avoid failing any currently executing script. + if (xdrEncoder_->resultCode() & JS::TranscodeResult_Failure) + return true; + + return false; } failureCase.release(); @@ -2110,8 +2112,14 @@ ScriptSource::xdrEncodeFunction(JSContext* cx, HandleFunction fun, HandleScriptS }); RootedFunction f(cx, fun); - if (!xdrEncoder_->codeFunction(&f, sourceObject)) + if (!xdrEncoder_->codeFunction(&f, sourceObject)) { + // On encoding failure, let failureCase destroy encoder and return true + // to avoid failing any currently executing script. + if (xdrEncoder_->resultCode() & JS::TranscodeResult_Failure) + return true; + return false; + } failureCase.release(); return true; diff --git a/js/src/vm/Xdr.cpp b/js/src/vm/Xdr.cpp index ddc25b17127d..ec184f31e0c7 100644 --- a/js/src/vm/Xdr.cpp +++ b/js/src/vm/Xdr.cpp @@ -31,11 +31,14 @@ template void XDRState::postProcessContextErrors(JSContext* cx) { - if (!cx->helperThread() && cx->isExceptionPending()) { - MOZ_ASSERT(resultCode_ == JS::TranscodeResult_Ok || - resultCode_ == JS::TranscodeResult_Throw); + // NOTE: This should only be called on transcode failure. Not all failure + // paths call XDRState::fail(...), so we should update resultCode_ if it + // doesn't hold a specific transcode error. + + if (resultCode_ & JS::TranscodeResult_Failure) + MOZ_ASSERT_IF(!cx->helperThread(), !cx->isExceptionPending()); + else resultCode_ = JS::TranscodeResult_Throw; - } } template @@ -80,8 +83,11 @@ static bool VersionCheck(XDRState* xdr) { JS::BuildIdCharVector buildId; - if (!xdr->cx()->buildIdOp() || !xdr->cx()->buildIdOp()(&buildId)) - return xdr->fail(JS::TranscodeResult_Failure_BadBuildId); + MOZ_ASSERT(xdr->cx()->buildIdOp()); + if (!xdr->cx()->buildIdOp()(&buildId)) { + ReportOutOfMemory(xdr->cx()); + return xdr->fail(JS::TranscodeResult_Throw); + } MOZ_ASSERT(!buildId.empty()); uint32_t buildIdLength;