From 35ab5765346f0f961ef65935580243d05c64c7c8 Mon Sep 17 00:00:00 2001 From: Hannes Verschore Date: Fri, 16 May 2014 12:37:13 +0200 Subject: [PATCH] Bug 911738 - IonMonkey: Recompile function when a non-inlined function gets hot enough to inline, r=jandem --- js/src/jit/CodeGenerator.cpp | 32 +++++++++++++--- js/src/jit/Ion.cpp | 56 +++++++++++++--------------- js/src/jit/Ion.h | 2 +- js/src/jit/IonBuilder.cpp | 24 +++++++++++- js/src/jit/IonBuilder.h | 4 +- js/src/jit/IonOptimizationLevels.cpp | 1 + js/src/jit/IonOptimizationLevels.h | 9 +++++ js/src/jit/MIR.h | 38 +++++++++++++++++-- js/src/jit/VMFunctions.cpp | 16 +++++++- js/src/jit/VMFunctions.h | 1 + 10 files changed, 139 insertions(+), 44 deletions(-) diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 1a79f7b0aa57..51245092fb3f 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -4008,6 +4008,7 @@ CodeGenerator::emitDebugResultChecks(LInstruction *ins) default: return true; } + return true; } #endif @@ -7459,10 +7460,18 @@ CodeGenerator::link(JSContext *cx, types::CompilerConstraintList *constraints) // Check to make sure we didn't have a mid-build invalidation. If so, we // will trickle to jit::Compile() and return Method_Skipped. + uint32_t warmUpCount = script->getWarmUpCount(); types::RecompileInfo recompileInfo; if (!types::FinishCompilation(cx, script, executionMode, constraints, &recompileInfo)) return true; + // IonMonkey could have inferred better type information during + // compilation. Since adding the new information to the actual type + // information can reset the usecount, increase it back to what it was + // before. + if (warmUpCount > script->getWarmUpCount()) + script->incWarmUpCounter(warmUpCount - script->getWarmUpCount()); + uint32_t scriptFrameSize = frameClass_ == FrameSizeClass::None() ? frameDepth_ : FrameSizeClass::FromDepth(frameDepth_).frameSize(); @@ -9720,18 +9729,31 @@ CodeGenerator::visitAsmJSInterruptCheck(LAsmJSInterruptCheck *lir) typedef bool (*RecompileFn)(JSContext *); static const VMFunction RecompileFnInfo = FunctionInfo(Recompile); +typedef bool (*ForcedRecompileFn)(JSContext *); +static const VMFunction ForcedRecompileFnInfo = FunctionInfo(ForcedRecompile); + bool CodeGenerator::visitRecompileCheck(LRecompileCheck *ins) { Label done; Register tmp = ToRegister(ins->scratch()); - OutOfLineCode *ool = oolCallVM(RecompileFnInfo, ins, (ArgList()), StoreRegisterTo(tmp)); + OutOfLineCode *ool; + if (ins->mir()->forceRecompilation()) + ool = oolCallVM(ForcedRecompileFnInfo, ins, (ArgList()), StoreRegisterTo(tmp)); + else + ool = oolCallVM(RecompileFnInfo, ins, (ArgList()), StoreRegisterTo(tmp)); // Check if warm-up counter is high enough. - masm.movePtr(ImmPtr(ins->mir()->script()->addressOfWarmUpCounter()), tmp); - Address ptr(tmp, 0); - masm.add32(Imm32(1), tmp); - masm.branch32(Assembler::BelowOrEqual, ptr, Imm32(ins->mir()->recompileThreshold()), &done); + AbsoluteAddress warmUpCount = AbsoluteAddress(ins->mir()->script()->addressOfWarmUpCounter()); + if (ins->mir()->increaseWarmUpCounter()) { + masm.load32(warmUpCount, tmp); + masm.add32(Imm32(1), tmp); + masm.store32(tmp, warmUpCount); + masm.branch32(Assembler::BelowOrEqual, tmp, Imm32(ins->mir()->recompileThreshold()), &done); + } else { + masm.branch32(Assembler::BelowOrEqual, warmUpCount, Imm32(ins->mir()->recompileThreshold()), + &done); + } // Check if not yet recompiling. CodeOffsetLabel label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp); diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index 10f82ef79e01..13b005764d21 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -2221,7 +2221,7 @@ GetOptimizationLevel(HandleScript script, jsbytecode *pc, ExecutionMode executio static MethodStatus Compile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode *osrPc, - bool constructing, ExecutionMode executionMode) + bool constructing, ExecutionMode executionMode, bool forceRecompile = false) { MOZ_ASSERT(jit::IsIonEnabled(cx)); MOZ_ASSERT(jit::IsBaselineEnabled(cx)); @@ -2258,35 +2258,17 @@ Compile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode if (!scriptIon->method()) return Method_CantCompile; - MethodStatus failedState = Method_Compiled; - - // If we keep failing to enter the script due to an OSR pc mismatch, - // recompile with the right pc. - if (osrPc && script->ionScript()->osrPc() != osrPc) { - uint32_t count = script->ionScript()->incrOsrPcMismatchCounter(); - if (count <= js_JitOptions.osrPcMismatchesBeforeRecompile) - return Method_Skipped; - - failedState = Method_Skipped; - } - // Don't recompile/overwrite higher optimized code, // with a lower optimization level. - if (optimizationLevel < scriptIon->optimizationLevel()) - return failedState; - - if (optimizationLevel == scriptIon->optimizationLevel() && - (!osrPc || script->ionScript()->osrPc() == osrPc)) - { - return failedState; - } + if (optimizationLevel <= scriptIon->optimizationLevel() && !forceRecompile) + return Method_Compiled; // Don't start compiling if already compiling if (scriptIon->isRecompiling()) - return failedState; + return Method_Compiled; if (osrPc) - script->ionScript()->resetOsrPcMismatchCounter(); + scriptIon->resetOsrPcMismatchCounter(); recompile = true; } @@ -2305,11 +2287,8 @@ Compile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode } // Compilation succeeded or we invalidated right away or an inlining/alloc abort - if (HasIonScript(script, executionMode)) { - if (osrPc && script->ionScript()->osrPc() != osrPc) - return Method_Skipped; + if (HasIonScript(script, executionMode)) return Method_Compiled; - } return Method_Skipped; } @@ -2347,6 +2326,16 @@ jit::CanEnterAtBranch(JSContext *cx, JSScript *script, BaselineFrame *osrFrame, return Method_CantCompile; } + // By default a recompilation doesn't happen on osr mismatch. + // Decide if we want to force a recompilation if this happens too much. + bool force = false; + if (script->hasIonScript() && pc != script->ionScript()->osrPc()) { + uint32_t count = script->ionScript()->incrOsrPcMismatchCounter(); + if (count <= js_JitOptions.osrPcMismatchesBeforeRecompile) + return Method_Skipped; + force = true; + } + // Attempt compilation. // - Returns Method_Compiled if the right ionscript is present // (Meaning it was present or a sequantial compile finished) @@ -2354,13 +2343,20 @@ jit::CanEnterAtBranch(JSContext *cx, JSScript *script, BaselineFrame *osrFrame, // (This means a background thread compilation with that pc could have started or not.) RootedScript rscript(cx, script); MethodStatus status = Compile(cx, rscript, osrFrame, pc, osrFrame->isConstructing(), - SequentialExecution); + SequentialExecution, force); if (status != Method_Compiled) { if (status == Method_CantCompile) ForbidCompilation(cx, script); return status; } + // Return the compilation was skipped when the osr pc wasn't adjusted. + // This can happen when there was still an IonScript available and a + // background compilation started, but hasn't finished yet. + // Or when we didn't force a recompile. + if (pc != script->ionScript()->osrPc()) + return Method_Skipped; + return Method_Compiled; } @@ -2461,14 +2457,14 @@ jit::CompileFunctionForBaseline(JSContext *cx, HandleScript script, BaselineFram MethodStatus jit::Recompile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode *osrPc, - bool constructing) + bool constructing, bool force) { MOZ_ASSERT(script->hasIonScript()); if (script->ionScript()->isRecompiling()) return Method_Compiled; MethodStatus status = - Compile(cx, script, osrFrame, osrPc, constructing, SequentialExecution); + Compile(cx, script, osrFrame, osrPc, constructing, SequentialExecution, force); if (status != Method_Compiled) { if (status == Method_CantCompile) ForbidCompilation(cx, script); diff --git a/js/src/jit/Ion.h b/js/src/jit/Ion.h index a46ece40dc54..a2c2fc66ea6a 100644 --- a/js/src/jit/Ion.h +++ b/js/src/jit/Ion.h @@ -93,7 +93,7 @@ MethodStatus CanEnterInParallel(JSContext *cx, HandleScript script); MethodStatus Recompile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode *osrPc, - bool constructing); + bool constructing, bool force); enum IonExecStatus { diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index 2cd15b8cd808..3bbbac9fa8e2 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -4454,7 +4454,9 @@ IonBuilder::makeInliningDecision(JSFunction *target, CallInfo &callInfo) !targetScript->baselineScript()->ionCompiledOrInlined() && info().executionMode() != DefinitePropertiesAnalysis) { - return DontInline(targetScript, "Vetoed: callee is insufficiently hot."); + JitSpew(JitSpew_Inlining, "Cannot inline %s:%u: callee is insufficiently hot.", + targetScript->filename(), targetScript->lineno()); + return InliningDecision_WarmUpCountTooLow; } } @@ -4489,6 +4491,7 @@ IonBuilder::selectInliningTargets(ObjectVector &targets, CallInfo &callInfo, Boo case InliningDecision_Error: return false; case InliningDecision_DontInline: + case InliningDecision_WarmUpCountTooLow: inlineable = false; break; case InliningDecision_Inline: @@ -4664,6 +4667,8 @@ IonBuilder::inlineCallsite(ObjectVector &targets, ObjectVector &originals, return InliningStatus_Error; case InliningDecision_DontInline: return InliningStatus_NotInlined; + case InliningDecision_WarmUpCountTooLow: + return InliningStatus_WarmUpCountTooLow; case InliningDecision_Inline: break; } @@ -5278,6 +5283,7 @@ IonBuilder::jsop_funcall(uint32_t argc) case InliningDecision_Error: return false; case InliningDecision_DontInline: + case InliningDecision_WarmUpCountTooLow: break; case InliningDecision_Inline: if (target->isInterpreted()) @@ -5418,6 +5424,7 @@ IonBuilder::jsop_funapplyarguments(uint32_t argc) case InliningDecision_Error: return false; case InliningDecision_DontInline: + case InliningDecision_WarmUpCountTooLow: break; case InliningDecision_Inline: if (target->isInterpreted()) @@ -5488,6 +5495,14 @@ IonBuilder::jsop_call(uint32_t argc, bool constructing) if (targets.length() == 1) target = &targets[0]->as(); + if (target && status == InliningStatus_WarmUpCountTooLow) { + MRecompileCheck *check = + MRecompileCheck::New(alloc(), target->nonLazyScript(), + optimizationInfo().inliningRecompileThreshold(), + MRecompileCheck::RecompileCheck_Inlining); + current->add(check); + } + return makeCall(target, callInfo, hasClones); } @@ -6504,7 +6519,9 @@ IonBuilder::insertRecompileCheck() OptimizationLevel nextLevel = js_IonOptimizations.nextLevel(curLevel); const OptimizationInfo *info = js_IonOptimizations.get(nextLevel); uint32_t warmUpThreshold = info->compilerWarmUpThreshold(topBuilder->script()); - current->add(MRecompileCheck::New(alloc(), topBuilder->script(), warmUpThreshold)); + MRecompileCheck *check = MRecompileCheck::New(alloc(), topBuilder->script(), warmUpThreshold, + MRecompileCheck::RecompileCheck_OptimizationLevel); + current->add(check); } JSObject * @@ -9425,6 +9442,7 @@ IonBuilder::getPropTryCommonGetter(bool *emitted, MDefinition *obj, PropertyName switch (status) { case InliningStatus_Error: return false; + case InliningStatus_WarmUpCountTooLow: case InliningStatus_NotInlined: break; case InliningStatus_Inlined: @@ -9441,6 +9459,7 @@ IonBuilder::getPropTryCommonGetter(bool *emitted, MDefinition *obj, PropertyName case InliningDecision_Error: return false; case InliningDecision_DontInline: + case InliningDecision_WarmUpCountTooLow: break; case InliningDecision_Inline: inlineable = true; @@ -9865,6 +9884,7 @@ IonBuilder::setPropTryCommonSetter(bool *emitted, MDefinition *obj, case InliningDecision_Error: return false; case InliningDecision_DontInline: + case InliningDecision_WarmUpCountTooLow: break; case InliningDecision_Inline: if (!inlineScriptedCall(callInfo, commonSetter)) diff --git a/js/src/jit/IonBuilder.h b/js/src/jit/IonBuilder.h index 594217e459c4..95b2bb6114d4 100644 --- a/js/src/jit/IonBuilder.h +++ b/js/src/jit/IonBuilder.h @@ -666,6 +666,7 @@ class IonBuilder { InliningStatus_Error, InliningStatus_NotInlined, + InliningStatus_WarmUpCountTooLow, InliningStatus_Inlined }; @@ -673,7 +674,8 @@ class IonBuilder { InliningDecision_Error, InliningDecision_Inline, - InliningDecision_DontInline + InliningDecision_DontInline, + InliningDecision_WarmUpCountTooLow }; static InliningDecision DontInline(JSScript *targetScript, const char *reason); diff --git a/js/src/jit/IonOptimizationLevels.cpp b/js/src/jit/IonOptimizationLevels.cpp index 9aeaee5871d4..6b55ba77e0f4 100644 --- a/js/src/jit/IonOptimizationLevels.cpp +++ b/js/src/jit/IonOptimizationLevels.cpp @@ -42,6 +42,7 @@ OptimizationInfo::initNormalOptimizationInfo() smallFunctionMaxInlineDepth_ = 10; compilerWarmUpThreshold_ = 1000; inliningWarmUpThresholdFactor_ = 0.125; + inliningRecompileThresholdFactor_ = 4; } void diff --git a/js/src/jit/IonOptimizationLevels.h b/js/src/jit/IonOptimizationLevels.h index 2b5b6c6c1492..213dfe2ff034 100644 --- a/js/src/jit/IonOptimizationLevels.h +++ b/js/src/jit/IonOptimizationLevels.h @@ -108,6 +108,11 @@ class OptimizationInfo // are inlined, as a fraction of compilerWarmUpThreshold. double inliningWarmUpThresholdFactor_; + // How many invocations or loop iterations are needed before a function + // is hot enough to recompile the outerScript to inline that function, + // as a multiplication of inliningWarmUpThreshold. + uint32_t inliningRecompileThresholdFactor_; + OptimizationInfo() { } @@ -194,6 +199,10 @@ class OptimizationInfo compilerWarmUpThreshold = js_JitOptions.forcedDefaultIonWarmUpThreshold; return compilerWarmUpThreshold * inliningWarmUpThresholdFactor_; } + + uint32_t inliningRecompileThreshold() const { + return inliningWarmUpThreshold() * inliningRecompileThresholdFactor_; + } }; class OptimizationInfos diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index f76194c1fb29..a7f6338aa49c 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -11205,21 +11205,45 @@ class MHasClass // outermost script (i.e. not the inlined script). class MRecompileCheck : public MNullaryInstruction { + public: + enum RecompileCheckType { + RecompileCheck_OptimizationLevel, + RecompileCheck_Inlining + }; + + private: JSScript *script_; uint32_t recompileThreshold_; + bool forceRecompilation_; + bool increaseWarmUpCounter_; - MRecompileCheck(JSScript *script, uint32_t recompileThreshold) + MRecompileCheck(JSScript *script, uint32_t recompileThreshold, RecompileCheckType type) : script_(script), recompileThreshold_(recompileThreshold) { + switch (type) { + case RecompileCheck_OptimizationLevel: + forceRecompilation_ = false; + increaseWarmUpCounter_ = true; + break; + case RecompileCheck_Inlining: + forceRecompilation_ = true; + increaseWarmUpCounter_ = false; + break; + default: + MOZ_CRASH("Unexpected recompile check type"); + } + setGuard(); } public: INSTRUCTION_HEADER(RecompileCheck); - static MRecompileCheck *New(TempAllocator &alloc, JSScript *script_, uint32_t recompileThreshold) { - return new(alloc) MRecompileCheck(script_, recompileThreshold); + static MRecompileCheck *New(TempAllocator &alloc, JSScript *script_, uint32_t recompileThreshold, + RecompileCheckType type) + { + return new(alloc) MRecompileCheck(script_, recompileThreshold, type); } JSScript *script() const { @@ -11230,6 +11254,14 @@ class MRecompileCheck : public MNullaryInstruction return recompileThreshold_; } + bool forceRecompilation() const { + return forceRecompilation_; + } + + bool increaseWarmUpCounter() const { + return increaseWarmUpCounter_; + } + AliasSet getAliasSet() const { return AliasSet::None(); } diff --git a/js/src/jit/VMFunctions.cpp b/js/src/jit/VMFunctions.cpp index 154a00746100..ff24ea90ab1d 100644 --- a/js/src/jit/VMFunctions.cpp +++ b/js/src/jit/VMFunctions.cpp @@ -1049,7 +1049,7 @@ StringReplace(JSContext *cx, HandleString string, HandleString pattern, HandleSt } bool -Recompile(JSContext *cx) +RecompileImpl(JSContext *cx, bool force) { MOZ_ASSERT(cx->currentlyRunningInJit()); JitActivationIterator activations(cx->runtime()); @@ -1065,13 +1065,25 @@ Recompile(JSContext *cx) if (!IsIonEnabled(cx)) return true; - MethodStatus status = Recompile(cx, script, nullptr, nullptr, isConstructing); + MethodStatus status = Recompile(cx, script, nullptr, nullptr, isConstructing, force); if (status == Method_Error) return false; return true; } +bool +ForcedRecompile(JSContext *cx) +{ + return RecompileImpl(cx, /* force = */ true); +} + +bool +Recompile(JSContext *cx) +{ + return RecompileImpl(cx, /* force = */ false); +} + bool SetDenseElement(JSContext *cx, HandleNativeObject obj, int32_t index, HandleValue value, bool strict) diff --git a/js/src/jit/VMFunctions.h b/js/src/jit/VMFunctions.h index 271eaabd2649..649faffdb4df 100644 --- a/js/src/jit/VMFunctions.h +++ b/js/src/jit/VMFunctions.h @@ -724,6 +724,7 @@ JSObject *CreateDerivedTypedObj(JSContext *cx, HandleObject descr, bool ArraySpliceDense(JSContext *cx, HandleObject obj, uint32_t start, uint32_t deleteCount); bool Recompile(JSContext *cx); +bool ForcedRecompile(JSContext *cx); JSString *RegExpReplace(JSContext *cx, HandleString string, HandleObject regexp, HandleString repl); JSString *StringReplace(JSContext *cx, HandleString string, HandleString pattern,