зеркало из https://github.com/dotnet/llilc.git
Major change of runtest and checkdiff
This is a major change to runtest and checkdiff. First, it is changed to user pre-created baseline. Instead of huge LLVM IR baseline stored in GitHub that caused issues. Second, summary comparison is introduced. With one line per method, the runtest and checkdiff process is much more swift with summary only than previous approach. No dump runtest is also provided. Verbose dump functionality is still there. A normalization speed improvement is expected after this change. Third, LLVM IR dump level is introduced into LLILCJit. Right now it read env var directly. It will be updated when CLR Config is supported in LLILCJit. A GitHub wiki page is created to illustrate the usage. https://github.com/dotnet/llilc/wiki/Dev-Env-and-Test-Harness Below are Typical use cases: Developer Usage 1.Update your master to latest master 2.Build your baseline JIT: BuildAll or Build 3.Create a summary baseline: RunTest -Jit baseline-jit -Result SummaryBase -Dump Summary 4.Create a verbose baseline: RunTest -Jit baseline-jit -Result VerboseBase -Dump Verbose 5.Create your own branch 6.Do the new development or fix issues 7.Build your new JIT: Build 8.RunTest -Jit new-jit If failed with overall pass/failure and no dump, back to step 6 for fixing. 9.RunTest -Jit new-jit -Result SummaryDiff -Dump Summary 10.CheckDiff -Base SummaryBase -Diff SummaryDiff -Summary -UseDiffTool If failed with summary, find out which method is a new failure, back to step 6 for fixing. 11.RunTest -Jit new-jit -Result VerboseDiff -Dump Verbose 12.CheckDiff -Base VerboseBase -Diff VerboseDiff -UseDiffTool If any LLVM IR diff is not benign, back to step 6 for fixing. 13.Ready for a pull request. Lab PR Usage RunTest Lab PR usage is most simplified version. It only checks for overall pass/fail and does not keep any result. Lab Nightly Usage 1.RunTest -Jit last-known-good-jit -Result SummaryBase -Dump Summary or a summary result of running last-know-good-jit is kept in SummaryBase 2.RunTest -Jit latest-jit -Result SummaryDiff -Dump Summary 3.CheckDiff -Base SummaryBase -Diff SummaryDiff -Summary 4.If CheckDiff result is zero, mark latest-jit as last-known-good-jit. Result SummaryDiff could be kept as SummaryBase. 5.If CheckDiff result is not zero, send out failure notice to branch owner. Branch owner should analyze the case and bring relevant developer for fixing.
This commit is contained in:
Bengu Li
Родитель
7bb36962df
Коммит
34df455822
|
|
@ -22,6 +22,13 @@
|
|||
#include "llvm/Support/ManagedStatic.h"
|
||||
#include "llvm/Support/ThreadLocal.h"
|
||||
|
||||
/// \brief Enum for LLVM IR Dump Level
|
||||
enum LLVMDumpLevel {
|
||||
NODUMP, ///< Do not dump any LLVM IR or summary
|
||||
SUMMARY, ///< Only dump one line summary per method
|
||||
VERBOSE ///< Dump full LLVM IR and method summary
|
||||
};
|
||||
|
||||
struct LLILCJitPerThreadState;
|
||||
|
||||
/// \brief This struct holds per-jit request state.
|
||||
|
|
|
|||
|
|
@ -39,6 +39,24 @@
|
|||
|
||||
using namespace llvm;
|
||||
|
||||
// Get the LLVM IR dump level. For now this is done by directly
|
||||
// accessing environment variable. When CLR config support is
|
||||
// included, update it here.
|
||||
LLVMDumpLevel dumpLevel() {
|
||||
const char *LevelCStr = getenv("DUMPLLVMIR");
|
||||
if (LevelCStr) {
|
||||
std::string Level = LevelCStr;
|
||||
std::transform(Level.begin(), Level.end(), Level.begin(), ::toupper);
|
||||
if (Level.compare("VERBOSE") == 0) {
|
||||
return VERBOSE;
|
||||
}
|
||||
if (Level.compare("SUMMARY") == 0) {
|
||||
return SUMMARY;
|
||||
}
|
||||
}
|
||||
return NODUMP;
|
||||
}
|
||||
|
||||
// The one and only Jit Object.
|
||||
LLILCJit *LLILCJit::TheJit = nullptr;
|
||||
|
||||
|
|
@ -177,7 +195,9 @@ CorJitResult LLILCJit::compileMethod(ICorJitInfo *JitInfo,
|
|||
|
||||
// Now jit the method.
|
||||
CorJitResult Result = CORJIT_INTERNALERROR;
|
||||
Context.outputDebugMethodName();
|
||||
if (dumpLevel() == VERBOSE) {
|
||||
Context.outputDebugMethodName();
|
||||
}
|
||||
bool HasMethod = this->readMethod(&Context);
|
||||
|
||||
if (HasMethod) {
|
||||
|
|
@ -261,6 +281,8 @@ bool LLILCJit::readMethod(LLILCJitContext *JitContext) {
|
|||
return true;
|
||||
}
|
||||
|
||||
LLVMDumpLevel DumpLevel = dumpLevel();
|
||||
|
||||
LLILCJitPerThreadState *PerThreadState = State.get();
|
||||
GenIR Reader(JitContext, &PerThreadState->ClassTypeMap,
|
||||
&PerThreadState->ArrayTypeMap, &PerThreadState->FieldIndexMap);
|
||||
|
|
@ -270,7 +292,9 @@ bool LLILCJit::readMethod(LLILCJitContext *JitContext) {
|
|||
try {
|
||||
Reader.msilToIR();
|
||||
} catch (NotYetImplementedException &Nyi) {
|
||||
errs() << "Failed to read " << FuncName << '[' << Nyi.reason() << "]\n";
|
||||
if (DumpLevel >= SUMMARY) {
|
||||
errs() << "Failed to read " << FuncName << '[' << Nyi.reason() << "]\n";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -278,12 +302,18 @@ bool LLILCJit::readMethod(LLILCJitContext *JitContext) {
|
|||
bool IsOk = !verifyFunction(*Func, &dbgs());
|
||||
|
||||
if (IsOk) {
|
||||
errs() << "Successfully read " << FuncName << '\n';
|
||||
if (DumpLevel >= SUMMARY) {
|
||||
errs() << "Successfully read " << FuncName << '\n';
|
||||
}
|
||||
} else {
|
||||
errs() << "Read " << FuncName << " but failed verification\n";
|
||||
if (DumpLevel >= SUMMARY) {
|
||||
errs() << "Read " << FuncName << " but failed verification\n";
|
||||
}
|
||||
}
|
||||
|
||||
Func->dump();
|
||||
if (DumpLevel == VERBOSE) {
|
||||
Func->dump();
|
||||
}
|
||||
|
||||
JitContext->MethodName = FuncName;
|
||||
|
||||
|
|
|
|||
|
|
@ -932,27 +932,81 @@ function Global:CheckFailure([string]$Arch="x64", [string]$Build="Release")
|
|||
#
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
function Global:RunTest([string]$Arch="x64", [string]$Build="Release")
|
||||
function Global:RunTest
|
||||
{
|
||||
Param(
|
||||
[string]$Arch="x64",
|
||||
[string]$Build="Release",
|
||||
[string]$Jit="",
|
||||
[string]$Result="",
|
||||
[string]$Dump="NoDump"
|
||||
)
|
||||
|
||||
$CoreCLRTestAssets = CoreCLRTestAssets
|
||||
$CoreCLRRuntime = CoreCLRRuntime
|
||||
$CoreCLRVersion = CoreCLRVersion
|
||||
$LLILCTest = LLILCTest
|
||||
$LLILCTestResult = LLILCTestResult
|
||||
$CoreCLRTestTargetBinaries = CoreCLRTestTargetBinaries -Arch $Arch -Build $Build
|
||||
|
||||
# Workaround exception handling issue
|
||||
chcp 65001 | Out-Null
|
||||
|
||||
# Reserve the old jit and copy in the specified jit.
|
||||
if ($Jit -ne "") {
|
||||
$CoreCLRRuntime = CoreCLRRuntime
|
||||
$CoreCLRVersion = CoreCLRVersion
|
||||
Rename-Item $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll.backupcopy
|
||||
Copy-Item $Jit $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll
|
||||
}
|
||||
|
||||
# Protect old value and set the new value for DUMPLLVMIR
|
||||
$DumpExists = Test-Path Env:\DUMPLLVMIR
|
||||
if ($DumpExists) {
|
||||
$OldDump = $Env:DUMPLLVMIR
|
||||
}
|
||||
$Env:DUMPLLVMIR = $Dump
|
||||
|
||||
# The set of test cases that are currently ignored
|
||||
$Env:SkipTestAssemblies = "Common;Exceptions;GC;Loader;managed;packages;Regressions;runtime;Tests;TestWrappers_x64_release;Threading"
|
||||
|
||||
# Run the test
|
||||
pushd .
|
||||
cd $CoreCLRTestAssets\coreclr\tests
|
||||
.\runtest $Arch $Build TestEnv $LLILCTest\LLILCTestEnv.cmd $CoreCLRRuntime\$CoreCLRVersion\bin | Write-Host
|
||||
$NumDiff = CheckDiff -Create $True -UseDiffTool $False -Arch $Arch -Build $Build
|
||||
$NumFailures = CheckFailure -Arch $Arch -Build $Build
|
||||
popd
|
||||
|
||||
# If there aren't any failures or diffs, return $True to say we passed
|
||||
# Restore old value for DUMPLLVMIR
|
||||
if ($DumpExists) {
|
||||
$Env:DUMPLLVMIR = $OldDump;
|
||||
}
|
||||
|
||||
# Restore the old jit
|
||||
if ($Jit -ne "") {
|
||||
Remove-Item $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll | Out-Null
|
||||
Rename-Item $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll.backupcopy $CoreCLRRuntime\$CoreCLRVersion\bin\LLILCJit.dll | Out-Null
|
||||
}
|
||||
|
||||
# Copy out result, normalize it in case of verbose dump.
|
||||
if ($Result -ne "") {
|
||||
$ResultExists = Test-Path $LLILCTestResult\$Result
|
||||
if ($ResuLtExists) {
|
||||
Remove-Item -recurse -force $LLILCTestResult\$Result | Out-Null
|
||||
}
|
||||
New-Item -itemtype directory $LLILCTestResult\$Result | Out-Null
|
||||
Copy-Item -recurse "$CoreCLRTestTargetBinaries\Reports\*" -Destination $LLILCTestResult\$Result
|
||||
Get-ChildItem -recurse -path $LLILCTestResult\$Result | Where {$_.FullName -match "output.txt"} | Remove-Item -force
|
||||
if ($Dump -eq "Verbose") {
|
||||
Write-Host ("Applying filter on verbose LLVM IR dump...")
|
||||
Get-ChildItem -recurse -path $LLILCTestResult\$Result | Where {$_.FullName -match "error.txt"} | ApplyFilterAll
|
||||
}
|
||||
}
|
||||
|
||||
$NumFailures = CheckFailure -Arch $Arch -Build $Build
|
||||
|
||||
# If there aren't any failures, return $True to say we passed
|
||||
# Otherwise return false
|
||||
if (($NumDiff -eq 0) -and ($NumFailures -eq 0)) {
|
||||
if ($NumFailures -eq 0) {
|
||||
return $True
|
||||
}
|
||||
else {
|
||||
|
|
@ -960,118 +1014,114 @@ function Global:RunTest([string]$Arch="x64", [string]$Build="Release")
|
|||
}
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
#
|
||||
# Re-create the base line for all LLILC enabled regression test cases.
|
||||
#
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
function Global:ReBaseAll([string]$Arch="x64", [string]$Build="Release")
|
||||
{
|
||||
$LLILCTest = LLILCTest
|
||||
$CoreCLRTestTargetBinaries = CoreCLRTestTargetBinaries -Arch $Arch -Build $Build
|
||||
|
||||
$BaseLineExists = Test-Path $LLILCTest\BaseLine
|
||||
if ($BaseLineExists) {
|
||||
Remove-Item -recurse -force $LLILCTest\BaseLine | Out-Null
|
||||
}
|
||||
New-Item -itemtype directory $LLILCTest\BaseLine | Out-Null
|
||||
|
||||
Copy-Item -recurse "$CoreCLRTestTargetBinaries\Reports\*" -Destination $LLILCTest\BaseLine
|
||||
Get-ChildItem -recurse -path $LLILCTest\BaseLine | Where {$_.FullName -match "output.txt"} | Remove-Item -force
|
||||
Get-ChildItem -recurse -path $LLILCTest\BaseLine | Where {$_.FullName -match "error.txt"} | ApplyFilterAll
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
#
|
||||
# Check the LLVM IR dump difference against baseline.
|
||||
#
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
function Global:CheckDiff([bool]$Create = $false, [bool]$UseDiffTool = $True, [string]$Arch="x64", [string]$Build="Release")
|
||||
function Global:CheckDiff
|
||||
{
|
||||
$LLILCTest = LLILCTest
|
||||
$LLILCTestResult = LLILCTestResult
|
||||
$CoreCLRTestTargetBinaries = CoreCLRTestTargetBinaries -Arch $Arch -Build $Build
|
||||
Param(
|
||||
[Parameter(Mandatory=$True)][string]$Base,
|
||||
[Parameter(Mandatory=$True)][string]$Diff,
|
||||
[Parameter(Mandatory=$False)][switch]$Summary,
|
||||
[Parameter(Mandatory=$False)][switch]$UseDiffTool
|
||||
)
|
||||
|
||||
$LLILCTestResult = LLILCTestResult
|
||||
|
||||
Write-Host ("Checking diff...")
|
||||
|
||||
if ($UseDiffTool) {
|
||||
# Validate Diff Tool
|
||||
|
||||
# Validate Diff Tool
|
||||
IsOnPath -executable "sgdm.exe" -software "DiffMerge"
|
||||
}
|
||||
|
||||
$DiffExists = Test-Path $LLILCTestResult\Diff
|
||||
if ($Create) {
|
||||
if ($DiffExists) {
|
||||
Remove-Item -recurse -force $LLILCTestResult\Diff | Out-Null
|
||||
}
|
||||
$CompareExists = Test-Path $LLILCTestResult\Compare
|
||||
|
||||
New-Item -itemtype directory $LLILCTestResult\Diff | Out-Null
|
||||
New-Item -itemtype directory $LLILCTestResult\Diff\Base | Out-Null
|
||||
New-Item -itemtype directory $LLILCTestResult\Diff\Run | Out-Null
|
||||
# Refresh a new Compare subdirectory
|
||||
if ($CompareExists) {
|
||||
Remove-Item -recurse -force $LLILCTestResult\Compare | Out-Null
|
||||
}
|
||||
|
||||
New-Item -itemtype directory $LLILCTestResult\Compare | Out-Null
|
||||
New-Item -itemtype directory $LLILCTestResult\Compare\$Base | Out-Null
|
||||
New-Item -itemtype directory $LLILCTestResult\Compare\$Diff | Out-Null
|
||||
|
||||
if (!$Summary) {
|
||||
# Deal with verbose LLVM IR comparison
|
||||
$TotalCount = 0
|
||||
$DiffCount = 0
|
||||
Get-ChildItem -recurse -path $CoreCLRTestTargetBinaries\Reports | Where {$_.FullName -match "error.txt"} | `
|
||||
Get-ChildItem -recurse -path $LLILCTestResult\$Diff | Where {$_.FullName -match "error.txt"} | `
|
||||
Foreach-Object {
|
||||
$TotalCount = $TotalCount + 1
|
||||
$RunFile = $_.FullName
|
||||
$PartialPathMatch = $_.FullName -match "Reports\\(.*)"
|
||||
$DiffFile = $_.FullName
|
||||
$PartialPathMatch = $_.FullName -match "$Diff\\(.*)"
|
||||
$PartialPath = $matches[1]
|
||||
$BaseFile = "$LLILCTest\BaseLine\$PartialPath"
|
||||
copy $RunFile $LLILCTestResult\Diff\Run
|
||||
ApplyFilter("$LLILCTestResult\Diff\Run\$_")
|
||||
$DiffResult = Compare-Object -Ref (Get-Content $BaseFile) -Diff (Get-Content $LLILCTestResult\Diff\Run\$_)
|
||||
$BaseFile = "$LLILCTestResult\$Base\$PartialPath"
|
||||
$DiffResult = Compare-Object -Ref (Get-Content $BaseFile) -Diff (Get-Content $DiffFile)
|
||||
if ($DiffResult.Count -ne 0) {
|
||||
copy $BaseFile $LLILCTestResult\Diff\Base
|
||||
Copy-Item $BaseFile $LLILCTestResult\Compare\$Base
|
||||
Copy-Item $DiffFile $LLILCTestResult\Compare\$Diff
|
||||
$DiffCount = $DiffCount + 1
|
||||
}
|
||||
else {
|
||||
Remove-Item -force $LLILCTestResult\Diff\Run\$_ | Out-Null
|
||||
}
|
||||
if ($DiffCount -eq 0) {
|
||||
Write-Host ("There is no diff.")
|
||||
Remove-Item -recurse -force $LLILCTestResult\Compare | Out-Null
|
||||
}
|
||||
else {
|
||||
Write-Host ("$DiffCount out of $TotalCount have diff.")
|
||||
if ($UseDiffTool) {
|
||||
& sgdm -t1=Base -t2=Diff $LLILCTestResult\Compare\$Base $LLILCTestResult\Compare\$Diff
|
||||
}
|
||||
}
|
||||
return $DiffCount
|
||||
}
|
||||
else {
|
||||
# Deal with summary comparison
|
||||
$TotalCount = 0
|
||||
$DiffCount = 0
|
||||
$NewlyFailedMethods = 0
|
||||
Get-ChildItem -recurse -path $LLILCTestResult\$Diff | Where {$_.FullName -match "error.txt"} | `
|
||||
Foreach-Object {
|
||||
$TotalCount = $TotalCount + 1
|
||||
$DiffFile = $_.FullName
|
||||
$PartialPathMatch = $_.FullName -match "$Diff\\(.*)"
|
||||
$PartialPath = $matches[1]
|
||||
$BaseFile = "$LLILCTestResult\$Base\$PartialPath"
|
||||
$DiffResult = Compare-Object -Ref (Get-Content $BaseFile) -Diff (Get-Content $DiffFile)
|
||||
if ($DiffResult.Count -ne 0) {
|
||||
Copy-Item $BaseFile $LLILCTestResult\Compare\$Base
|
||||
Copy-Item $DiffFile $LLILCTestResult\Compare\$Diff
|
||||
$DiffCount = $DiffCount + 1
|
||||
}
|
||||
Foreach ($SummaryLine in $DiffResult) {
|
||||
if ($SummaryLine -match 'Successfully(.*)(<=)') {
|
||||
$NewlyFailedMethods++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($DiffCount -eq 0) {
|
||||
Write-Host ("There is no diff.")
|
||||
Remove-Item -recurse -force $LLILCTestResult\Diff | Out-Null
|
||||
Remove-Item -recurse -force $LLILCTestResult\Compare | Out-Null
|
||||
}
|
||||
else {
|
||||
Write-Host ("$DiffCount out of $TotalCount have diff.")
|
||||
if ($UseDiffTool) {
|
||||
& sgdm -t1=Base -t2=Run $LLILCTestResult\Diff\Base $LLILCTestResult\Diff\Run
|
||||
}
|
||||
}
|
||||
|
||||
return $DiffCount
|
||||
}
|
||||
else {
|
||||
if (!$DiffExists) {
|
||||
Write-Host ("There is no diff.")
|
||||
return 0
|
||||
}
|
||||
else {
|
||||
if ($UseDiffTool) {
|
||||
& sgdm -t1=Base -t2=Run $LLILCTestResult\Diff\Base $LLILCTestResult\Diff\Run
|
||||
return 0
|
||||
if ($NewlyFailedMethods -eq 0) {
|
||||
Write-Host ("All previously successfully jitted methods passed jitting with diff jit.")
|
||||
Write-Host ("More methods passed jitting in diff jit than in base jit.")
|
||||
}
|
||||
else {
|
||||
$TotalCount = 0
|
||||
$DiffCount = 0
|
||||
Get-ChildItem -recurse -path $CoreCLRTestTargetBinaries\Reports | Where {$_.FullName -match "error.txt"} | `
|
||||
Foreach-Object {
|
||||
$TotalCount = $TotalCount + 1
|
||||
}
|
||||
|
||||
Get-ChildItem -recurse -path $LLILCTestResult\Diff\Run | Where {$_.FullName -match "error.txt"} | `
|
||||
Foreach-Object {
|
||||
$DiffCount = $DiffCount + 1
|
||||
}
|
||||
Write-Host ("$DiffCount out of $TotalCount have diff.")
|
||||
return $DiffCount
|
||||
Write-Host ("$NewlyFailedMethods methods successfully jitted by base jit now FAILED in diff jit.")
|
||||
}
|
||||
if ($UseDiffTool) {
|
||||
& sgdm -t1=Base -t2=Diff $LLILCTestResult\Compare\$Base $LLILCTestResult\Compare\$Diff
|
||||
}
|
||||
}
|
||||
return $NewlyFailedMethods
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1109,8 +1159,8 @@ function Global:llilc([string]$Command="")
|
|||
}
|
||||
|
||||
if ($ListAll -Or ($Command -eq "CheckDiff")) {
|
||||
Write-Output("CheckDiff - Check the LLVM IR dump diff between run and baseline.")
|
||||
Write-Output(" Example: CheckDiff -Create `$False -UseDiffTool `$True -Arch x64 -Build Release")
|
||||
Write-Output("CheckDiff - Check the LLVM IR dump or summary diff between run and baseline.")
|
||||
Write-Output(" Example: CheckDiff -Base BaseResultName -Diff DiffResultName -Summary -UseDiffTool")
|
||||
}
|
||||
|
||||
if ($ListAll -Or ($Command -eq "CheckEnv")) {
|
||||
|
|
@ -1128,14 +1178,9 @@ function Global:llilc([string]$Command="")
|
|||
Write-Output(" Example: llilc RunTest")
|
||||
}
|
||||
|
||||
if ($ListAll -Or ($Command -eq "ReBaseAll")) {
|
||||
Write-Output("ReBaseAll - Re-create the base line for all regression test cases.")
|
||||
Write-Output(" Example: ReBaseAll -Arch x64 -Build Release")
|
||||
}
|
||||
|
||||
if ($ListAll -Or ($Command -eq "RunTest")) {
|
||||
Write-Output("RunTest - Run LLILC enabled CoreCLR regression tests.")
|
||||
Write-Output(" Example: RunTest -Arch x64 -Build Release")
|
||||
Write-Output(" Example: RunTest -Arch x64 -Build Release -Jit FullPathToJit -Result ResultName -Dump Summary")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче