Re-enable code coverage reporting (#4046)

Each build leg (i.e., Windows and Ubuntu) produce their respective code
coverage reports, which are then get merged by `CodeCoverageReport`
build job. This job then generates and publishes an HTML report.

Locally code coverage can be collected via:

PS> .\build.cmd -testCoverage
$ ./build.sh --testCoverage
This commit is contained in:
Igor Velikorossov 2023-06-21 12:11:34 +10:00 коммит произвёл GitHub
Родитель a0e9c8794e
Коммит 78375bffad
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
13 изменённых файлов: 189 добавлений и 153 удалений

Просмотреть файл

@ -3,7 +3,7 @@
"isRoot": true,
"tools": {
"dotnet-coverage": {
"version": "17.6.9",
"version": "17.7.1",
"commands": [
"dotnet-coverage"
]

Просмотреть файл

@ -44,7 +44,6 @@
<DebugType>portable</DebugType>
<DebugSymbols>true</DebugSymbols>
<SuppressNETCoreSdkPreviewMessage>true</SuppressNETCoreSdkPreviewMessage>
<RunSettingsFilePath>$(MSBuildThisFileDirectory)\eng\Common.runsettings</RunSettingsFilePath>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
<Deterministic>true</Deterministic>
<Features>debug-determinism</Features>

Просмотреть файл

@ -51,9 +51,8 @@ variables:
- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
value: true
# TEMP until all the x-cutting refactoring is complete
- name: SkipCodeCoverage
value: true
- name: SkipQualityGates
value: false
- name: runAsPublic
value: ${{ eq(variables['System.TeamProject'], 'public') }}
@ -140,6 +139,10 @@ stages:
clean: all
jobs:
# ----------------------------------------------------------------
# This job build and run tests on Windows
# ----------------------------------------------------------------
- job: Windows
timeoutInMinutes: 180
testResultsFormat: VSTest
@ -163,17 +166,20 @@ stages:
fetchDepth: $(_FetchDepth)
steps:
- template: \eng\pipelines\templates\BuildAndTest.yml
- template: /eng/pipelines/templates/BuildAndTest.yml
parameters:
buildScript: $(_buildScript)
buildConfig: $(_BuildConfig)
repoLogPath: $(Build.Arcade.LogsPath)
repoTestResultsPath: $(Build.Arcade.TestResultsPath)
skipCodeCoverage: ${{ eq(variables['SkipCodeCoverage'], 'true') }}
skipQualityGates: ${{ eq(variables['SkipQualityGates'], 'true') }}
isDeltaBuild: $(IsDeltaBuild)
isWindows: true
warnAsError: 0
# ----------------------------------------------------------------
# This job build and run tests on Ubuntu
# ----------------------------------------------------------------
- job: Ubuntu
timeoutInMinutes: 180
testResultsFormat: VSTest
@ -197,18 +203,73 @@ stages:
fetchDepth: $(_FetchDepth)
steps:
- template: \eng\pipelines\templates\BuildAndTest.yml
- template: /eng/pipelines/templates/BuildAndTest.yml
parameters:
buildScript: $(_buildScript)
buildConfig: $(_BuildConfig)
repoLogPath: $(Build.Arcade.LogsPath)
repoTestResultsPath: $(Build.Arcade.TestResultsPath)
skipCodeCoverage: true
skipQualityGates: ${{ eq(variables['SkipQualityGates'], 'true') }}
isDeltaBuild: $(IsDeltaBuild)
isWindows: false
warnAsError: 0
# ----------------------------------------------------------------
# This stage performs quality gates enforcements
# ----------------------------------------------------------------
- stage: qualitygates
displayName: QualityGates
condition: ne(variables['SkipQualityGates'], 'true')
dependsOn:
- build
variables:
- template: /eng/common/templates/variables/pool-providers.yml
jobs:
- template: /eng/common/templates/jobs/jobs.yml
parameters:
enableMicrobuild: true
enableTelemetry: true
runAsPublic: ${{ variables['runAsPublic'] }}
workspace:
clean: all
# ----------------------------------------------------------------
# This stage downloads the code coverage reports from the build jobs,
# merges those and validates the combined test coverage.
# ----------------------------------------------------------------
jobs:
- job: CodeCoverageReport
timeoutInMinutes: 180
pool:
${{ if eq(variables['runAsPublic'], 'true') }}:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals build.ubuntu.2004.amd64.open
# Non-public (i.e., official builds)
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals build.ubuntu.2004.amd64
preSteps:
- checkout: self
clean: true
persistCredentials: true
fetchDepth: 1
steps:
- script: $(Build.SourcesDirectory)/build.sh --ci --restore
displayName: Init toolset
- template: /eng/pipelines/templates/VerifyCoverageReport.yml
parameters:
isDeltaBuild: $(IsDeltaBuild)
# ----------------------------------------------------------------
# This stage only performs a build treating warnings as errors
# to detect any kind of code style violations
# ----------------------------------------------------------------
- stage: correctness
displayName: Correctness
dependsOn: []
@ -252,8 +313,8 @@ stages:
buildConfig: $(_BuildConfig)
repoLogPath: $(Build.Arcade.LogsPath)
repoTestResultsPath: $(Build.Arcade.TestResultsPath)
skipCodeCoverage: true
skipTests: true
skipQualityGates: true
isDeltaBuild: $(IsDeltaBuild)
isWindows: false
@ -264,6 +325,7 @@ stages:
parameters:
validateDependsOn:
- build
- qualitygates
- correctness
publishingInfraVersion: 3
enableSymbolValidation: false

Просмотреть файл

@ -1,5 +1,4 @@
@ECHO OFF
SETLOCAL EnableDelayedExpansion
SET _args=%*
IF "%~1"=="-?" SET _args=-help
@ -10,23 +9,5 @@ IF ["%_args%"] == [""] (
SET _args=-restore -build
)
FOR %%x IN (%*) DO (
SET _arg=%%x
IF /I "%%x%"=="-coverage" GOTO RUN_CODE_COVERAGE
)
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\build.ps1""" %_args%"
EXIT /b %ERRORLEVEL%
:RUN_CODE_COVERAGE
SET DOTNET_ROOT=%~dp0.dotnet
:: This tells .NET Core not to go looking for .NET Core in other places
SET DOTNET_MULTILEVEL_LOOKUP=0
dotnet dotnet-coverage collect --settings ./eng/CodeCoverage.config --output ./artifacts/TestResults/ "build.cmd -test -bl"
dotnet reportgenerator -reports:./artifacts/TestResults/*.cobertura.xml -targetdir:./artifacts/TestResults/CoverageResultsHtml -reporttypes:HtmlInline_AzurePipelines
start ./artifacts/TestResults/CoverageResultsHtml/index.html
powershell -ExecutionPolicy ByPass -NoProfile -command "./scripts/ValidateProjectCoverage.ps1 -CoberturaReportXml ./artifacts/TestResults/.cobertura.xml"
EXIT /b %ErrorLevel%

Просмотреть файл

@ -1,20 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<RunSettings>
<RunConfiguration>
<TargetPlatform>x64</TargetPlatform>
<DisableAppDomain>true</DisableAppDomain>
</RunConfiguration>
<DataCollectionRunSettings>
<DataCollectors>
<DataCollector friendlyName="XPlat code coverage">
<Configuration>
<Format>cobertura</Format>
<Exclude>[*.Tests]*</Exclude> <!-- [Assembly-Filter]Type-Filter -->
<Include>[Microsoft.*]*,[System*]*</Include>
<ExcludeByAttribute>ExcludeFromCodeCoverageAttribute,DebuggerNonUserCodeAttribute,DebuggerHiddenAttribute,GeneratedCodeAttribute</ExcludeByAttribute>
</Configuration>
</DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>

Просмотреть файл

@ -58,6 +58,10 @@ Param(
[Parameter(ParameterSetName='CommandLine')]
[switch] $help,
# Run tests with code coverage
[Parameter(ParameterSetName='CommandLine')]
[switch] $testCoverage,
[Parameter(ParameterSetName='CommandLine')]
[Parameter(ParameterSetName='VisualStudio')]
[string[]] $onlyTfms = $null,
@ -72,6 +76,7 @@ Param(
function Print-Usage() {
Write-Host "Custom settings:"
Write-Host " -testCoverage Run unit tests and capture code coverage information."
Write-Host " -vs <value> Comma delimited list of keywords to filter the projects in the solution."
Write-Host " Pass '*' to generate a solution with all projects."
Write-Host " -noLaunch Don't open the generated solution in Visual Studio (only if -vs specified)"
@ -96,7 +101,7 @@ if ($filter.Count -ne 0) {
try {
# Install required toolset
. $PSScriptRoot/common/tools.ps1
InitializeDotNetCli -install $true
InitializeDotNetCli -install $true | Out-Null
Push-Location $PSScriptRoot/../
if ($filter -eq '*') {
@ -116,7 +121,9 @@ if ($filter.Count -ne 0) {
$restore = $true;
}
catch {
exit $LASTEXITCODE;
Write-Host $_.Exception.Message -Foreground "Red"
Write-Host $_.ScriptStackTrace -Foreground "DarkGray"
exit $global:LASTEXITCODE;
}
finally {
Pop-Location
@ -175,3 +182,32 @@ if ([string]::IsNullOrWhiteSpace($projects)) {
-help:$help `
@properties
# Perform code coverage as the last operation, this enables the following scenarios:
# .\build.cmd -restore -build -c Release -testCoverage
if ($testCoverage) {
try {
# Install required toolset
. $PSScriptRoot/common/tools.ps1
InitializeDotNetCli -install $true | Out-Null
Push-Location $PSScriptRoot/../
$testResultPath = "./artifacts/TestResults/$configuration";
# Run tests and collect code coverage
./.dotnet/dotnet dotnet-coverage collect --settings ./eng/CodeCoverage.config --output $testResultPath/local.cobertura.xml "build.cmd -test -configuration $configuration -bl:`$$binaryLog"
# Generate the code coverage report and open it in the browser
./.dotnet/dotnet reportgenerator -reports:$testResultPath/*.cobertura.xml -targetdir:$testResultPath/CoverageResultsHtml -reporttypes:HtmlInline_AzurePipelines
Start-Process $testResultPath/CoverageResultsHtml/index.html
}
catch {
Write-Host $_.Exception.Message -Foreground "Red"
Write-Host $_.ScriptStackTrace -Foreground "DarkGray"
exit $global:LASTEXITCODE;
}
finally {
Pop-Location
}
}

Просмотреть файл

@ -10,6 +10,7 @@ set -e
usage()
{
echo "Custom settings:"
echo " --testCoverage Run unit tests and capture code coverage information."
echo " --vs <value> Comma delimited list of keywords to filter the projects in the solution"
echo " Pass '*' to generate a solution with all projects."
echo " --onlyTfms <value> Semi-colon delimited list of TFMs to build (e.g. 'net8.0;net6.0')"
@ -24,6 +25,8 @@ onlyTfms=''
hasProjects=false
hasWarnAsError=false
hasRestore=false
configuration=''
testCoverage=false
properties=''
@ -35,7 +38,7 @@ while [[ $# > 0 ]]; do
"$DIR/common/build.sh" --help
exit 0
;;
-vs)
-vs)
filter=true
shift
keywords=$1
@ -50,7 +53,7 @@ while [[ $# > 0 ]]; do
properties="$properties $1 $(realpath $2)"
shift
;;
-restore)
-restore)
hasRestore=true
properties="$properties $1"
;;
@ -64,6 +67,14 @@ while [[ $# > 0 ]]; do
properties="$properties $1 $value"
shift
;;
-configuration|-c)
configuration=$2
properties="$properties $1 $2"
shift
;;
-testcoverage)
testCoverage=true
;;
*)
properties="$properties $1"
;;
@ -79,7 +90,7 @@ fi
if [[ "$filter" == true ]]; then
# Install required toolset
. "$DIR/common/tools.sh"
InitializeDotNetCli true
InitializeDotNetCli true > /dev/null
# Invoke the solution generator
script=$(realpath $DIR/../scripts/Slngen.ps1)
@ -123,3 +134,24 @@ if [[ "$hasWarnAsError" == false ]]; then
fi
"$DIR/common/build.sh" $properties
# Perform code coverage as the last operation, this enables the following scenarios:
# .\build.sh --restore --build --c Release --testCoverage
if [[ "$testCoverage" == true ]]; then
# Install required toolset
. "$DIR/common/tools.sh"
InitializeDotNetCli true > /dev/null
repoRoot=$(realpath $DIR/../)
testResultPath="$repoRoot/artifacts/TestResults/$configuration"
# Run tests and collect code coverage
$repoRoot/.dotnet/dotnet 'dotnet-coverage' collect --settings $repoRoot/eng/CodeCoverage.config --output $testResultPath/local.cobertura.xml "$repoRoot/build.sh --test --configuration $configuration"
# Generate the code coverage report and open it in the browser
$repoRoot/.dotnet/dotnet reportgenerator -reports:$testResultPath/*.cobertura.xml -targetdir:$testResultPath/CoverageResultsHtml -reporttypes:HtmlInline_AzurePipelines
echo ""
echo -e "\e[32mCode coverage results:\e[0m $testResultPath/CoverageResultsHtml/index.html"
echo ""
fi

Просмотреть файл

@ -11,10 +11,10 @@ parameters:
type: string
- name: isWindows
type: string
- name: skipCodeCoverage
- name: skipTests
type: boolean
default: false
- name: skipTests
- name: skipQualityGates
type: boolean
default: false
- name: warnAsError
@ -30,14 +30,8 @@ steps:
# Write-Host 'repoTestResultsPath: ${{ parameters.repoTestResultsPath }}'
# Write-Host 'isDeltaBuild: ${{ parameters.isDeltaBuild }}'
# Write-Host 'isWindows: ${{ parameters.isWindows }}'
# Write-Host 'skipCodeCoverage: ${{ parameters.skipCodeCoverage }}'
# Write-Host 'skipCodeCoverage == true: ${{ eq(parameters.skipCodeCoverage, true) }}'
# Write-Host "skipCodeCoverage == 'true': ${{ eq(parameters.skipCodeCoverage, 'true') }}"
# Write-Host 'skipCodeCoverage != true: ${{ ne(parameters.skipCodeCoverage, true) }}'
# Write-Host "skipCodeCoverage != 'true': ${{ ne(parameters.skipCodeCoverage, 'true') }}"
# Write-Host 'running: ${{ and(eq(parameters.isWindows, 'true'), ne(parameters.skipCodeCoverage, true)) }}'
# Write-Host 'skipTests: ${{ parameters.skipTests }}'
# Write-Host 'skipQualityGates: ${{ parameters.skipQualityGates }}'
# Get-ChildItem env:* | Sort-Object Name
# displayName: Debug
@ -62,7 +56,7 @@ steps:
- ${{ if ne(parameters.skipTests, 'true') }}:
- script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage collect
--settings $(Build.SourcesDirectory)/eng/CodeCoverage.config
--output ${{ parameters.repoTestResultsPath }}
--output ${{ parameters.repoTestResultsPath }}/$(Agent.Os)_$(Agent.JobName).cobertura.xml
"${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)"
displayName: Run tests
@ -78,22 +72,17 @@ steps:
Copy-Item -Path $_.FullName -Destination $destination -Force;
}
}
displayName: Copy crash results to logs
condition: always()
continueOnError: true
# Run code coverage reporting and validation on Windows
# a) we run a subset of tests on Linux
# b) we can only publish a single coverage report,
# see https://learn.microsoft.com/azure/devops/pipelines/tasks/reference/publish-code-coverage-results-v2#is-code-coverage-data-merged-when-multiple-files-are-provided-as-input-to-the-task-or-multiple-tasks-are-used-in-the-pipeline
- ${{ if and(eq(parameters.isWindows, 'true'), ne(parameters.skipCodeCoverage, 'true')) }}:
- template: \eng\pipelines\templates\TestCoverageReport.yml
parameters:
isDeltaBuild: ${{ parameters.isDeltaBuild }}
repoLogPath: ${{ parameters.repoLogPath }}
testResultsPath: ${{ parameters.repoTestResultsPath }}
testResultsFile: ${{ parameters.repoTestResultsPath }}/.cobertura.xml
- ${{ if ne(parameters.skipQualityGates, 'true') }}:
- task: PublishBuildArtifacts@1
displayName: Publish coverage results (cobertura.xml)
inputs:
PathtoPublish: '${{ parameters.repoTestResultsPath }}/$(Agent.Os)_$(Agent.JobName).cobertura.xml'
PublishLocation: Container
ArtifactName: CodeCoverageResults
- ${{ if eq(parameters.isWindows, 'true') }}:
- script: ${{ parameters.buildScript }}

Просмотреть файл

@ -1,75 +0,0 @@
parameters:
- name: isDeltaBuild
type: string
- name: repoLogPath
type: string
- name: testResultsPath
type: string
- name: testResultsFile
type: string
steps:
- task: PowerShell@2
displayName: Check coverage report exists
inputs:
targetType: 'inline'
script: |
Write-Host 'isDeltaBuild: ${{ parameters.isDeltaBuild }}'
Write-Host 'repoLogPath: ${{ parameters.repoLogPath }}'
Write-Host 'testResultsFile: ${{ parameters.testResultsFile }}'
if (Test-Path '${{ parameters.testResultsFile }}') {
function Export { param($i); Write-Host "$i"; Write-Host "##$i" }
Export "vso[task.setvariable variable=PerformCoverageCheck]True"
}
else {
Write-Host "No coverage reports."
}
- script: $(Build.SourcesDirectory)/.dotnet/dotnet reportgenerator
-reports:"${{ parameters.testResultsFile }}"
-targetdir:"${{ parameters.testResultsPath }}/CoverageResultsHtml"
-reporttypes:HtmlInline_AzurePipelines
displayName: Generate code coverage report
condition: and(succeeded(), eq(variables['PerformCoverageCheck'], 'True'))
- task: PublishCodeCoverageResults@1
displayName: Publish coverage report
condition: and(succeeded(), eq(variables['PerformCoverageCheck'], 'True'))
env:
DISABLE_COVERAGE_AUTOGENERATE: 'true'
inputs:
codeCoverageTool: cobertura
summaryFileLocation: '${{ parameters.testResultsFile }}'
pathToSources: $(Build.SourcesDirectory)
reportDirectory: '${{ parameters.testResultsPath }}/CoverageResultsHtml'
- task: PublishBuildArtifacts@1
displayName: Publish coverage results (cobertura.xml)
condition: and(always(), eq(variables['PerformCoverageCheck'], 'True'))
inputs:
PathtoPublish: '${{ parameters.testResultsFile }}'
PublishLocation: Container
ArtifactName: '$(Agent.Os)_$(Agent.JobName)'
continueOnError: true
- task: PowerShell@2
displayName: Check per-project coverage
condition: and(succeeded(), ne(variables['IsDeltaBuild'], 'True'), eq(variables['PerformCoverageCheck'], 'True'))
inputs:
targetType: 'filePath'
filePath: $(Build.SourcesDirectory)/scripts/ValidateProjectCoverage.ps1
arguments: >
-CoberturaReportXml '${{ parameters.testResultsFile }}'
- task: PowerShell@2
displayName: ∆ Check per-project coverage
condition: and(succeeded(), eq(variables['IsDeltaBuild'], 'True'), eq(variables['PerformCoverageCheck'], 'True'))
inputs:
targetType: 'inline'
script: |
$DeltaBuildJsonFile = '${{ parameters.repoLogPath }}\DeltaBuild.json';
$AffectedProjectChain = (Get-Content $DeltaBuildJsonFile | ConvertFrom-Json).AffectedProjectChain;
$(Build.SourcesDirectory)/scripts/ValidateProjectCoverage.ps1 `
-CoberturaReportXml '${{ parameters.testResultsFile }}' `
-OnlyForProjects $AffectedProjectChain

Просмотреть файл

@ -0,0 +1,32 @@
parameters:
- name: isDeltaBuild
type: string
steps:
- task: DownloadBuildArtifacts@0
displayName: Download code coverage reports
inputs:
artifactName: CodeCoverageResults
downloadPath: $(System.DefaultWorkingDirectory)
- script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage merge
$(System.DefaultWorkingDirectory)/CodeCoverageResults/*.cobertura.xml
--output-format cobertura
--output ./merged.cobertura.xml
displayName: Merge code coverage reports
- script: $(Build.SourcesDirectory)/.dotnet/dotnet reportgenerator
-reports:./merged.cobertura.xml
-targetdir:./CoverageResultsHtml
-reporttypes:HtmlInline_AzurePipelines
displayName: Generate code coverage report
- task: PublishCodeCoverageResults@1
displayName: Publish coverage report
env:
DISABLE_COVERAGE_AUTOGENERATE: 'true'
inputs:
codeCoverageTool: cobertura
summaryFileLocation: ./merged.cobertura.xml
pathToSources: $(Build.SourcesDirectory)
reportDirectory: ./CoverageResultsHtml

2
scripts/Slngen.ps1 Normal file → Executable file
Просмотреть файл

@ -115,7 +115,7 @@ function Invoke-SlngenExe
$MSBuild
)
dotnet tool restore --verbosity minimal
dotnet tool restore --verbosity minimal | Out-Null
$process = Start-Process `
-FilePath 'dotnet' `
-ArgumentList @("slngen", "--folders $Folders", "--collapsefolders $Folders", "--ignoreMainProject", "--nologo", "-o $OutSln", "$Globs", "--launch $(!$NoLaunch) $Exclude $ConsoleOutput $MSBuild") `

0
scripts/ValidateProjectCoverage.ps1 Normal file → Executable file
Просмотреть файл