Merge pull request #1902 from github/codecov-again

Setting up CodeCov again

.gitignore

@@ -232,6 +232,9 @@ $RECYCLE.BIN/
 *.msm
 *.msp
 
+#OpenCover output
+coverage.xml
+
 AkavacheSqliteLinkerOverride.cs
 NuGetBuild
 WiX.Toolset.DummyFile.txt

appveyor.yml

@@ -29,3 +29,5 @@ on_success:
     if ($full_build) {
       script\Sign-Package -AppVeyor
     }
+on_finish:
+- ps: scripts\Run-CodeCoverage.ps1 -AppVeyor -Configuration:Release

scripts/Run-CodeCoverage.ps1

@@ -0,0 +1,68 @@
+<#
+.SYNOPSIS
+    Runs NUnit
+#>
+
+[CmdletBinding()]
+Param(
+    [string]
+    $Configuration
+    ,
+    [switch]
+    $AppVeyor = $false
+)
+
+$scriptsDirectory = $PSScriptRoot
+$rootDirectory = Split-Path ($scriptsDirectory)
+. $scriptsDirectory\modules.ps1 | out-null
+
+$nunitDirectory = Join-Path $rootDirectory packages\NUnit.ConsoleRunner.3.7.0\tools
+$nunitConsoleRunner = Join-Path $nunitDirectory nunit3-console.exe
+
+$testAssemblies = @(
+    "test\GitHub.Api.UnitTests\bin\$Configuration\GitHub.Api.UnitTests.dll",
+    "test\GitHub.App.UnitTests\bin\$Configuration\GitHub.App.UnitTests.dll",
+    "test\GitHub.Exports.Reactive.UnitTests\bin\$Configuration\GitHub.Exports.Reactive.UnitTests.dll",
+    "test\GitHub.Exports.UnitTests\bin\$Configuration\GitHub.Exports.UnitTests.dll",
+    "test\GitHub.Extensions.UnitTests\bin\$Configuration\GitHub.Extensions.UnitTests.dll",
+    "test\GitHub.InlineReviews.UnitTests\bin\$Configuration\GitHub.InlineReviews.UnitTests.dll",
+    "test\GitHub.Primitives.UnitTests\bin\$Configuration\GitHub.Primitives.UnitTests.dll",
+    "test\GitHub.TeamFoundation.UnitTests\bin\$Configuration\GitHub.TeamFoundation.UnitTests.dll",
+    "test\GitHub.UI.UnitTests\bin\$Configuration\GitHub.UI.UnitTests.dll",
+    "test\GitHub.VisualStudio.UnitTests\bin\$Configuration\GitHub.VisualStudio.UnitTests.dll",
+    "test\MetricsTests\MetricsTests\bin\$Configuration\MetricsTests.dll",
+    "test\TrackingCollectionTests\bin\$Configuration\TrackingCollectionTests.dll"
+)
+
+$opencoverTargetArgs = ($testAssemblies -join " ") + " --where \`"cat!=Timings and cat!=CodeCoverageFlake\`" --inprocess --noresult"
+
+$opencoverDirectory = Join-Path $rootDirectory packages\OpenCover.4.6.519\tools
+$opencover = Join-Path $opencoverDirectory OpenCover.Console.exe
+$opencoverArgs = @(
+    "-target:`"$nunitConsoleRunner`"",
+    "-targetargs:`"$opencoverTargetArgs`"",
+    "-filter:`"+[GitHub*]* -[GitHub*UnitTests]*`"",
+    "-register:user -output:$rootDirectory\coverage.xml"
+) -join " "
+
+$codecovDirectory = Join-Path $rootDirectory packages\Codecov.1.0.5\tools
+$codecov = Join-Path $codecovDirectory codecov.exe
+$codecovArgs = "-f $rootDirectory\coverage.xml"
+
+& {
+    Trap {
+        Write-Output "$Project tests failed"
+        exit 0
+    }
+
+    Run-Process 600 $opencover $opencoverArgs
+
+    if($AppVeyor) {
+        Run-Process 60 $codecov $codecovArgs
+    }
+
+    if (!$?) {
+        Write-Output "$Project tests failed"
+        exit 0
+    }
+}

scripts/modules.ps1

@@ -73,7 +73,7 @@ New-Module -ScriptBlock {
             $output += Get-Content $outputPath
             $exitCode = $process.ExitCode
         } else {
-            $output += "Tests timed out. Backtrace:"
+            $output += "Process timed out. Backtrace:"
             $output += Get-DotNetStack $process.Id
             $exitCode = 9999
         }

test/GitHub.InlineReviews.UnitTests/Services/PullRequestSessionManagerTests.cs

@@ -447,7 +447,7 @@ Line 4";
                 }
             }
 
-            [Test]
+            [Test, NUnit.Framework.Category("CodeCoverageFlake")]
             public async Task UpdatesInlineCommentThreadsFromEditorContent()
             {
                 var baseContents = @"Line 1
@@ -502,7 +502,7 @@ Line 4";
                 }
             }
 
-            [Test]
+            [Test, NUnit.Framework.Category("CodeCoverageFlake")]
             public async Task UpdatesReviewCommentWithNewBody()
             {
                 var baseContents = @"Line 1

test/MetricsTests/MetricsTests/packages.config

@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Codecov" version="1.0.5" targetFramework="net461" />
   <package id="NUnit" version="3.9.0" targetFramework="net452" />
   <package id="NUnit.ConsoleRunner" version="3.7.0" targetFramework="net452" />
   <package id="NUnit3TestAdapter" version="3.9.0" targetFramework="net452" />
+  <package id="OpenCover" version="4.6.519" targetFramework="net461" />
 </packages>

test/TrackingCollectionTests/TrackingCollectionTests.cs

@@ -1832,7 +1832,7 @@ public class TrackingTests : TestBase
         col.Dispose();
     }
 
-    [Test]
+    [Test, Category("CodeCoverageFlake")]
     public void RemovingFirstItemWithFilterWorks()
     {
         var source = new Subject<Thing>();