Enable test coverage (#2230)

* Revert "exclude test folder from codecov. (#2227)"

This reverts commit eed91b988f.

* Use a fake target path to avoid tonerdo/coverlet#318

* Manually apply codecov flags through multiple uploads

Eventually codecov.io should support the automatic application of flags
based on directory structure. In the meantime, upload the coverage
report twice and allow the server to filter and apply the desired flags.

* Show production code numbers when coverage is reported

README.md

@@ -42,7 +42,7 @@ Daily NuGet builds of the project are also available in our [MyGet](https://dotn
 
 To build ML.NET from source please visit our [developers guide](docs/project-docs/developer-guide.md).
 
-[![codecov](https://codecov.io/gh/dotnet/machinelearning/branch/master/graph/badge.svg)](https://codecov.io/gh/dotnet/machinelearning)
+[![codecov](https://codecov.io/gh/dotnet/machinelearning/branch/master/graph/badge.svg?flag=production)](https://codecov.io/gh/dotnet/machinelearning)
 
 |    | Debug | Release |
 |:---|----------------:|------------------:|

build/Codecoverage.proj

@@ -28,12 +28,22 @@
       <_CodecovArgs Include="--branch;$(_BranchName)" Condition="'$(_BranchName)' != ''" />
       <_CodecovArgs Include="-c;$(BUILD_SOURCEVERSION)" Condition="'$(BUILD_SOURCEVERSION)' != ''" />
       <_CodecovArgs Include="-n;$(BUILD_DEFINITIONNAME)" Condition="'$(BUILD_DEFINITIONNAME)' != ''" />
-      <_CodecovArgs Include="--flag;$(Configuration)" Condition="'$(Configuration)' != ''" />
       <_CodecovArgs Include="-t;$(CodeCovToken)" Condition="'$(CodeCovToken)' != ''" />
+
+      <_CodecovFlags Include="$(Configuration)" Condition="'$(Configuration)' != ''" />
+      <_CodecovProductionFlags Include="@(_CodecovFlags)" />
+      <_CodecovProductionFlags Include="production" />
+      <_CodecovTestFlags Include="@(_CodecovFlags)" />
+      <_CodecovTestFlags Include="test" />
     </ItemGroup>
 
-    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
-    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
+    <!-- Upload the coverage file with a 'production' flag, which will be filtered by codecov.io to production code -->
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovProductionFlags, ',')" />
+    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovProductionFlags, ',')" />
+
+    <!-- Upload the coverage file with a 'test' flag, which will be filtered by codecov.io to test code -->
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovTestFlags, ',')" />
+    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ') --flag @(_CodecovTestFlags, ',')" />
   </Target>
 
 </Project>

codecov.yml

@@ -9,11 +9,9 @@ coverage:
       default: false
   fixes:
     - "build/::/"
-  ignore:
-    - "test"
 
 comment:
-  layout: "diff"
+  layout: "diff,flags"
 
 flags:
   production:

test/Directory.Build.targets

@@ -2,4 +2,18 @@
 
   <Import Project="..\Directory.Build.targets" />
 
+  <!-- Workaround for https://github.com/tonerdo/coverlet/pull/318 -->
+  <Target Name="UpdateTargetPathForCoverage" BeforeTargets="InstrumentModulesNoBuild;InstrumentModulesAfterBuild">
+    <PropertyGroup>
+      <_CoverletOriginalTargetPath>$(TargetPath)</_CoverletOriginalTargetPath>
+      <TargetPath>$([System.IO.Path]::GetDirectoryName($(TargetPath)))\NonExistent.dll</TargetPath>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="UpdateTargetPathAfterCoverage" AfterTargets="InstrumentModulesNoBuild;InstrumentModulesAfterBuild">
+    <PropertyGroup>
+      <TargetPath>$(_CoverletOriginalTargetPath)</TargetPath>
+    </PropertyGroup>
+  </Target>
+
 </Project>