From eb5f931f200c253f8a3336fcb01cdaa58c6207be Mon Sep 17 00:00:00 2001
From: Anton Schwaighofer <antonsc@microsoft.com>
Date: Wed, 10 Feb 2021 14:29:20 +0000
Subject: [PATCH] Fix error messages in test coverage reporting  (#394)

- Coverage reporting complains that it does not like the HTML output folder.
- Exclude the Tests* folders from the report, so that the overall coverage figures make more sense
---
 .coveragerc                  |  5 +++++
 Tests/ML/test_runner.sh      | 19 -------------------
 azure-pipelines/build-pr.yml |  3 +++
 azure-pipelines/build.yaml   |  6 +++---
 4 files changed, 11 insertions(+), 22 deletions(-)
 create mode 100644 .coveragerc
 delete mode 100644 Tests/ML/test_runner.sh

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..724a6c1c
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,5 @@
+[run]
+omit =
+    Tests/*
+    TestsOutsidePackage/*
+    TestSubmodule/*
diff --git a/Tests/ML/test_runner.sh b/Tests/ML/test_runner.sh
deleted file mode 100644
index f43d95f1..00000000
--- a/Tests/ML/test_runner.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# Assumes all tests if no argument was given
-# cpu: test everything that does need a gpu
-# gpu: test everything that needs a gpu
-# all: run all tests
-
-if [[ $# -eq 0 ]]
-then
-    folder=../tests
-    echo running all tests
-else
-    echo unknown arg, exiting
-    exit -1
-fi
-
-cd ../ML
-
-python -m pytest -v -s --cov-report term-missing --cov=. ${folder}
diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml
index 229a4f3b..832ad623 100644
--- a/azure-pipelines/build-pr.yml
+++ b/azure-pipelines/build-pr.yml
@@ -12,6 +12,9 @@ variables:
   tag: ''
   number_of_cross_validation_splits: 0
   cluster: 'training-nc12'
+  # Disable a spurious warning
+  # https://stackoverflow.com/questions/56859264/publishing-code-coverage-results-from-reportgenerator-not-working
+  disable.coverage.autogenerate: 'true'
 
 jobs:
   - job: Windows
diff --git a/azure-pipelines/build.yaml b/azure-pipelines/build.yaml
index 02f9b23b..db461414 100644
--- a/azure-pipelines/build.yaml
+++ b/azure-pipelines/build.yaml
@@ -74,7 +74,7 @@ steps:
   # hence don't set PYTHONPATH
   - bash: |
       source activate InnerEye
-      pytest ./Tests/ -m "not gpu and not azureml and not after_training_single_run and not after_training_ensemble_run and not inference and not after_training_2node" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html -n 2 --dist=loadscope --verbose
+      pytest ./Tests/ -m "not (gpu or azureml or after_training_single_run or after_training_ensemble_run or inference or after_training_2node)" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-config=.coveragerc --cov-report=xml -n 2 --dist=loadscope --verbose
     env:
       APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey)
       DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey)
@@ -103,8 +103,8 @@ steps:
   - task: PublishCodeCoverageResults@1
     inputs:
       codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
-      reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
+      summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+      reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
     condition: succeededOrFailed()
 
   - task: ComponentGovernanceComponentDetection@0