Fix pytests (#274)

- Marks tests as `gpu`, `cpu_and_gpu` or `azureml`. Tests marked `gpu`  and `azureml` are not run in the normal test set, only on the AzureML run triggered by the PR builds. Long tests like test_submit_for_inference are no longer run as part of the main set.
- Cleans up pytest.ini

Tests/Common/test_build_config.py

@@ -95,14 +95,13 @@ def test_config_non_overridable_params() -> None:
         assert "The following parameters cannot be overriden" in ex.value.args[0]
 
 
-@pytest.mark.gpu
 def test_config_with_typo() -> None:
     with pytest.raises(ValueError) as ex:
         ModelConfigBase(num_epochsi=100)
     assert "The following parameters do not exist: ['num_epochsi']" in ex.value.args[0]
 
 
-@pytest.mark.gpu
+@pytest.mark.cpu_and_gpu
 def test_dataset_reader_workers() -> None:
     """
     Test to make sure the number of dataset reader workers are set correctly

Tests/ML/pipelines/test_forward_pass.py

@@ -194,7 +194,7 @@ def test_amp_activated(use_model_parallel: bool,
 
 
 @pytest.mark.skipif(common_util.is_windows(), reason="Has issues on windows build")
-@pytest.mark.gpu
+@pytest.mark.cpu_and_gpu
 @pytest.mark.parametrize("use_gpu_override", [False, True])
 def test_use_gpu_flag(use_gpu_override: bool) -> None:
     config = DeepLearningConfig(should_validate=False)
@@ -217,8 +217,8 @@ def test_use_gpu_flag(use_gpu_override: bool) -> None:
             assert config.use_gpu == use_gpu_override
 
 
-@pytest.mark.gpu
-def test_mean_teacher_model() -> None:
+@pytest.mark.azureml
+def test_mean_teacher_model(test_output_dirs: TestOutputDirectories) -> None:
     """
     Test training and weight updates of the mean teacher model computation.
     """
@@ -233,6 +233,8 @@ def test_mean_teacher_model() -> None:
             return model.parameters()
 
     config = DummyClassification()
+    config.set_output_to(test_output_dirs.root_dir)
+
     config.num_epochs = 1
     # Set train batch size to be arbitrary big to ensure we have only one training step
     # i.e. one mean teacher update.

Tests/ML/runners/test_submit_for_inference.py

@@ -15,9 +15,9 @@ from Tests.Common.test_util import DEFAULT_MODEL_ID_NUMERIC
 
 
 @pytest.mark.skipif(common_util.is_windows(), reason="Testing on Linux is enough")
-# Test does not actually need GPU, but we put it in the GPU set as it takes a while to run
+# Put it in the azureml set as it takes a while to run
 # and we don't want to make the main set even longer.
-@pytest.mark.gpu
+@pytest.mark.azureml
 def test_submit_for_inference() -> None:
     args = ["--image_file", "Tests/ML/test_data/train_and_test_data/id1_channel1.nii.gz",
             "--model_id", DEFAULT_MODEL_ID_NUMERIC,

azure-pipelines/build-pr.yml

@@ -31,7 +31,7 @@ jobs:
       - template: train_template.yml
         parameters:
           wait_for_completion: 'True'
-          pytest_mark: 'gpu'
+          pytest_mark: 'gpu or cpu_and_gpu or azureml'
           max_run_duration: '1h'
           more_switches: '--use_dataset_mount=True'
       - task: PublishTestResults@2

azure-pipelines/build.yaml

@@ -65,7 +65,7 @@ steps:
   # hence don't set PYTHONPATH
   - bash: |
       source activate InnerEye
-      pytest ./Tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html -n 2 --dist=loadscope --verbose
+      pytest ./Tests/ -m "not gpu and not azureml" --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html -n 2 --dist=loadscope --verbose
     env:
       APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey)
       DATASETS_ACCOUNT_KEY: $(InnerEyePublicDatasetsStorageKey)

conftest.py

@@ -17,7 +17,7 @@ from InnerEye.Common.output_directories import TestOutputDirectories, make_test_
 from Tests.fixed_paths_for_tests import TEST_OUTPUTS_PATH
 
 
-@pytest.yield_fixture(autouse=True, scope='session')
+@pytest.fixture(autouse=True, scope='session')
 def test_suite_setup() -> Generator:
     # create a default outputs root for all tests
     make_test_output_dir(TEST_OUTPUTS_PATH)

pytest.ini

@@ -1,3 +1,8 @@
 [pytest]
-testpaths=ML/tests Common/tests InnerEye/tests Azure/tests
-norecursedirs=lib include build test_data logs
+testpaths=Tests TestsOutsidePackage TestSubmodule
+norecursedirs=azure-pipelines docs InnerEye logs outputs test_data
+addopts=--strict-markers
+markers=
+    gpu: Test needs a GPU to run
+    cpu_and_gpu: Test needs to run twice, once on CPU and once on GPU
+    azureml: Long running test, run this test as part of the PR build run on AzureML