ENH: Improve regression tests (#827)

Closes #740. Updates the lung model regression test to use the latest
parameters and train for a substantial number of steps to ensure
training is progressing as expecting. The small number of epochs and
smaller data subset is used as running a full training run isn't
feasible. The new test runs in < 30 minutes but on real data.

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Bypassing branch protections for failing PyTest as the test that is failing is `Tests/SSL/test_ssl_containers.py::test_innereye_ssl_container_cifar10_resnet_simclr` as the URL for CIFAR seems to be broken currently (returning 500 server error).

InnerEye/ML/configs/regression/LungRegression.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+from InnerEye.ML.configs.segmentation.Lung import Lung
+from InnerEye.ML.utils.split_dataset import DatasetSplits
+
+
+class LungRegression(Lung):
+    """
+    Model used for regression testing the Lung model. Uses the same data as the published model
+    but only runs for a small number of epochs and a smaller subset of the data. This is to ensure that
+    the training time is kept under 30 minutes to prevents slowing down PRs significantly.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(
+            azure_dataset_id="lung_for_regression_test_11_2022",
+            train_batch_size=3,
+            num_epochs=25,
+            pl_deterministic=True,
+            test_crop_size=(64, 256, 256),
+        )
+
+    def get_model_train_test_dataset_splits(self, dataset_df: pd.DataFrame) -> DatasetSplits:
+        train = list(map(str, range(0, 10)))
+        val = list(map(str, range(10, 12)))
+        test = list(map(str, range(12, 13)))
+
+        return DatasetSplits.from_subject_ids(
+            df=dataset_df,
+            train_ids=train,
+            val_ids=val,
+            test_ids=test,
+        )

InnerEye/ML/configs/segmentation/Lung.py

@@ -4,7 +4,7 @@
 #  ------------------------------------------------------------------------------------------
 from typing import Any
 
-import numpy
+import numpy as np
 import pandas as pd
 
 from InnerEye.ML.config import PhotometricNormalizationMethod, SegmentationLoss, \
@@ -71,7 +71,7 @@ class Lung(SegmentationModelBase):
         test = list(map(str, range(0, 9)))
         train_val = list(dataset_df[~dataset_df.subject.isin(test)].subject.unique())
 
-        val = list(map(str, numpy.random.choice(train_val, int(len(train_val) * 0.1), replace=False)))
+        val = list(map(str, np.random.choice(train_val, int(len(train_val) * 0.1), replace=False)))
         train = [str(x) for x in train_val if x not in val]
 
         return DatasetSplits.from_subject_ids(

InnerEye/ML/run_ml.py

@@ -38,7 +38,7 @@ from InnerEye.Common.common_util import (
 )
 from InnerEye.Common.fixed_paths import INNEREYE_PACKAGE_NAME, PYTHON_ENVIRONMENT_NAME, PYTHON_ENVIRONMENT_VERSION
 from InnerEye.Common.type_annotations import PathOrString
-from InnerEye.ML.baselines_util import compare_folders_and_run_outputs
+from InnerEye.ML.baselines_util import compare_folders_and_run_outputs, compare_scores_against_baselines
 from InnerEye.ML.common import (
     CHECKPOINT_FOLDER, EXTRA_RUN_SUBFOLDER, FINAL_ENSEMBLE_MODEL_FOLDER, FINAL_MODEL_FOLDER, ModelExecutionMode
 )
@@ -62,7 +62,8 @@ from InnerEye.ML.scalar_config import ScalarModelBase
 from InnerEye.ML.utils.checkpoint_handling import CheckpointHandler, download_all_checkpoints_from_run
 from InnerEye.ML.visualizers import activation_maps
 from InnerEye.ML.visualizers.plot_cross_validation import (
-    get_config_and_results_for_offline_runs, plot_cross_validation_from_files
+    crossval_config_from_model_config, get_config_and_results_for_offline_runs,
+    plot_cross_validation, plot_cross_validation_from_files, unroll_aggregate_metrics
 )
 
 ModelDeploymentHookSignature = Callable[[LightningContainer, AzureConfig, Model, ModelProcessing], Any]
@@ -520,7 +521,6 @@ class MLRunner:
         """
         if not isinstance(self.model_config, SegmentationModelBase):  # keep type checker happy
             return
-        from InnerEye.ML.baselines_util import compare_scores_against_baselines
         with logging_section("Comparing scores against baselines"):
             compare_scores_against_baselines(self.model_config, self.azure_config, model_proc)
 
@@ -812,9 +812,6 @@ class MLRunner:
         remove_file_or_directory(other_runs_dir)
 
     def plot_cross_validation_and_upload_results(self) -> Path:
-        from InnerEye.ML.visualizers.plot_cross_validation import (
-            crossval_config_from_model_config, plot_cross_validation, unroll_aggregate_metrics
-        )
 
         # perform aggregation as cross val splits are now ready
         plot_crossval_config = crossval_config_from_model_config(self.innereye_config)

RegressionTestResults/PR_LungRegression/OUTPUT/Train/epoch_metrics.csv

@@ -0,0 +1,26 @@
+subject_count,loss,learning_rate,Dice/AverageAcrossStructures,Dice/spinalcord,Dice/lung_r,Dice/lung_l,Dice/heart,Dice/esophagus,VoxelCount/spinalcord,VoxelCount/lung_r,VoxelCount/lung_l,VoxelCount/heart,VoxelCount/esophagus,epoch,cross_validation_split_index
+10.000000,0.832762,0.001000,0.088804,0.008171,0.130580,0.179863,0.123554,0.001854,6686.799805,216837.703125,257620.000000,193190.703125,6582.299805,0,-1
+10.000000,0.747988,0.000964,0.195486,0.039705,0.300637,0.385019,0.246324,0.005748,10107.599609,322953.000000,375242.187500,224126.593750,8658.200195,1,-1
+10.000000,0.733532,0.000928,0.207074,0.044004,0.172670,0.627448,0.182372,0.008876,8958.400391,269941.312500,387651.312500,152001.093750,9244.200195,2,-1
+10.000000,0.738423,0.000892,0.158516,0.050240,0.233535,0.309313,0.195073,0.004419,9492.099609,353211.187500,187868.093750,170496.000000,8188.899902,3,-1
+10.000000,0.670263,0.000856,0.258717,0.065482,0.505172,0.454871,0.262194,0.005867,9068.200195,412575.812500,348539.093750,233761.296875,8355.000000,4,-1
+10.000000,0.699686,0.000820,0.212306,0.096302,0.436839,0.360685,0.164160,0.003543,11119.799805,419661.312500,191479.703125,126601.898438,8169.100098,5,-1
+10.000000,0.660631,0.000783,0.262458,0.075867,0.532807,0.503855,0.196402,0.003359,8739.599609,344446.687500,317570.593750,171165.906250,7891.500000,6,-1
+10.000000,0.677271,0.000747,0.228661,0.123588,0.295020,0.534879,0.188688,0.001133,11059.500000,305233.906250,249293.093750,154528.406250,6892.700195,7,-1
+10.000000,0.601600,0.000710,0.327465,0.295261,0.471335,0.638725,0.231815,0.000188,12510.900391,356627.000000,325584.000000,177084.593750,8867.799805,8,-1
+10.000000,0.667036,0.000673,0.254113,0.095632,0.400353,0.619570,0.153179,0.001832,6007.600098,288038.812500,217527.906250,123174.601562,6042.299805,9,-1
+10.000000,0.594998,0.000635,0.343973,0.268345,0.451255,0.691011,0.309053,0.000201,9545.299805,277721.312500,360818.812500,188714.703125,8314.799805,10,-1
+10.000000,0.584550,0.000597,0.378147,0.472501,0.376896,0.754000,0.287211,0.000128,10003.099609,208687.593750,335941.812500,167208.296875,8359.000000,11,-1
+10.000000,0.577460,0.000560,0.366391,0.424670,0.429965,0.768400,0.208871,0.000049,8855.099609,234015.296875,309029.500000,120215.898438,8596.400391,12,-1
+10.000000,0.561530,0.000521,0.385332,0.506304,0.401121,0.706737,0.312497,0.000000,10543.099609,238946.500000,358745.312500,203072.796875,8593.500000,13,-1
+10.000000,0.529850,0.000483,0.447172,0.647319,0.559346,0.710558,0.318601,0.000036,11070.799805,331267.312500,269231.093750,130471.296875,8828.700195,14,-1
+10.000000,0.527176,0.000444,0.430784,0.465458,0.476638,0.786585,0.425240,0.000000,8821.900391,241096.796875,318621.812500,177420.000000,9640.200195,15,-1
+10.000000,0.489316,0.000405,0.493526,0.767579,0.501706,0.813794,0.384549,0.000000,12552.700195,222082.406250,392410.093750,175042.296875,8950.299805,16,-1
+10.000000,0.474892,0.000365,0.499195,0.723972,0.639440,0.770296,0.362267,0.000000,12930.500000,373130.500000,283151.500000,148432.796875,8927.799805,17,-1
+10.000000,0.573210,0.000325,0.373418,0.420964,0.591545,0.519953,0.334629,0.000000,7047.000000,317429.812500,170473.500000,169108.906250,6494.600098,18,-1
+10.000000,0.485945,0.000284,0.489797,0.586358,0.574224,0.858835,0.429566,0.000000,8980.400391,295972.500000,311818.812500,192087.906250,8515.500000,19,-1
+10.000000,0.486404,0.000243,0.465467,0.390454,0.692863,0.804858,0.439158,0.000000,6716.500000,313480.000000,235133.703125,183772.406250,8114.500000,20,-1
+10.000000,0.408884,0.000200,0.601795,0.851341,0.678334,0.891533,0.587767,0.000000,12468.099609,439076.500000,322387.312500,218996.703125,9099.900391,21,-1
+10.000000,0.463869,0.000157,0.510398,0.622862,0.571223,0.841766,0.516138,0.000000,9174.400391,295294.312500,350301.593750,225853.703125,8625.500000,22,-1
+10.000000,0.445714,0.000112,0.528343,0.641444,0.673257,0.905913,0.421103,0.000000,9266.099609,312185.687500,330079.312500,147796.000000,8245.500000,23,-1
+10.000000,0.430474,0.000065,0.561517,0.809915,0.697628,0.863772,0.436269,0.000000,12641.200195,352071.000000,284182.812500,161978.000000,8730.500000,24,-1

RegressionTestResults/PR_LungRegression/OUTPUT/Val/epoch_metrics.csv

@@ -0,0 +1,26 @@
+subject_count,loss,Dice/AverageAcrossStructures,Dice/spinalcord,Dice/lung_r,Dice/lung_l,Dice/heart,Dice/esophagus,VoxelCount/spinalcord,VoxelCount/lung_r,VoxelCount/lung_l,VoxelCount/heart,VoxelCount/esophagus,epoch,cross_validation_split_index
+2.000000,0.893225,0.082663,0.005430,0.013595,0.393810,0.000000,0.000481,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,0,-1
+2.000000,0.919233,0.069938,0.005234,0.022307,0.318086,0.000000,0.004062,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,1,-1
+2.000000,0.918322,0.068914,0.005276,0.028076,0.310201,0.000000,0.001016,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,2,-1
+2.000000,0.905026,0.072065,0.005652,0.021452,0.331951,0.000000,0.001269,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,3,-1
+2.000000,0.820528,0.108847,0.007308,0.031401,0.498667,0.000474,0.006385,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,4,-1
+2.000000,0.786676,0.127091,0.007832,0.095858,0.522245,0.003091,0.006428,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,5,-1
+2.000000,0.713197,0.182942,0.012165,0.227228,0.618754,0.048619,0.007946,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,6,-1
+2.000000,0.629321,0.242206,0.062172,0.216384,0.664476,0.265469,0.002531,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,7,-1
+2.000000,0.621613,0.296482,0.225135,0.303712,0.647584,0.301176,0.004803,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,8,-1
+2.000000,0.662866,0.273958,0.099871,0.465255,0.501049,0.295022,0.008592,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,9,-1
+2.000000,0.575193,0.402411,0.331851,0.527946,0.768343,0.383913,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,10,-1
+2.000000,0.561944,0.420336,0.316582,0.590333,0.815079,0.379688,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,11,-1
+2.000000,0.547804,0.440356,0.352461,0.642426,0.804500,0.402393,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,12,-1
+2.000000,0.546901,0.434397,0.268577,0.637747,0.817397,0.448263,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,13,-1
+2.000000,0.545300,0.432454,0.331116,0.626299,0.767622,0.436889,0.000346,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,14,-1
+2.000000,0.521060,0.445840,0.306907,0.676425,0.848107,0.397760,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,15,-1
+2.000000,0.509127,0.455875,0.447679,0.631471,0.832142,0.368082,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,16,-1
+2.000000,0.528844,0.429593,0.346822,0.597915,0.819371,0.383855,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,17,-1
+2.000000,0.520242,0.440362,0.383577,0.579982,0.801499,0.436751,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,18,-1
+2.000000,0.484258,0.473486,0.405042,0.620975,0.859299,0.482115,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,19,-1
+2.000000,0.486916,0.465447,0.406759,0.645090,0.858185,0.417199,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,20,-1
+2.000000,0.483584,0.472214,0.471022,0.649105,0.861842,0.379099,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,21,-1
+2.000000,0.482063,0.488386,0.596112,0.614544,0.857001,0.374274,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,22,-1
+2.000000,0.476012,0.498064,0.613599,0.610088,0.857787,0.408848,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,23,-1
+2.000000,0.462136,0.517481,0.620620,0.629822,0.881337,0.455624,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,24,-1

RegressionTestResults/PR_LungRegression/OUTPUT/dataset.csv

@@ -0,0 +1,79 @@
+subject,filePath,channel,seriesId
+0,0/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+1,1/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+2,2/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+3,3/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+4,4/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+5,5/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+6,6/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+7,7/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+8,8/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+9,9/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+10,10/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+11,11/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+12,12/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461

RegressionTestResults/PR_LungRegression/OUTPUT/test_dataset.csv

@@ -0,0 +1,7 @@
+subject,filePath,channel,seriesId
+12,12/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461
+12,12/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461

RegressionTestResults/PR_LungRegression/OUTPUT/train_dataset.csv

@@ -0,0 +1,61 @@
+subject,filePath,channel,seriesId
+0,0/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+0,0/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186
+1,1/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+1,1/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444
+2,2/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+2,2/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264
+3,3/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+3,3/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553
+4,4/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+4,4/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548
+5,5/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+5,5/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243
+6,6/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+6,6/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416
+7,7/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+7,7/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855
+8,8/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+8,8/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012
+9,9/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534
+9,9/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534

RegressionTestResults/PR_LungRegression/OUTPUT/val_dataset.csv

@@ -0,0 +1,13 @@
+subject,filePath,channel,seriesId
+10,10/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+10,10/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216
+11,11/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711
+11,11/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711

azure-pipelines/build-pr.yml

@@ -49,7 +49,7 @@ jobs:
     steps:
       - template: build.yaml
 
-  - job: TrainInAzureML
+  - job: BasicModelRegressionTest
     dependsOn: CancelPreviousJobs
     variables:
       - name: tag
@@ -110,7 +110,6 @@ jobs:
           pytest_mark: "inference or after_training"
           test_run_title: tests_after_train_submodule
 
-
   # Train a 2-element ensemble model
   - job: TrainEnsemble
     dependsOn: CancelPreviousJobs
@@ -199,23 +198,19 @@ jobs:
           pytest_mark: after_training_hello_container
           test_run_title: tests_after_training_hello_container
 
-  # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against
+  - job: TrainLungRegression
-  # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to
-  # trigger the bug, total runtime 10min
-  - job: TrainLung
     dependsOn: CancelPreviousJobs
     variables:
       - name: model
-        value: 'Lung'
+        value: 'LungRegression'
       - name: tag
-        value: 'LungPR'
+        value: 'TrainLungRegressionModel'
       - name: more_switches
-        value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False '
+        value: '--log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_LungRegression --regression_test_csv_tolerance=0.001'
     pool:
       vmImage: 'ubuntu-20.04'
     steps:
       - template: train_template.yml
         parameters:
           wait_for_completion: 'True'
-          pytest_mark: ''
           max_run_duration: '30m'