From 9083966673f040154cfd2913deb2bd192593aed8 Mon Sep 17 00:00:00 2001 From: Peter Hessey <45258028+peterhessey@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:05:46 +0000 Subject: [PATCH] ENH: Improve regression tests (#827) Closes #740. Updates the lung model regression test to use the latest parameters and train for a substantial number of steps to ensure training is progressing as expecting. The small number of epochs and smaller data subset is used as running a full training run isn't feasible. The new test runs in < 30 minutes but on real data. Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Bypassing branch protections for failing PyTest as the test that is failing is `Tests/SSL/test_ssl_containers.py::test_innereye_ssl_container_cifar10_resnet_simclr` as the URL for CIFAR seems to be broken currently (returning 500 server error). --- .../ML/configs/regression/LungRegression.py | 33 ++++++++ InnerEye/ML/configs/segmentation/Lung.py | 4 +- InnerEye/ML/run_ml.py | 9 +-- .../OUTPUT/Train/epoch_metrics.csv | 26 ++++++ .../OUTPUT/Val/epoch_metrics.csv | 26 ++++++ .../PR_LungRegression/OUTPUT/dataset.csv | 79 +++++++++++++++++++ .../PR_LungRegression/OUTPUT/test_dataset.csv | 7 ++ .../OUTPUT/train_dataset.csv | 61 ++++++++++++++ .../PR_LungRegression/OUTPUT/val_dataset.csv | 13 +++ azure-pipelines/build-pr.yml | 15 ++-- 10 files changed, 255 insertions(+), 18 deletions(-) create mode 100644 InnerEye/ML/configs/regression/LungRegression.py create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/Train/epoch_metrics.csv create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/Val/epoch_metrics.csv create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/dataset.csv create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/test_dataset.csv create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/train_dataset.csv create mode 100755 RegressionTestResults/PR_LungRegression/OUTPUT/val_dataset.csv diff --git a/InnerEye/ML/configs/regression/LungRegression.py b/InnerEye/ML/configs/regression/LungRegression.py new file mode 100644 index 00000000..0ab94f35 --- /dev/null +++ b/InnerEye/ML/configs/regression/LungRegression.py @@ -0,0 +1,33 @@ +import pandas as pd + +from InnerEye.ML.configs.segmentation.Lung import Lung +from InnerEye.ML.utils.split_dataset import DatasetSplits + + +class LungRegression(Lung): + """ + Model used for regression testing the Lung model. Uses the same data as the published model + but only runs for a small number of epochs and a smaller subset of the data. This is to ensure that + the training time is kept under 30 minutes to prevents slowing down PRs significantly. + """ + + def __init__(self) -> None: + super().__init__( + azure_dataset_id="lung_for_regression_test_11_2022", + train_batch_size=3, + num_epochs=25, + pl_deterministic=True, + test_crop_size=(64, 256, 256), + ) + + def get_model_train_test_dataset_splits(self, dataset_df: pd.DataFrame) -> DatasetSplits: + train = list(map(str, range(0, 10))) + val = list(map(str, range(10, 12))) + test = list(map(str, range(12, 13))) + + return DatasetSplits.from_subject_ids( + df=dataset_df, + train_ids=train, + val_ids=val, + test_ids=test, + ) diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index 26c76fff..6f949a83 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------------------ from typing import Any -import numpy +import numpy as np import pandas as pd from InnerEye.ML.config import PhotometricNormalizationMethod, SegmentationLoss, \ @@ -71,7 +71,7 @@ class Lung(SegmentationModelBase): test = list(map(str, range(0, 9))) train_val = list(dataset_df[~dataset_df.subject.isin(test)].subject.unique()) - val = list(map(str, numpy.random.choice(train_val, int(len(train_val) * 0.1), replace=False))) + val = list(map(str, np.random.choice(train_val, int(len(train_val) * 0.1), replace=False))) train = [str(x) for x in train_val if x not in val] return DatasetSplits.from_subject_ids( diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py index 73ea7277..8266ea4a 100644 --- a/InnerEye/ML/run_ml.py +++ b/InnerEye/ML/run_ml.py @@ -38,7 +38,7 @@ from InnerEye.Common.common_util import ( ) from InnerEye.Common.fixed_paths import INNEREYE_PACKAGE_NAME, PYTHON_ENVIRONMENT_NAME, PYTHON_ENVIRONMENT_VERSION from InnerEye.Common.type_annotations import PathOrString -from InnerEye.ML.baselines_util import compare_folders_and_run_outputs +from InnerEye.ML.baselines_util import compare_folders_and_run_outputs, compare_scores_against_baselines from InnerEye.ML.common import ( CHECKPOINT_FOLDER, EXTRA_RUN_SUBFOLDER, FINAL_ENSEMBLE_MODEL_FOLDER, FINAL_MODEL_FOLDER, ModelExecutionMode ) @@ -62,7 +62,8 @@ from InnerEye.ML.scalar_config import ScalarModelBase from InnerEye.ML.utils.checkpoint_handling import CheckpointHandler, download_all_checkpoints_from_run from InnerEye.ML.visualizers import activation_maps from InnerEye.ML.visualizers.plot_cross_validation import ( - get_config_and_results_for_offline_runs, plot_cross_validation_from_files + crossval_config_from_model_config, get_config_and_results_for_offline_runs, + plot_cross_validation, plot_cross_validation_from_files, unroll_aggregate_metrics ) ModelDeploymentHookSignature = Callable[[LightningContainer, AzureConfig, Model, ModelProcessing], Any] @@ -520,7 +521,6 @@ class MLRunner: """ if not isinstance(self.model_config, SegmentationModelBase): # keep type checker happy return - from InnerEye.ML.baselines_util import compare_scores_against_baselines with logging_section("Comparing scores against baselines"): compare_scores_against_baselines(self.model_config, self.azure_config, model_proc) @@ -812,9 +812,6 @@ class MLRunner: remove_file_or_directory(other_runs_dir) def plot_cross_validation_and_upload_results(self) -> Path: - from InnerEye.ML.visualizers.plot_cross_validation import ( - crossval_config_from_model_config, plot_cross_validation, unroll_aggregate_metrics - ) # perform aggregation as cross val splits are now ready plot_crossval_config = crossval_config_from_model_config(self.innereye_config) diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/Train/epoch_metrics.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/Train/epoch_metrics.csv new file mode 100755 index 00000000..71d849af --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/Train/epoch_metrics.csv @@ -0,0 +1,26 @@ +subject_count,loss,learning_rate,Dice/AverageAcrossStructures,Dice/spinalcord,Dice/lung_r,Dice/lung_l,Dice/heart,Dice/esophagus,VoxelCount/spinalcord,VoxelCount/lung_r,VoxelCount/lung_l,VoxelCount/heart,VoxelCount/esophagus,epoch,cross_validation_split_index +10.000000,0.832762,0.001000,0.088804,0.008171,0.130580,0.179863,0.123554,0.001854,6686.799805,216837.703125,257620.000000,193190.703125,6582.299805,0,-1 +10.000000,0.747988,0.000964,0.195486,0.039705,0.300637,0.385019,0.246324,0.005748,10107.599609,322953.000000,375242.187500,224126.593750,8658.200195,1,-1 +10.000000,0.733532,0.000928,0.207074,0.044004,0.172670,0.627448,0.182372,0.008876,8958.400391,269941.312500,387651.312500,152001.093750,9244.200195,2,-1 +10.000000,0.738423,0.000892,0.158516,0.050240,0.233535,0.309313,0.195073,0.004419,9492.099609,353211.187500,187868.093750,170496.000000,8188.899902,3,-1 +10.000000,0.670263,0.000856,0.258717,0.065482,0.505172,0.454871,0.262194,0.005867,9068.200195,412575.812500,348539.093750,233761.296875,8355.000000,4,-1 +10.000000,0.699686,0.000820,0.212306,0.096302,0.436839,0.360685,0.164160,0.003543,11119.799805,419661.312500,191479.703125,126601.898438,8169.100098,5,-1 +10.000000,0.660631,0.000783,0.262458,0.075867,0.532807,0.503855,0.196402,0.003359,8739.599609,344446.687500,317570.593750,171165.906250,7891.500000,6,-1 +10.000000,0.677271,0.000747,0.228661,0.123588,0.295020,0.534879,0.188688,0.001133,11059.500000,305233.906250,249293.093750,154528.406250,6892.700195,7,-1 +10.000000,0.601600,0.000710,0.327465,0.295261,0.471335,0.638725,0.231815,0.000188,12510.900391,356627.000000,325584.000000,177084.593750,8867.799805,8,-1 +10.000000,0.667036,0.000673,0.254113,0.095632,0.400353,0.619570,0.153179,0.001832,6007.600098,288038.812500,217527.906250,123174.601562,6042.299805,9,-1 +10.000000,0.594998,0.000635,0.343973,0.268345,0.451255,0.691011,0.309053,0.000201,9545.299805,277721.312500,360818.812500,188714.703125,8314.799805,10,-1 +10.000000,0.584550,0.000597,0.378147,0.472501,0.376896,0.754000,0.287211,0.000128,10003.099609,208687.593750,335941.812500,167208.296875,8359.000000,11,-1 +10.000000,0.577460,0.000560,0.366391,0.424670,0.429965,0.768400,0.208871,0.000049,8855.099609,234015.296875,309029.500000,120215.898438,8596.400391,12,-1 +10.000000,0.561530,0.000521,0.385332,0.506304,0.401121,0.706737,0.312497,0.000000,10543.099609,238946.500000,358745.312500,203072.796875,8593.500000,13,-1 +10.000000,0.529850,0.000483,0.447172,0.647319,0.559346,0.710558,0.318601,0.000036,11070.799805,331267.312500,269231.093750,130471.296875,8828.700195,14,-1 +10.000000,0.527176,0.000444,0.430784,0.465458,0.476638,0.786585,0.425240,0.000000,8821.900391,241096.796875,318621.812500,177420.000000,9640.200195,15,-1 +10.000000,0.489316,0.000405,0.493526,0.767579,0.501706,0.813794,0.384549,0.000000,12552.700195,222082.406250,392410.093750,175042.296875,8950.299805,16,-1 +10.000000,0.474892,0.000365,0.499195,0.723972,0.639440,0.770296,0.362267,0.000000,12930.500000,373130.500000,283151.500000,148432.796875,8927.799805,17,-1 +10.000000,0.573210,0.000325,0.373418,0.420964,0.591545,0.519953,0.334629,0.000000,7047.000000,317429.812500,170473.500000,169108.906250,6494.600098,18,-1 +10.000000,0.485945,0.000284,0.489797,0.586358,0.574224,0.858835,0.429566,0.000000,8980.400391,295972.500000,311818.812500,192087.906250,8515.500000,19,-1 +10.000000,0.486404,0.000243,0.465467,0.390454,0.692863,0.804858,0.439158,0.000000,6716.500000,313480.000000,235133.703125,183772.406250,8114.500000,20,-1 +10.000000,0.408884,0.000200,0.601795,0.851341,0.678334,0.891533,0.587767,0.000000,12468.099609,439076.500000,322387.312500,218996.703125,9099.900391,21,-1 +10.000000,0.463869,0.000157,0.510398,0.622862,0.571223,0.841766,0.516138,0.000000,9174.400391,295294.312500,350301.593750,225853.703125,8625.500000,22,-1 +10.000000,0.445714,0.000112,0.528343,0.641444,0.673257,0.905913,0.421103,0.000000,9266.099609,312185.687500,330079.312500,147796.000000,8245.500000,23,-1 +10.000000,0.430474,0.000065,0.561517,0.809915,0.697628,0.863772,0.436269,0.000000,12641.200195,352071.000000,284182.812500,161978.000000,8730.500000,24,-1 diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/Val/epoch_metrics.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/Val/epoch_metrics.csv new file mode 100755 index 00000000..2b86e159 --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/Val/epoch_metrics.csv @@ -0,0 +1,26 @@ +subject_count,loss,Dice/AverageAcrossStructures,Dice/spinalcord,Dice/lung_r,Dice/lung_l,Dice/heart,Dice/esophagus,VoxelCount/spinalcord,VoxelCount/lung_r,VoxelCount/lung_l,VoxelCount/heart,VoxelCount/esophagus,epoch,cross_validation_split_index +2.000000,0.893225,0.082663,0.005430,0.013595,0.393810,0.000000,0.000481,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,0,-1 +2.000000,0.919233,0.069938,0.005234,0.022307,0.318086,0.000000,0.004062,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,1,-1 +2.000000,0.918322,0.068914,0.005276,0.028076,0.310201,0.000000,0.001016,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,2,-1 +2.000000,0.905026,0.072065,0.005652,0.021452,0.331951,0.000000,0.001269,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,3,-1 +2.000000,0.820528,0.108847,0.007308,0.031401,0.498667,0.000474,0.006385,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,4,-1 +2.000000,0.786676,0.127091,0.007832,0.095858,0.522245,0.003091,0.006428,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,5,-1 +2.000000,0.713197,0.182942,0.012165,0.227228,0.618754,0.048619,0.007946,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,6,-1 +2.000000,0.629321,0.242206,0.062172,0.216384,0.664476,0.265469,0.002531,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,7,-1 +2.000000,0.621613,0.296482,0.225135,0.303712,0.647584,0.301176,0.004803,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,8,-1 +2.000000,0.662866,0.273958,0.099871,0.465255,0.501049,0.295022,0.008592,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,9,-1 +2.000000,0.575193,0.402411,0.331851,0.527946,0.768343,0.383913,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,10,-1 +2.000000,0.561944,0.420336,0.316582,0.590333,0.815079,0.379688,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,11,-1 +2.000000,0.547804,0.440356,0.352461,0.642426,0.804500,0.402393,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,12,-1 +2.000000,0.546901,0.434397,0.268577,0.637747,0.817397,0.448263,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,13,-1 +2.000000,0.545300,0.432454,0.331116,0.626299,0.767622,0.436889,0.000346,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,14,-1 +2.000000,0.521060,0.445840,0.306907,0.676425,0.848107,0.397760,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,15,-1 +2.000000,0.509127,0.455875,0.447679,0.631471,0.832142,0.368082,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,16,-1 +2.000000,0.528844,0.429593,0.346822,0.597915,0.819371,0.383855,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,17,-1 +2.000000,0.520242,0.440362,0.383577,0.579982,0.801499,0.436751,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,18,-1 +2.000000,0.484258,0.473486,0.405042,0.620975,0.859299,0.482115,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,19,-1 +2.000000,0.486916,0.465447,0.406759,0.645090,0.858185,0.417199,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,20,-1 +2.000000,0.483584,0.472214,0.471022,0.649105,0.861842,0.379099,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,21,-1 +2.000000,0.482063,0.488386,0.596112,0.614544,0.857001,0.374274,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,22,-1 +2.000000,0.476012,0.498064,0.613599,0.610088,0.857787,0.408848,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,23,-1 +2.000000,0.462136,0.517481,0.620620,0.629822,0.881337,0.455624,0.000000,7443.000000,336188.500000,478771.500000,242730.500000,12696.500000,24,-1 diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/dataset.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/dataset.csv new file mode 100755 index 00000000..29acdadf --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/dataset.csv @@ -0,0 +1,79 @@ +subject,filePath,channel,seriesId +0,0/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +1,1/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +2,2/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +3,3/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +4,4/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +5,5/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +6,6/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +7,7/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +8,8/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +9,9/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +10,10/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +11,11/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +12,12/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/test_dataset.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/test_dataset.csv new file mode 100755 index 00000000..a77d06ef --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/test_dataset.csv @@ -0,0 +1,7 @@ +subject,filePath,channel,seriesId +12,12/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 +12,12/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.303839966804062210456566892461 diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/train_dataset.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/train_dataset.csv new file mode 100755 index 00000000..23515467 --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/train_dataset.csv @@ -0,0 +1,61 @@ +subject,filePath,channel,seriesId +0,0/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +0,0/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.492964872630309412859177308186 +1,1/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +1,1/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.238478060677653494401819620444 +2,2/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +2,2/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.816327386653410772475068003264 +3,3/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +3,3/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.829677454205016768063779242553 +4,4/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +4,4/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.346635067461584156068474273548 +5,5/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +5,5/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.311348596911338124468620593243 +6,6/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +6,6/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.126657353437037202011096529416 +7,7/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +7,7/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.387640041506394166677046351855 +8,8/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +8,8/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.177073219925349425820713784012 +9,9/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 +9,9/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.304805754789102895117932115534 diff --git a/RegressionTestResults/PR_LungRegression/OUTPUT/val_dataset.csv b/RegressionTestResults/PR_LungRegression/OUTPUT/val_dataset.csv new file mode 100755 index 00000000..eb6e4576 --- /dev/null +++ b/RegressionTestResults/PR_LungRegression/OUTPUT/val_dataset.csv @@ -0,0 +1,13 @@ +subject,filePath,channel,seriesId +10,10/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +10,10/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.205539610686399129384253975216 +11,11/spinalcord.nii.gz,spinalcord,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/lung_r.nii.gz,lung_r,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/lung_l.nii.gz,lung_l,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/heart.nii.gz,heart,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/esophagus.nii.gz,esophagus,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 +11,11/ct.nii.gz,ct,1.3.6.1.4.1.14519.5.2.1.7014.4598.189721990631943197010794868711 diff --git a/azure-pipelines/build-pr.yml b/azure-pipelines/build-pr.yml index 7a5f3d94..b3487c4b 100644 --- a/azure-pipelines/build-pr.yml +++ b/azure-pipelines/build-pr.yml @@ -49,7 +49,7 @@ jobs: steps: - template: build.yaml - - job: TrainInAzureML + - job: BasicModelRegressionTest dependsOn: CancelPreviousJobs variables: - name: tag @@ -110,7 +110,6 @@ jobs: pytest_mark: "inference or after_training" test_run_title: tests_after_train_submodule - # Train a 2-element ensemble model - job: TrainEnsemble dependsOn: CancelPreviousJobs @@ -199,23 +198,19 @@ jobs: pytest_mark: after_training_hello_container test_run_title: tests_after_training_hello_container - # Run the Lung model. This is a large model requiring a docker image with large memory. This tests against - # regressions in AML when requesting more than the default amount of memory. This needs to run with all subjects to - # trigger the bug, total runtime 10min - - job: TrainLung + - job: TrainLungRegression dependsOn: CancelPreviousJobs variables: - name: model - value: 'Lung' + value: 'LungRegression' - name: tag - value: 'LungPR' + value: 'TrainLungRegressionModel' - name: more_switches - value: '--pl_deterministic --num_epochs=1 --feature_channels=16 --show_patch_sampling=0 --train_batch_size=4 --inference_on_val_set=False --inference_on_test_set=False ' + value: '--log_level=DEBUG --regression_test_folder=RegressionTestResults/PR_LungRegression --regression_test_csv_tolerance=0.001' pool: vmImage: 'ubuntu-20.04' steps: - template: train_template.yml parameters: wait_for_completion: 'True' - pytest_mark: '' max_run_duration: '30m'