27 строки
2.5 KiB
YAML
27 строки
2.5 KiB
YAML
steps:
|
|
- template: inner_eye_env.yml
|
|
|
|
# Create a directory structure with a runner script and the InnerEye submodule, starting training from there.
|
|
# Then do a recovery run in AzureML to see if that works well.
|
|
# After the end of this step, most_recent_run.txt will contain the run recovery ID of the training run, not
|
|
# the recovery run.
|
|
- bash: |
|
|
set -e # This makes the script fail if any command in it fails, not just the last one
|
|
source activate InnerEye
|
|
mkdir $(Agent.TempDirectory)/InnerEye/
|
|
cp -r $(Build.SourcesDirectory)/TestSubmodule $(Agent.TempDirectory)/InnerEye/TestSubmodule
|
|
cp -r $(Build.SourcesDirectory) $(Agent.TempDirectory)/InnerEye/Submodule
|
|
cp -r $(Build.SourcesDirectory)/TestSubmodule/environment.yml $(Agent.TempDirectory)/InnerEye
|
|
cp -r $(Build.SourcesDirectory)/InnerEyePrivateSettings.yml $(Agent.TempDirectory)/InnerEye
|
|
branch_prefix="refs/heads/"
|
|
full_branch_name=$(Build.SourceBranch)
|
|
branch_name_without_prefix=${full_branch_name#$branch_prefix}
|
|
python $(Agent.TempDirectory)/InnerEye/TestSubmodule/test_submodule_runner.py --azureml=True --model="$(model)" --train="$(train)" $(more_switches) --wait_for_completion="${{parameters.wait_for_completion}}" --max_run_duration="${{parameters.max_run_duration}}" --cluster="$(cluster)" --tag="$(tag)" --build_number=$(Build.BuildId) --build_user="$(Build.RequestedFor)" --build_user_email="" --build_branch="$branch_name_without_prefix" --build_source_repository="$(Build.Repository.Name)" --monitoring_interval_seconds=5 --show_patch_sampling=0
|
|
mv most_recent_run.txt training_run.txt
|
|
python $(Agent.TempDirectory)/InnerEye/TestSubmodule/test_submodule_runner.py --run_recovery_id=`cat training_run.txt` --num_epochs=4 --azureml=True --model="$(model)" --train="$(train)" $(more_switches) --wait_for_completion="${{parameters.wait_for_completion}}" --max_run_duration="${{parameters.max_run_duration}}" --cluster="$(cluster)" --tag="$(tag)" --build_number=$(Build.BuildId) --build_user="$(Build.RequestedFor)" --build_user_email="" --build_branch="$branch_name_without_prefix" --build_source_id="$(Build.SourceVersion)" --build_source_message="$source_version_message" --build_source_author="$(Build.SourceVersionAuthor)" --build_source_repository="$(Build.Repository.Name)" --show_patch_sampling=0
|
|
mv training_run.txt most_recent_run.txt
|
|
env:
|
|
PYTHONPATH: $(Agent.TempDirectory)/InnerEye
|
|
APPLICATION_KEY: $(InnerEyeDeepLearningServicePrincipalKey)
|
|
displayName: 'Training and Recovery in AzureML'
|