Changed azureml dependency. Changed test to always restore conda environment

tests/ci/cpu_unit_tests_linux.yml

@@ -3,60 +3,54 @@
 
 # Pull request against these branches will trigger this build
 pr:
-- master
-- staging
+  - master
+  - staging
 
 #Any commit to this branch will trigger the build.
 trigger:
-- staging
-- master
-
+  - staging
+  - master
 
 jobs:
-- job: cpu_unit_tests_linux
-  timeoutInMinutes: 12 # how long to run the job before automatically cancelling
-  pool:
-    # vmImage: 'ubuntu-16.04' # hosted machine 
-    name: nlpagentpool
+  - job: cpu_unit_tests_linux
+    timeoutInMinutes: 12 # how long to run the job before automatically cancelling
+    pool:
+      # vmImage: 'ubuntu-16.04' # hosted machine
+      name: nlpagentpool
 
-  steps:
+    steps:
+      # Uncomment if hosted machine
+      #  - task: UsePythonVersion@0
+      #    inputs:
+      #      versionSpec: '3.6.8'
+      #      architecture: 'x64'
+      #      addToPath: true
+      #    displayName: 'Use Python 3.6.8'
 
-# Uncomment if hosted machine
-#  - task: UsePythonVersion@0
-#    inputs:
-#      versionSpec: '3.6.8'
-#      architecture: 'x64'
-#      addToPath: true
-#    displayName: 'Use Python 3.6.8'
-  
-  - bash: |
-      echo "##vso[task.prependpath]/data/anaconda/bin"
-      conda env list
-    displayName: Add Conda to PATH
+      - bash: |
+          echo "##vso[task.prependpath]/data/anaconda/bin"
+          conda env list
+        displayName: Add Conda to PATH
 
-# Uncomment if needed
-# Conda creation can take around 10min
-#  - bash: |
-#      python tools/generate_conda_file.py
-#      conda env create -n nlp_cpu -f nlp_cpu.yaml
-#    displayName: 'Creating Conda Environment with dependencies'
+      # Conda creation can take around 10min
+      - bash: |
+          python tools/generate_conda_file.py
+          conda env create -n nlp_cpu -f nlp_cpu.yaml
+        displayName: "Creating Conda Environment with dependencies"
 
-  - bash: |
-      source activate nlp_cpu
-      pytest --durations=0 tests/unit -m "not notebooks and not gpu and not azureml" --junitxml=junit/test-unitttest.xml
-    displayName: 'Run Unit tests'
-
-# Uncomment if needed
-#  - bash: |
-#      echo Remove Conda Environment
-#      conda remove -n nlp_cpu --all -q --force -y
-#      echo Done Cleanup
-#    displayName: 'Cleanup Task'
-#    condition: always()
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '**/test-unitttest.xml'
-      testRunTitle: 'Test results for PyTest'
+      - bash: |
+          source activate nlp_cpu
+          pytest --durations=0 tests/unit -m "not notebooks and not gpu and not azureml" --junitxml=junit/test-unitttest.xml
+        displayName: "Run Unit tests"
 
+      - bash: |
+          echo Remove Conda Environment
+          conda remove -n nlp_cpu --all -q --force -y
+          echo Done Cleanup
+        displayName: "Cleanup Task"
+        condition: always()
 
+      - task: PublishTestResults@2
+        inputs:
+          testResultsFiles: "**/test-unitttest.xml"
+          testRunTitle: "Test results for PyTest"

tests/ci/gpu_unit_tests_linux.yml

@@ -3,49 +3,45 @@
 
 # Pull request against these branches will trigger this build
 pr:
-- master
-- staging
+  - master
+  - staging
 
 #Any commit to this branch will trigger the build.
 trigger:
-- staging
-- master
+  - staging
+  - master
 
 jobs:
-- job: gpu_unit_tests_linux
-  timeoutInMinutes: 30 # how long to run the job before automatically cancelling
-  pool:
-    name: nlpagentpool
+  - job: gpu_unit_tests_linux
+    timeoutInMinutes: 30 # how long to run the job before automatically cancelling
+    pool:
+      name: nlpagentpool
 
-  steps:
-  - bash: |
-      echo "##vso[task.prependpath]/data/anaconda/bin"
-      conda env list
-    displayName: Add Conda to PATH
+    steps:
+      - bash: |
+          echo "##vso[task.prependpath]/data/anaconda/bin"
+          conda env list
+        displayName: Add Conda to PATH
 
-# Uncomment if needed
-# Conda creation can take around 10min
-#  - bash: |
-#      python tools/generate_conda_file.py --gpu
-#      conda env create -n nlp_gpu -f nlp_gpu.yaml
-#    displayName: 'Creating Conda Environment with dependencies'
+      # Conda creation can take around 10min
+      - bash: |
+          python tools/generate_conda_file.py --gpu
+          conda env create -n nlp_gpu -f nlp_gpu.yaml
+        displayName: "Creating Conda Environment with dependencies"
 
-  - bash: |
-      source activate nlp_gpu
-      pytest --durations=0 tests/unit -m "not notebooks and gpu and not azureml" --junitxml=junit/test-unitttest.xml
-    displayName: 'Run Unit tests'
-
-# Uncomment if needed
-#  - bash: |
-#      echo Remove Conda Environment
-#      conda remove -n nlp_gpu --all -q --force -y
-#      echo Done Cleanup
-#    displayName: 'Cleanup Task'
-#    condition: always()
-
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '**/test-unitttest.xml'
-      testRunTitle: 'Test results for PyTest'
+      - bash: |
+          source activate nlp_gpu
+          pytest --durations=0 tests/unit -m "not notebooks and gpu and not azureml" --junitxml=junit/test-unitttest.xml
+        displayName: "Run Unit tests"
 
+      - bash: |
+          echo Remove Conda Environment
+          conda remove -n nlp_gpu --all -q --force -y
+          echo Done Cleanup
+        displayName: "Cleanup Task"
+        condition: always()
 
+      - task: PublishTestResults@2
+        inputs:
+          testResultsFiles: "**/test-unitttest.xml"
+          testRunTitle: "Test results for PyTest"

tests/ci/notebooks_cpu_unit_tests_linux.yml

@@ -23,25 +23,23 @@ jobs:
       conda env list
     displayName: Add Conda to PATH
 
-# Uncomment if needed
 # Conda creation can take around 10min
-#  - bash: |
-#      python tools/generate_conda_file.py
-#      conda env create -n nlp_cpu -f nlp_cpu.yaml
-#    displayName: 'Creating Conda Environment with dependencies'
+  - bash: |
+      python tools/generate_conda_file.py
+      conda env create -n nlp_cpu -f nlp_cpu.yaml
+    displayName: 'Creating Conda Environment with dependencies'
 
   - bash: |
       source activate nlp_cpu
       pytest --durations=0 tests/unit -m "notebooks and not gpu and not azureml" --junitxml=junit/test-unitttest.xml
     displayName: 'Run Unit tests'
 
-# Uncomment if needed
-#  - bash: |
-#      echo Remove Conda Environment
-#      conda remove -n nlp_cpu --all -q --force -y
-#      echo Done Cleanup
-#    displayName: 'Cleanup Task'
-#    condition: always()
+  - bash: |
+      echo Remove Conda Environment
+      conda remove -n nlp_cpu --all -q --force -y
+      echo Done Cleanup
+    displayName: 'Cleanup Task'
+    condition: always()
 
   - task: PublishTestResults@2
     inputs:

tests/ci/notebooks_gpu_unit_tests_linux.yml

@@ -23,25 +23,22 @@ jobs:
       conda env list
     displayName: Add Conda to PATH
 
-# Uncomment if needed
-# Conda creation can take around 10min
-#  - bash: |
-#      python tools/generate_conda_file.py --gpu
-#      conda env create -n nlp_gpu -f nlp_gpu.yaml
-#    displayName: 'Creating Conda Environment with dependencies'
+  - bash: |
+      python tools/generate_conda_file.py --gpu
+      conda env create -n nlp_gpu -f nlp_gpu.yaml
+    displayName: 'Creating Conda Environment with dependencies'
 
   - bash: |
       source activate nlp_gpu
       pytest --durations=0 tests/unit -m "notebooks and gpu and not azureml" --junitxml=junit/test-unitttest.xml
     displayName: 'Run Unit tests'
 
-# Uncomment if needed
-#  - bash: |
-#      echo Remove Conda Environment
-#      conda remove -n nlp_gpu --all -q --force -y
-#      echo Done Cleanup
-#    displayName: 'Cleanup Task'
-#    condition: always()
+  - bash: |
+      echo Remove Conda Environment
+      conda remove -n nlp_gpu --all -q --force -y
+      echo Done Cleanup
+    displayName: 'Cleanup Task'
+    condition: always()
 
   - task: PublishTestResults@2
     inputs:

tools/generate_conda_file.py

@@ -58,11 +58,10 @@ CONDA_GPU = {
 
 PIP_BASE = {
     "allennlp": "allennlp==0.8.4",
-    "azureml-sdk": "azureml-sdk[automl,notebooks,contrib]==1.0.85",
-    "azureml-train-automl": "azureml-train-automl==1.0.85",
-    "azureml-dataprep": "azureml-dataprep==1.1.8",
-    "azureml-widgets": "azureml-widgets==1.0.85",
-    "azureml-mlflow": "azureml-mlflow==1.0.85",
+    "azureml-sdk": "azureml-sdk[automl,notebooks,contrib]==1.20.0",
+    "azureml-train-automl": "azureml-train-automl==1.20.0",
+    "azureml-widgets": "azureml-widgets==1.20.0",
+    "azureml-mlflow": "azureml-mlflow==1.20.0",
     "black": "black>=18.6b4",
     "cached-property": "cached-property==1.5.1",
     "jsonlines": "jsonlines>=1.2.0",