Release AML-unit-test-linux-nb-gpu test, chg test kernel_name from cv to python3 and minor updates (#270)

* chg kernel_name = cv to = python3 for nb tests * nb test update for not azuremlnotebooks * add not azuremlnotebooks * add tags for reponame and branch in python cmd * stage badges from AzureML tests
2019-07-24 12:02:48 -04:00 · 2019-07-24 12:02:48 -04:00 · e936d036af
--- a/.ci/AML-unit-test-linux-cpu.yml
+++ b/.ci/AML-unit-test-linux-cpu.yml
@ -8,11 +8,15 @@ variables:
 - name  : 'testfolder'
  value : 'tests/unit'
 - name  : 'testmarkers'
-  value : '"not notebooks and not spark and not gpu"'
+  value : '"not notebooks and not spark and not gpu and not azuremlnotebooks"'
 - name  : 'junitxml'
  value : 'reports/test-unit.xml'
 - name  : 'maxnodes'
  value : 4
+- name  : 'reponame'
+  value : 'computervision'
+- name  : 'branch'
+  value : 'TestJob'
 - name  : 'clustername'
  value : 'cv-cpu'
 - name  : 'vmsize'
@ -27,6 +31,9 @@ variables:
  value : 'cvbp_project_resources'
 - name  : 'wsname'
  value : 'cvws'
+- name  : 'prtag'
+  value : 'UnitTestRun'
+
 trigger: none

 pr:
@ -50,17 +57,15 @@ steps:
  displayName: 'Login to Azure'

 - script:
-    # bug fix for azureml sdk 7/18/2019
    pip install paramiko==2.4.2;
    pip install azureml-sdk;
    pip install azure-cli;
-    python .ci/submit_azureml_pytest.py --subid $(subscriptionid) --test $(test) --testfolder $(testfolder) --condafile $(condafile) --testmarkers $(testmarkers) --clustername $(clustername) --expname $(expname) --dockerproc $(dockerproc) --junitxml $(junitxml) --reponame $(Build.Repository.Name) --branch $(Build.SourceBranchName)  --rg $(rg) --wsname $(wsname)
+    python .ci/submit_azureml_pytest.py --subid $(subscriptionid) --test $(test) --testfolder $(testfolder) --condafile $(condafile) --testmarkers $(testmarkers) --clustername $(clustername) --expname $(expname) --dockerproc $(dockerproc) --junitxml $(junitxml) --reponame $(Build.Repository.Name) --branch $(Build.SourceBranchName) --pr $(prtag) --rg $(rg) --wsname $(wsname)
  displayName: 'submit_azureml_pytest'

 - task: PublishTestResults@2
  displayName: 'Publish Test Results **/test-*.xml'
-  condition: always()
  inputs:
    testResultsFiles: '**/test-*.xml'
    failTaskOnFailedTests: true
-    testRunTitle: 'Test results for CV AzureML PyTest'
+  condition: succeededOrFailed()
--- a/.ci/AML-unit-test-linux-nb-gpu.yml
+++ b/.ci/AML-unit-test-linux-nb-gpu.yml
@ -0,0 +1,70 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+#
+variables:
+- group: AzureMLConfig
+- name  : 'test'
+  value : '.ci/run_pytest.py'
+- name  : 'testfolder'
+  value : 'tests/unit'
+- name  : 'testmarkers'
+  # not gpu is to match how the existing tests run
+  value : '"notebooks and not spark and not gpu and not azuremlnotebooks"'
+- name  : 'junitxml'
+  value : 'reports/test-unit.xml'
+- name  : 'maxnodes'
+  value : 4
+- name  : 'clustername'
+  value : 'cv-gpu'
+- name  : 'vmsize'
+  value : 'STANDARD_NC6'
+- name  : 'dockerproc'
+  value : 'gpu'
+- name  : 'expname'
+  value : 'unit-test-linux-gpu-nb'
+- name  : 'condafile'
+  value : 'environment.yml'
+- name  : 'rg'
+  value : 'cvbp_project_resources'
+- name  : 'wsname'
+  value : 'cvws'
+- name  : 'prtag'
+  value : 'NBTestRun'
+
+trigger: none
+
+pr:
+- staging 
+- master
+  
+pool:
+ vmImage: 'ubuntu-16.04'
+
+steps:
+
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '3.6'
+    architecture: 'x64'
+    addToPath: true
+  displayName: 'Use Python 3.6'
+
+- script: | 
+    az login --service-principal -u $(spidentity) -p $(spsecret) --tenant $(sptenant)
+  displayName: 'Login to Azure'
+
+- script:
+    # bug fix for azureml sdk 7/18/2019
+    pip install paramiko==2.4.2;
+    pip install azureml-sdk;
+    pip install azure-cli;
+    python .ci/submit_azureml_pytest.py --subid $(subscriptionid) --test $(test) --testfolder $(testfolder) --condafile $(condafile) --testmarkers $(testmarkers) --clustername $(clustername) --expname $(expname) --dockerproc $(dockerproc) --junitxml $(junitxml) --reponame $(Build.Repository.Name) --branch $(Build.SourceBranchName)  --pr $(prtag) --rg $(rg) --wsname $(wsname)
+  displayName: 'submit_azureml_pytest'
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results **/test-*.xml'
+  condition: always()
+  inputs:
+    testResultsFiles: '**/test-*.xml'
+    failTaskOnFailedTests: true
+    testRunTitle: 'Test results for CV AzureML PyTest'
--- a/.ci/templates/unit-test-steps.yml
+++ b/.ci/templates/unit-test-steps.yml
@ -13,7 +13,7 @@ steps:

 - bash: |
    source activate cv
-    python -m ipykernel install --user --name cv --display-name "cv"
+    # python -m ipykernel install --user --name cv --display-name "cv"
    pytest --durations 100 tests --junitxml=junit/test-unitttest.xml -m "not azuremlnotebooks"
  displayName: 'Run unit and (only on Linux GPU) integration tests'

--- a/README.md
+++ b/README.md
@ -18,7 +18,10 @@ This repository provides implementations and best practice guidelines for buildi

 | Build Type | Branch | Status |  | Branch | Status | 
 | --- | --- | --- | --- | --- | --- | 
+| **Linxu GPU** | master | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/bp-azureml-unit-test-linux-gpu?branchName=master)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=41&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/bp-azureml-unit-test-linux-gpu?branchName=staging)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=41&branchName=staging)|
 | **Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/aml-unit-test-linux-cpu?branchName=master)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=37&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/aml-unit-test-linux-cpu?branchName=staging)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=37&branchName=staging)|
+| **Notebook unit GPU** | master | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/azureml-unit-test-linux-nb-gpu?branchName=master)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=42&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/azureml-unit-test-linux-nb-gpu?branchName=staging)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=42&branchName=staging) |
+| **Nightly GPU** | master | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/nightly-linux-gpu?branchName=master)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=46&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/azureml/nightly-linux-gpu?branchName=staging)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=46&branchName=staging) |


 ## Overview
--- a/tests/integration/classification/test_integration_classification_notebooks.py
+++ b/tests/integration/classification/test_integration_classification_notebooks.py
@ -11,7 +11,7 @@ import shutil
 from utils_cv.common.gpu import linux_with_gpu

 # Parameters
-KERNEL_NAME = "cv"
+KERNEL_NAME = "python3"
 OUTPUT_NOTEBOOK = "output.ipynb"


@ -35,7 +35,9 @@ def test_01_notebook_run(classification_notebooks):
@pytest.mark.notebooks
 def test_02_notebook_run(classification_notebooks):
    if linux_with_gpu():
-        notebook_path = classification_notebooks["02_multilabel_classification"]
+        notebook_path = classification_notebooks[
+            "02_multilabel_classification"
+        ]
        pm.execute_notebook(
            notebook_path,
            OUTPUT_NOTEBOOK,
@ -53,7 +55,9 @@ def test_02_notebook_run(classification_notebooks):
@pytest.mark.notebooks
 def test_03_notebook_run(classification_notebooks):
    if linux_with_gpu():
-        notebook_path = classification_notebooks["03_training_accuracy_vs_speed"]
+        notebook_path = classification_notebooks[
+            "03_training_accuracy_vs_speed"
+        ]
        pm.execute_notebook(
            notebook_path,
            OUTPUT_NOTEBOOK,
@ -70,17 +74,18 @@ def test_03_notebook_run(classification_notebooks):
@pytest.mark.notebooks
 def test_11_notebook_run(classification_notebooks, tiny_ic_data_path):
    if linux_with_gpu():
-        notebook_path = classification_notebooks["11_exploring_hyperparameters"]
+        notebook_path = classification_notebooks[
+            "11_exploring_hyperparameters"
+        ]
        pm.execute_notebook(
            notebook_path,
            OUTPUT_NOTEBOOK,
            parameters=dict(
                PM_VERSION=pm.__version__,
-
                # Speed up testing since otherwise would take ~12 minutes on V100
                DATA=[tiny_ic_data_path],
                REPS=1,
-                IM_SIZES=[60,100],
+                IM_SIZES=[60, 100],
            ),
            kernel_name=KERNEL_NAME,
        )
@ -88,7 +93,10 @@ def test_11_notebook_run(classification_notebooks, tiny_ic_data_path):
        nb_output = sb.read_notebook(OUTPUT_NOTEBOOK)
        assert nb_output.scraps["nr_elements"].data == 6
        assert nb_output.scraps["max_accuray"].data > 0.5
-        assert nb_output.scraps["max_duration"].data > 1.2 * nb_output.scraps["min_duration"].data
+        assert (
+            nb_output.scraps["max_duration"].data
+            > 1.2 * nb_output.scraps["min_duration"].data
+        )


@pytest.mark.notebooks
--- a/tests/integration/similarity/test_integration_similarity_notebooks.py
+++ b/tests/integration/similarity/test_integration_similarity_notebooks.py
@ -9,7 +9,7 @@ from torch.cuda import is_available
 from utils_cv.common.gpu import linux_with_gpu

 # Parameters
-KERNEL_NAME = "cv"
+KERNEL_NAME = "python3"
 OUTPUT_NOTEBOOK = "output.ipynb"


--- a/tests/unit/classification/test_classification_notebooks.py
+++ b/tests/unit/classification/test_classification_notebooks.py
@ -14,7 +14,7 @@ import shutil
 # Unless manually modified, python3 should be
 # the name of the current jupyter kernel
 # that runs on the activated conda environment
-KERNEL_NAME = "cv"
+KERNEL_NAME = "python3"
 OUTPUT_NOTEBOOK = "output.ipynb"


--- a/tests/unit/similarity/test_similarity_notebooks.py
+++ b/tests/unit/similarity/test_similarity_notebooks.py
@ -12,7 +12,7 @@ import scrapbook as sb
 # Unless manually modified, python3 should be
 # the name of the current jupyter kernel
 # that runs on the activated conda environment
-KERNEL_NAME = "cv"
+KERNEL_NAME = "python3"
 OUTPUT_NOTEBOOK = "output.ipynb"


@ -20,15 +20,16 @@ OUTPUT_NOTEBOOK = "output.ipynb"
 def test_00_notebook_run(similarity_notebooks):
    notebook_path = similarity_notebooks["00"]
    pm.execute_notebook(
-        notebook_path,
-        OUTPUT_NOTEBOOK,
-        kernel_name=KERNEL_NAME,
+        notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME
    )

    nb_output = sb.read_notebook(OUTPUT_NOTEBOOK)
    assert len(nb_output.scraps["query_feature"].data) == 512
    assert min(nb_output.scraps["query_feature"].data) >= 0
-    assert min([dist for (path,dist) in nb_output.scraps["distances"].data]) < 1e-3
+    assert (
+        min([dist for (path, dist) in nb_output.scraps["distances"].data])
+        < 1e-3
+    )


@pytest.mark.notebooks