From cc31cf5866db629dbeb84d043c57e3fa2f63437b Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Tue, 26 Jan 2021 16:09:18 -0500
Subject: [PATCH 01/20] Add postargs to tox

---
 tox.ini | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 2961cdf..f320f45 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,7 +14,13 @@ passenv =
 # https://tox.readthedocs.io/en/latest/example/basic.html#depending-on-requirements-txt-or-defining-constraints
 deps = -rrequirements-dev.txt
 commands = 
-    pytest
+    # {posargs} will be substituded by arguments after the `--` when running.
+    # This will allow running subset of the test suite via tox. 
+    #
+    # EX: tox -- -m "not azure and not slow" 
+    #     will pass {-m "not azure and not slow"} to `pytest`
+    # See https://tox.readthedocs.io/en/latest/example/general.html for more details
+    pytest {posargs}
 
 
 [testenv:flake8]
@@ -32,6 +38,7 @@ junit_family = xunit2
 markers = 
     # These two markers allow to us to run faster subset of the test:
     # EX: pytest -m "not slow and not azure"
+    # See https://docs.pytest.org/en/stable/example/markers.html#registering-markers
     slow: marks tests as slow (deselect with '-m "not slow"')
     azure: marks as integration tests that require azure resource
     io: marks integration tests involving some form of I/O operations (disk, internet, etc)

From 7ae1d6c24432e05f2e86be6ca81df23cac2996fc Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Tue, 26 Jan 2021 17:41:29 -0500
Subject: [PATCH 02/20] Add CI templates

---
 devops/pr-gate-os.yml           | 58 ++++++++++++++++++++++++++++++
 devops/templates/run-linter.yml | 24 +++++++++++++
 devops/templates/run-tests.yml  | 63 +++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+)
 create mode 100644 devops/pr-gate-os.yml
 create mode 100644 devops/templates/run-linter.yml
 create mode 100644 devops/templates/run-tests.yml

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
new file mode 100644
index 0000000..a1cbc0f
--- /dev/null
+++ b/devops/pr-gate-os.yml
@@ -0,0 +1,58 @@
+name: $(Date:yyyyMMdd).$(Rev:r)
+
+strategy:
+  matrix:
+    linux_x64_py3.6:
+      imageName: 'ubuntu-18.04'
+      pyVersion: '3.6'
+
+    # windows_x64_py3.6:
+    #   imageName: 'windows-2019'
+    #   pyVersion: '3.6'
+
+    # macos_x64_py3.6:
+    #   imageName: 'macOS-10.14'
+    #   pyVersion: '3.6'
+
+    # To build using more python versions follow the syntax below:
+    # linux_x64_py3.5:
+    #   imageName: 'ubuntu-16.04'
+    #   pyVersion: '3.5'
+
+pool:
+  vmImage: '$(imageName)'
+  
+steps:
+- template: templates/run-linter.yml
+  parameters:
+    pyVersion: $(python.version)
+
+- template: templates/run-tests.yml
+  parameters:
+    testType: 'unit'
+    imageName: $(imageName)
+    pyVersion: $(python.version)
+
+# - template: templates/run-tests.yml
+#   parameters:
+#     testType: 'azure'
+#     imageName: $(imageName)
+#     pyVersion: $(python.version)
+  
+# - bash: |
+#     python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
+#   workingDirectory: $(Build.SourcesDirectory)
+#   displayName: 'Building wheel package'
+
+# - bash: |
+#     mkdir $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
+#     cp -r dist/* $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
+#   workingDirectory: $(Build.SourcesDirectory)
+#   displayName: 'Copying builds to artifact staging dir'
+
+# - task: PublishBuildArtifacts@1
+#   inputs:
+#     PathtoPublish: $(Build.ArtifactStagingDirectory)
+#     ArtifactName: genalog
+#     publishLocation: 'Container'
+#   displayName: 'Publish build as artifacts'
diff --git a/devops/templates/run-linter.yml b/devops/templates/run-linter.yml
new file mode 100644
index 0000000..85731c4
--- /dev/null
+++ b/devops/templates/run-linter.yml
@@ -0,0 +1,24 @@
+# Template for running linter and other static analysis tools on the code
+parameters:
+- name: pyVersion
+  type: string
+  default: 3.6
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(pyVersion)'
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python $(pyVersion)'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install flake8 and other dev dependencies'
+
+- bash: |
+    tox -e flake8
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Run Linter (flake8)'
\ No newline at end of file
diff --git a/devops/templates/run-tests.yml b/devops/templates/run-tests.yml
new file mode 100644
index 0000000..3a5a8b6
--- /dev/null
+++ b/devops/templates/run-tests.yml
@@ -0,0 +1,63 @@
+# Template for running tests on multiple Python versions and platforms
+
+parameters:
+- name: imageName
+  type: string
+- name: testType
+  type: string
+  default: all
+  values:
+    - slow
+    - azure
+    - unit
+    - all
+- name: pyVersion
+  type: string
+  default: 3.6
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(pyVersion)'
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python $(pyVersion)'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements.txt 
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install dependencies'
+
+- bash: |
+    if [[ '${{parameters.testType}}' == 'all' ]]
+    then
+      tox -e py 
+    elif [[ '${{parameters.testType}}' == 'unit' ]]
+    then
+      tox -e py -- -m "not slow and not azure"
+    else
+      tox -e py -- -m "${{parameters.testType}}"
+    fi
+  env:
+    BLOB_KEY : $(BLOB_KEY)
+    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
+    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
+    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY)
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Test via tox'
+
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: 'JUnit' 
+    testResultsFiles: 'junit/*.xml' 
+    searchFolder: $(Build.SourcesDirectory)
+    testRunTitle: $(imageName)_$(pyVersion) Build
+    buildPlatform: $(imageName)
+  displayName: 'Publish unit test report'
+
+- task: PublishCodeCoverageResults@1
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
\ No newline at end of file

From 36875705b4b91b85a879a3ab71e7b0389b52e4dd Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Tue, 26 Jan 2021 18:02:50 -0500
Subject: [PATCH 03/20] Add slow tests

---
 devops/pr-gate-os.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index a1cbc0f..f448665 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -33,6 +33,12 @@ steps:
     imageName: $(imageName)
     pyVersion: $(python.version)
 
+- template: templates/run-tests.yml
+  parameters:
+    testType: 'slow'
+    imageName: $(imageName)
+    pyVersion: $(python.version)
+
 # - template: templates/run-tests.yml
 #   parameters:
 #     testType: 'azure'

From b67c6b34ab6136f0931d53fbaa50ccfc0487d49b Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Tue, 26 Jan 2021 18:23:01 -0500
Subject: [PATCH 04/20] Update templates

---
 devops/pr-gate-os.yml                         | 69 ++++++++++++-------
 devops/templates/install-dependencies.yml     |  8 +++
 devops/templates/publish-cov.yml              | 14 ++++
 devops/templates/run-linter.yml               |  6 +-
 .../run-tests-on-multiple-py-versions.yml     | 28 ++++++++
 devops/templates/run-tests.yml                | 39 +----------
 tox.ini                                       |  2 +-
 7 files changed, 102 insertions(+), 64 deletions(-)
 create mode 100644 devops/templates/install-dependencies.yml
 create mode 100644 devops/templates/publish-cov.yml
 create mode 100644 devops/templates/run-tests-on-multiple-py-versions.yml

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index f448665..9438f97 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -1,10 +1,34 @@
 name: $(Date:yyyyMMdd).$(Rev:r)
 
-strategy:
-  matrix:
-    linux_x64_py3.6:
-      imageName: 'ubuntu-18.04'
-      pyVersion: '3.6'
+stages:
+  - stage: static_analysis
+    jobs:
+    - job: flake8_linux_py36
+      pool:
+        vmImage: 'ubuntu-latest'
+      steps:
+        - template: templates/run-linter.yml
+          parameters:
+            pyVersion: '3.6'
+
+  - stage: run_tests
+    dependsOn: static_analysis
+    pool:
+      vmImage: 'ubuntu-18.04'
+    jobs:
+      - template: templates/run-tests-on-multiple-py-versions.yml
+        parameters:
+          pyVersions: ['3.6', '3.7']
+          testTypes: ['fast', 'slow']
+
+    
+    
+
+# strategy:
+#   matrix:
+#     linux_x64_py3.6:
+#       imageName: 'ubuntu-18.04'
+#       pyVersion: '3.6'
 
     # windows_x64_py3.6:
     #   imageName: 'windows-2019'
@@ -19,31 +43,28 @@ strategy:
     #   imageName: 'ubuntu-16.04'
     #   pyVersion: '3.5'
 
-pool:
-  vmImage: '$(imageName)'
-  
-steps:
-- template: templates/run-linter.yml
-  parameters:
-    pyVersion: $(python.version)
+# pool:
+#   vmImage: '$(imageName)'
 
-- template: templates/run-tests.yml
-  parameters:
-    testType: 'unit'
-    imageName: $(imageName)
-    pyVersion: $(python.version)
-
-- template: templates/run-tests.yml
-  parameters:
-    testType: 'slow'
-    imageName: $(imageName)
-    pyVersion: $(python.version)
+# steps:
+# - template: templates/run-linter.yml
+#   parameters:
+#     pyVersion: $(python.version)
 
 # - template: templates/run-tests.yml
 #   parameters:
-#     testType: 'azure'
+#     testType: 'fast'
 #     imageName: $(imageName)
 #     pyVersion: $(python.version)
+
+# - template: templates/run-tests.yml
+#   parameters:
+#     testType: 'slow'
+#     skipInstall: true
+#     imageName: $(imageName)
+#     pyVersion: $(python.version)
+
+# - template: templates/publish-cov.yml
   
 # - bash: |
 #     python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
diff --git a/devops/templates/install-dependencies.yml b/devops/templates/install-dependencies.yml
new file mode 100644
index 0000000..a543be2
--- /dev/null
+++ b/devops/templates/install-dependencies.yml
@@ -0,0 +1,8 @@
+# Assume a python version is enabled with "UsePythonVersion@0" task
+steps:
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements.txt 
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install dependencies'
diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
new file mode 100644
index 0000000..23b132c
--- /dev/null
+++ b/devops/templates/publish-cov.yml
@@ -0,0 +1,14 @@
+steps:
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: 'JUnit' 
+    testResultsFiles: 'junit/*.xml' 
+    searchFolder: $(Build.SourcesDirectory)
+    testRunTitle: $(imageName)_$(pyVersion) Build
+    buildPlatform: $(imageName)
+  displayName: 'Publish unit test report'
+
+- task: PublishCodeCoverageResults@1
+  inputs:
+    codeCoverageTool: Cobertura
+    summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
\ No newline at end of file
diff --git a/devops/templates/run-linter.yml b/devops/templates/run-linter.yml
index 85731c4..79fd571 100644
--- a/devops/templates/run-linter.yml
+++ b/devops/templates/run-linter.yml
@@ -2,15 +2,15 @@
 parameters:
 - name: pyVersion
   type: string
-  default: 3.6
+  default: '3.6'
 
 steps:
 - task: UsePythonVersion@0
   inputs:
-    versionSpec: '$(pyVersion)'
+    versionSpec: ${{ parameters.pyVersion }}
     addToPath: true
     architecture: 'x64'
-  displayName: 'Use Python $(pyVersion)'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
 
 - bash: |
     python -m pip install --upgrade pip setuptools wheel
diff --git a/devops/templates/run-tests-on-multiple-py-versions.yml b/devops/templates/run-tests-on-multiple-py-versions.yml
new file mode 100644
index 0000000..7a49545
--- /dev/null
+++ b/devops/templates/run-tests-on-multiple-py-versions.yml
@@ -0,0 +1,28 @@
+parameters:
+- name: pyVersions
+  type: object
+  default: ['3.6', '3.7', '3.8']
+- name: testTypes
+  type: object
+  default: ['fast', 'slow']
+
+jobs:
+- ${{ each pyVersion in parameters.pyVersions }}:
+  - job: run_tests_py${{ replace(pyVersion, '.', '_') }}
+    steps:
+    
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{pyVersion}}
+        addToPath: true
+        architecture: 'x64'
+      displayName: 'Use Python ${{pyVersion}}'
+
+    - template: install-dependencies.yml
+
+    - ${{ each testType in parameters.testTypes }}:
+      - template: run-tests.yml
+        parameters:
+          testType: ${{testType}}
+
+    - template: publish-cov.yml
\ No newline at end of file
diff --git a/devops/templates/run-tests.yml b/devops/templates/run-tests.yml
index 3a5a8b6..dab1a8d 100644
--- a/devops/templates/run-tests.yml
+++ b/devops/templates/run-tests.yml
@@ -1,40 +1,21 @@
 # Template for running tests on multiple Python versions and platforms
 
 parameters:
-- name: imageName
-  type: string
 - name: testType
   type: string
   default: all
   values:
     - slow
     - azure
-    - unit
+    - fast
     - all
-- name: pyVersion
-  type: string
-  default: 3.6
 
 steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(pyVersion)'
-    addToPath: true
-    architecture: 'x64'
-  displayName: 'Use Python $(pyVersion)'
-
-- bash: |
-    python -m pip install --upgrade pip setuptools wheel
-    python -m pip install -r requirements.txt 
-    python -m pip install -r requirements-dev.txt
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Install dependencies'
-
 - bash: |
     if [[ '${{parameters.testType}}' == 'all' ]]
     then
       tox -e py 
-    elif [[ '${{parameters.testType}}' == 'unit' ]]
+    elif [[ '${{parameters.testType}}' == 'fast' ]]
     then
       tox -e py -- -m "not slow and not azure"
     else
@@ -46,18 +27,4 @@ steps:
     COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
     COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY)
   workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Test via tox'
-
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'JUnit' 
-    testResultsFiles: 'junit/*.xml' 
-    searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName)_$(pyVersion) Build
-    buildPlatform: $(imageName)
-  displayName: 'Publish unit test report'
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
\ No newline at end of file
+  displayName: 'Running (${{parameters.testType}}) Tests'
diff --git a/tox.ini b/tox.ini
index f320f45..ed6da4f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -45,7 +45,7 @@ markers =
 testpaths =
     tests
 addopts = 
-    -rsx --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
+    -rsx --cov-append --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
 
 
 [flake8]

From 4ff93d3cb50fa18a94455f3e9c5b20c517754d7a Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Tue, 26 Jan 2021 21:36:34 -0500
Subject: [PATCH 05/20] Run tests in 3.6-8

---
 devops/pr-gate-os.yml            | 4 ++--
 devops/templates/publish-cov.yml | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 9438f97..fd25129 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -18,8 +18,8 @@ stages:
     jobs:
       - template: templates/run-tests-on-multiple-py-versions.yml
         parameters:
-          pyVersions: ['3.6', '3.7']
-          testTypes: ['fast', 'slow']
+          pyVersions: ['3.6', '3.7', '3.8']
+          testTypes: ['fast']
 
     
     
diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
index 23b132c..6f61c23 100644
--- a/devops/templates/publish-cov.yml
+++ b/devops/templates/publish-cov.yml
@@ -6,6 +6,7 @@ steps:
     searchFolder: $(Build.SourcesDirectory)
     testRunTitle: $(imageName)_$(pyVersion) Build
     buildPlatform: $(imageName)
+  condition: always() # Always publish test results
   displayName: 'Publish unit test report'
 
 - task: PublishCodeCoverageResults@1

From 05f9649e3daffe4d68efd76b970fc56277940f7b Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 09:07:56 -0500
Subject: [PATCH 06/20] Add detail test summary report

---
 tox.ini | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index ed6da4f..77c3b17 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,8 +44,10 @@ markers =
     io: marks integration tests involving some form of I/O operations (disk, internet, etc)
 testpaths =
     tests
-addopts = 
-    -rsx --cov-append --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
+addopts =
+    # reports all (except passed tests). See https://docs.pytest.org/en/latest/usage.html#detailed-summary-report
+    -ra 
+    --cov-append --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
 
 
 [flake8]

From 190d823afcaaacaa137f078466914bb0eebbf4aa Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 11:31:53 -0500
Subject: [PATCH 07/20] Mark more io tests

---
 devops/pr-gate-os.yml                         | 19 ++++++---
 devops/templates/publish-cov.yml              |  8 +++-
 .../templates/run-tests-on-multiple-os-py.yml | 40 +++++++++++++++++++
 .../run-tests-on-multiple-py-versions.yml     | 28 -------------
 devops/templates/run-tests.yml                |  7 ++--
 tests/e2e/test_image_channel.py               |  3 ++
 6 files changed, 67 insertions(+), 38 deletions(-)
 create mode 100644 devops/templates/run-tests-on-multiple-os-py.yml
 delete mode 100644 devops/templates/run-tests-on-multiple-py-versions.yml

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index fd25129..78d853b 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -11,15 +11,24 @@ stages:
           parameters:
             pyVersion: '3.6'
 
-  - stage: run_tests
+  - stage: run_full_test_matrix
     dependsOn: static_analysis
-    pool:
-      vmImage: 'ubuntu-18.04'
     jobs:
-      - template: templates/run-tests-on-multiple-py-versions.yml
+      - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6', '3.7', '3.8']
-          testTypes: ['fast']
+          testTypes: ['unit', 'io']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+
+  - stage: run_e2e_test_matrix
+    dependsOn: static_analysis
+    jobs:
+      - template: templates/run-tests-on-multiple-os-py.yml
+        parameters:
+          pyVersions: ['3.6']
+          testTypes: ['slow', 'azure']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
     
     
diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
index 6f61c23..8b78637 100644
--- a/devops/templates/publish-cov.yml
+++ b/devops/templates/publish-cov.yml
@@ -1,11 +1,15 @@
+parameters:
+- name: pyVersion
+  type: string
+
 steps:
 - task: PublishTestResults@2
   inputs:
     testResultsFormat: 'JUnit' 
     testResultsFiles: 'junit/*.xml' 
     searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName)_$(pyVersion) Build
-    buildPlatform: $(imageName)
+    testRunTitle: $(Agent.OS) py$(pyVersion) Build
+    buildPlatform: $(Agent.OS)
   condition: always() # Always publish test results
   displayName: 'Publish unit test report'
 
diff --git a/devops/templates/run-tests-on-multiple-os-py.yml b/devops/templates/run-tests-on-multiple-os-py.yml
new file mode 100644
index 0000000..8b3c1ac
--- /dev/null
+++ b/devops/templates/run-tests-on-multiple-os-py.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: pyVersions
+  type: object
+  default: ['3.6', '3.7', '3.8']
+- name: testTypes
+  type: object
+  default: ['fast', 'slow']
+- name: imageOSs
+  type: object
+  default: ['ubuntu-latest']
+
+jobs:
+  - ${{ each imageOS in parameters.imageOSs }}:
+    - ${{ each pyVersion in parameters.pyVersions }}:
+      - job:
+        displayName: ${{imageOS}} py${{pyVersion}}
+        pool:
+          vmImage: ${{imageOS}}
+        steps:
+
+        - task: UsePythonVersion@0
+          inputs:
+            versionSpec: ${{pyVersion}}
+            addToPath: true
+            architecture: 'x64'
+          displayName: 'Use Python ${{pyVersion}}'
+
+        - template: install-dependencies.yml
+
+        - ${{ each testType in parameters.testTypes }}:
+          - template: run-tests.yml
+            parameters:
+              testType: ${{testType}}
+          
+          - template: publish-test-results.yml
+            parameters:
+              pyVersion: ${{pyVersion}}
+
+        - template: publish-cov.yml
+            
\ No newline at end of file
diff --git a/devops/templates/run-tests-on-multiple-py-versions.yml b/devops/templates/run-tests-on-multiple-py-versions.yml
deleted file mode 100644
index 7a49545..0000000
--- a/devops/templates/run-tests-on-multiple-py-versions.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-parameters:
-- name: pyVersions
-  type: object
-  default: ['3.6', '3.7', '3.8']
-- name: testTypes
-  type: object
-  default: ['fast', 'slow']
-
-jobs:
-- ${{ each pyVersion in parameters.pyVersions }}:
-  - job: run_tests_py${{ replace(pyVersion, '.', '_') }}
-    steps:
-    
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: ${{pyVersion}}
-        addToPath: true
-        architecture: 'x64'
-      displayName: 'Use Python ${{pyVersion}}'
-
-    - template: install-dependencies.yml
-
-    - ${{ each testType in parameters.testTypes }}:
-      - template: run-tests.yml
-        parameters:
-          testType: ${{testType}}
-
-    - template: publish-cov.yml
\ No newline at end of file
diff --git a/devops/templates/run-tests.yml b/devops/templates/run-tests.yml
index dab1a8d..cff3a15 100644
--- a/devops/templates/run-tests.yml
+++ b/devops/templates/run-tests.yml
@@ -7,7 +7,8 @@ parameters:
   values:
     - slow
     - azure
-    - fast
+    - unit
+    - io
     - all
 
 steps:
@@ -15,9 +16,9 @@ steps:
     if [[ '${{parameters.testType}}' == 'all' ]]
     then
       tox -e py 
-    elif [[ '${{parameters.testType}}' == 'fast' ]]
+    elif [[ '${{parameters.testType}}' == 'unit' ]]
     then
-      tox -e py -- -m "not slow and not azure"
+      tox -e py -- tests/unit
     else
       tox -e py -- -m "${{parameters.testType}}"
     fi
diff --git a/tests/e2e/test_image_channel.py b/tests/e2e/test_image_channel.py
index 461a116..40c5382 100644
--- a/tests/e2e/test_image_channel.py
+++ b/tests/e2e/test_image_channel.py
@@ -15,6 +15,7 @@ def doc_generator():
     return DocumentGenerator(template_path=TEMPLATE_PATH)
 
 
+@pytest.mark.io
 def test_red_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -25,6 +26,7 @@ def test_red_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "red.png", img_array)
 
 
+@pytest.mark.io
 def test_green_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -35,6 +37,7 @@ def test_green_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "green.png", img_array)
 
 
+@pytest.mark.io
 def test_blue_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:

From b534f479edc522a7dd9aa29277e60c0d3afaeadc Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 14:12:40 -0500
Subject: [PATCH 08/20] Separate test report in a template

---
 devops/templates/publish-cov.yml          | 14 +-------------
 devops/templates/publish-test-results.yml | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 13 deletions(-)
 create mode 100644 devops/templates/publish-test-results.yml

diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
index 8b78637..efabfbb 100644
--- a/devops/templates/publish-cov.yml
+++ b/devops/templates/publish-cov.yml
@@ -1,18 +1,6 @@
-parameters:
-- name: pyVersion
-  type: string
+# Template for publishing code coverage report
 
 steps:
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'JUnit' 
-    testResultsFiles: 'junit/*.xml' 
-    searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(Agent.OS) py$(pyVersion) Build
-    buildPlatform: $(Agent.OS)
-  condition: always() # Always publish test results
-  displayName: 'Publish unit test report'
-
 - task: PublishCodeCoverageResults@1
   inputs:
     codeCoverageTool: Cobertura
diff --git a/devops/templates/publish-test-results.yml b/devops/templates/publish-test-results.yml
new file mode 100644
index 0000000..9f82c55
--- /dev/null
+++ b/devops/templates/publish-test-results.yml
@@ -0,0 +1,16 @@
+# Template for publishing test result report
+
+parameters:
+- name: pyVersion
+  type: string
+
+steps:
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: 'JUnit' 
+    testResultsFiles: 'junit/*.xml' 
+    searchFolder: $(Build.SourcesDirectory)
+    testRunTitle: $(Agent.OS) py$(pyVersion) Build
+    buildPlatform: $(Agent.OS)
+  condition: always() # Always publish test results
+  displayName: 'Publish unit test report'
\ No newline at end of file

From 67c12a3c82ce239372581345d939ec33480d9948 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 14:31:23 -0500
Subject: [PATCH 09/20] Add final cov stage

---
 devops/pr-gate-os.yml                     | 17 ++++++++++++-----
 devops/templates/publish-cov.yml          |  1 -
 devops/templates/publish-test-results.yml |  1 -
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 78d853b..522d9da 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -11,26 +11,33 @@ stages:
           parameters:
             pyVersion: '3.6'
 
-  - stage: run_full_test_matrix
+  - stage: run_unit_test_matrix
     dependsOn: static_analysis
     jobs:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
-          pyVersions: ['3.6', '3.7', '3.8']
+          pyVersions: ['3.6']
           testTypes: ['unit', 'io']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-
   - stage: run_e2e_test_matrix
     dependsOn: static_analysis
     jobs:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6']
-          testTypes: ['slow', 'azure']
+          testTypes: ['slow']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-    
+  - stage: collect_final_code_coverage
+    dependsOn: 
+      - run_unit_test_matrix
+      - run_e2e_test_matrix
+    jobs:
+      - job:
+        steps:
+          - template: templates/publish-cov.yml
+
     
 
 # strategy:
diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
index efabfbb..67388a5 100644
--- a/devops/templates/publish-cov.yml
+++ b/devops/templates/publish-cov.yml
@@ -1,5 +1,4 @@
 # Template for publishing code coverage report
-
 steps:
 - task: PublishCodeCoverageResults@1
   inputs:
diff --git a/devops/templates/publish-test-results.yml b/devops/templates/publish-test-results.yml
index 9f82c55..8ac2bb9 100644
--- a/devops/templates/publish-test-results.yml
+++ b/devops/templates/publish-test-results.yml
@@ -1,5 +1,4 @@
 # Template for publishing test result report
-
 parameters:
 - name: pyVersion
   type: string

From 6316a9c9536aaf1f9206f73c51fb1c50c080cde8 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 16:48:59 -0500
Subject: [PATCH 10/20] Cache code coverage report

---
 devops/pr-gate-os.yml                         | 57 +++----------------
 devops/templates/merge-cov-reports.yml        | 44 ++++++++++++++
 devops/templates/publish-cov.yml              |  6 --
 .../templates/run-tests-on-multiple-os-py.yml | 11 +++-
 4 files changed, 59 insertions(+), 59 deletions(-)
 create mode 100644 devops/templates/merge-cov-reports.yml
 delete mode 100644 devops/templates/publish-cov.yml

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 522d9da..46849bd 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -11,74 +11,31 @@ stages:
           parameters:
             pyVersion: '3.6'
 
-  - stage: run_unit_test_matrix
+  - stage: unit_tests
     dependsOn: static_analysis
     jobs:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6']
-          testTypes: ['unit', 'io']
+          testTypes: ['unit']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-  - stage: run_e2e_test_matrix
+  - stage: e2e_tests
     dependsOn: static_analysis
     jobs:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6']
-          testTypes: ['slow']
+          testTypes: ['io']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
   - stage: collect_final_code_coverage
     dependsOn: 
-      - run_unit_test_matrix
-      - run_e2e_test_matrix
+      - unit_tests
+      - e2e_tests
     jobs:
-      - job:
-        steps:
-          - template: templates/publish-cov.yml
+      - template: templates/merge-cov-reports.yml
 
-    
-
-# strategy:
-#   matrix:
-#     linux_x64_py3.6:
-#       imageName: 'ubuntu-18.04'
-#       pyVersion: '3.6'
-
-    # windows_x64_py3.6:
-    #   imageName: 'windows-2019'
-    #   pyVersion: '3.6'
-
-    # macos_x64_py3.6:
-    #   imageName: 'macOS-10.14'
-    #   pyVersion: '3.6'
-
-    # To build using more python versions follow the syntax below:
-    # linux_x64_py3.5:
-    #   imageName: 'ubuntu-16.04'
-    #   pyVersion: '3.5'
-
-# pool:
-#   vmImage: '$(imageName)'
-
-# steps:
-# - template: templates/run-linter.yml
-#   parameters:
-#     pyVersion: $(python.version)
-
-# - template: templates/run-tests.yml
-#   parameters:
-#     testType: 'fast'
-#     imageName: $(imageName)
-#     pyVersion: $(python.version)
-
-# - template: templates/run-tests.yml
-#   parameters:
-#     testType: 'slow'
-#     skipInstall: true
-#     imageName: $(imageName)
-#     pyVersion: $(python.version)
 
 # - template: templates/publish-cov.yml
   
diff --git a/devops/templates/merge-cov-reports.yml b/devops/templates/merge-cov-reports.yml
new file mode 100644
index 0000000..9d55e23
--- /dev/null
+++ b/devops/templates/merge-cov-reports.yml
@@ -0,0 +1,44 @@
+# Template to merge several code coverage reports (.coverage*)
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+jobs:
+  - job:
+    displayName: Merge Code Coverage Reports
+    pool:
+        vmImage: 'ubuntu-latest'
+
+    steps:
+    - checkout: none
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.pyVersion }}
+        addToPath: true
+        architecture: 'x64'
+      displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+    - bash: |
+        python -m pip --upgrade pip setuptools
+        python -m pip install coverage
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: 'Install coverage'
+         
+    # See https://docs.microsoft.com/en-us/azure/devops/pipelines/artifacts/pipeline-artifacts?view=azure-devops&tabs=yaml#multiple-artifacts
+    - download: current
+      patterns: '**/.coverage*'
+
+    - bash: |
+        python -m coverage combine $(Pipeline.Workspace)/**/.coverage*
+        python -m coverage report
+        python -m coverage xml
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: Show and merge cached coverage report
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: Cobertura
+        summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
+      displayName: 'Publish merged code coverage report'
+
+
diff --git a/devops/templates/publish-cov.yml b/devops/templates/publish-cov.yml
deleted file mode 100644
index 67388a5..0000000
--- a/devops/templates/publish-cov.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Template for publishing code coverage report
-steps:
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: '$(Build.SourcesDirectory)/**/coverage.xml'
\ No newline at end of file
diff --git a/devops/templates/run-tests-on-multiple-os-py.yml b/devops/templates/run-tests-on-multiple-os-py.yml
index 8b3c1ac..ea5b8c7 100644
--- a/devops/templates/run-tests-on-multiple-os-py.yml
+++ b/devops/templates/run-tests-on-multiple-os-py.yml
@@ -31,10 +31,15 @@ jobs:
           - template: run-tests.yml
             parameters:
               testType: ${{testType}}
-          
           - template: publish-test-results.yml
             parameters:
               pyVersion: ${{pyVersion}}
 
-        - template: publish-cov.yml
-            
\ No newline at end of file
+        - bash: |
+            mv .coverage .coverage_$(System.StageName)_${{imageOS}}_${{pyVersion}}
+            ls .coverage*
+          workingDirectory: $(Build.SourcesDirectory)
+          displayName: 'Rename coverage report'
+        # Cache the coverage report
+        - publish: $(Build.SourcesDirectory)/.coverage_$(System.StageName)_${{imageOS}}_${{pyVersion}}
+          artifact: cov_report_$(System.StageName)_${{imageOS}}_${{pyVersion}}
\ No newline at end of file

From 547aa2c0f940c2ed4ef5193e7f444f8ac6864e41 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 20:36:27 -0500
Subject: [PATCH 11/20] Skip install direct dep

---
 devops/templates/install-dependencies.yml | 1 -
 devops/templates/merge-cov-reports.yml    | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/devops/templates/install-dependencies.yml b/devops/templates/install-dependencies.yml
index a543be2..9c6fc87 100644
--- a/devops/templates/install-dependencies.yml
+++ b/devops/templates/install-dependencies.yml
@@ -2,7 +2,6 @@
 steps:
 - bash: |
     python -m pip install --upgrade pip setuptools wheel
-    python -m pip install -r requirements.txt 
     python -m pip install -r requirements-dev.txt
   workingDirectory: $(Build.SourcesDirectory)
   displayName: 'Install dependencies'
diff --git a/devops/templates/merge-cov-reports.yml b/devops/templates/merge-cov-reports.yml
index 9d55e23..1c5e8c2 100644
--- a/devops/templates/merge-cov-reports.yml
+++ b/devops/templates/merge-cov-reports.yml
@@ -5,12 +5,11 @@ parameters:
 
 jobs:
   - job:
-    displayName: Merge Code Coverage Reports
+    displayName: Merge cov reports
     pool:
         vmImage: 'ubuntu-latest'
 
     steps:
-    - checkout: none
     - task: UsePythonVersion@0
       inputs:
         versionSpec: ${{ parameters.pyVersion }}

From 60585902d15a20605c43d1b8957f2e828d4275b8 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 21:03:21 -0500
Subject: [PATCH 12/20] Add nightly build

---
 devops/nightly.yml                       | 12 +++++++++
 devops/templates/build_wheel_n_sdist.yml | 31 ++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 devops/nightly.yml
 create mode 100644 devops/templates/build_wheel_n_sdist.yml

diff --git a/devops/nightly.yml b/devops/nightly.yml
new file mode 100644
index 0000000..7a79ac6
--- /dev/null
+++ b/devops/nightly.yml
@@ -0,0 +1,12 @@
+name: $(Date:yyyyMMdd).$(Rev:r)
+
+stages:
+  - template: pr-gate-os.yml
+
+  - stage: publish_artifacts
+    jobs:
+      - job: archive_wheel_and_sdist
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - template: templates/build_wheel_n_sdist.yml
diff --git a/devops/templates/build_wheel_n_sdist.yml b/devops/templates/build_wheel_n_sdist.yml
new file mode 100644
index 0000000..3d5802c
--- /dev/null
+++ b/devops/templates/build_wheel_n_sdist.yml
@@ -0,0 +1,31 @@
+# Template to create wheel and source distribution
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pyVersion }}
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+  displayName: 'Update pip and setuptools'
+
+- bash: |
+    python setup.py bdist_wheel
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build wheel'
+
+- bash: |
+    python setup.py sdist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build source distribution'
+
+- bash: |
+    ls dist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Show artifacts in folder'

From be824f4890acb5ac359085ad4b1e0102a36c5fe9 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 21:06:14 -0500
Subject: [PATCH 13/20] Update PR-gate

---
 devops/pr-gate.yml | 121 +++++++++++++--------------------------------
 1 file changed, 34 insertions(+), 87 deletions(-)

diff --git a/devops/pr-gate.yml b/devops/pr-gate.yml
index dec22a3..d7e826f 100644
--- a/devops/pr-gate.yml
+++ b/devops/pr-gate.yml
@@ -1,93 +1,40 @@
-trigger:
-  branches:
-    include:
-      - main
-  
-strategy:
-  matrix:
-    linux_x64_py3.6:
-      imageName: 'ubuntu-18.04'
-      python.version: '3.6'
-
-    # windows_x64_py3.6:
-    #   imageName: 'windows-2019'
-    #   python.version: '3.6'
-
-    # macos_x64_py3.6:
-    #   imageName: 'macOS-10.14'
-    #   python.version: '3.6'
-
-    # To build using more python versions follow the syntax below:
-    # linux_x64_py3.5:
-    #   imageName: 'ubuntu-16.04'
-    #   python.version: '3.5'
-
-pool:
-  vmImage: '$(imageName)'
-  
 name: $(Date:yyyyMMdd).$(Rev:r)
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-    addToPath: true
-    architecture: 'x64'
-  displayName: 'Use Python $(python.version)'
 
-- bash: |
-    python -m pip install --upgrade pip 
-    python -m pip install setuptools wheel
-    python -m pip install -r requirements.txt 
-    python -m pip install -r requirements-dev.txt
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Install dependencies'
+pr:
+- main
 
-- bash: |
-    tox -e flake8
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Run Linter (flake8)'
+stages:
+  - stage: static_analysis
+    jobs:
+    - job: flake8_linux_py36
+      pool:
+        vmImage: 'ubuntu-latest'
+      steps:
+        - template: templates/run-linter.yml
+          parameters:
+            pyVersion: '3.6'
 
-- bash: |
-    tox -e py
-  env:
-    BLOB_KEY : $(BLOB_KEY)
-    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
-    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
-    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY) 
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Running unit tests'
+  - stage: unit_tests
+    dependsOn: static_analysis
+    jobs:
+      - template: templates/run-tests-on-multiple-os-py.yml
+        parameters:
+          pyVersions: ['3.6']
+          testTypes: ['unit']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
+  - stage: e2e_tests
+    dependsOn: static_analysis
+    jobs:
+      - template: templates/run-tests-on-multiple-os-py.yml
+        parameters:
+          pyVersions: ['3.6']
+          testTypes: ['io']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'JUnit' 
-    testResultsFiles: 'junit/*.xml' 
-    searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName) Build
-    buildPlatform: $(imageName)
-  displayName: 'Publish unit test report'
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: $(Build.SourcesDirectory)/coverage.xml
-    reportDirectory: $(Build.SourcesDirectory)/htmlcov
-  displayName: 'Publish test coverage'
-  
-- bash: |
-    python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Building wheel package'
-
-- bash: |
-    mkdir $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-    cp -r dist/* $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Copying builds to artifact staging dir'
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)
-    ArtifactName: genalog
-    publishLocation: 'Container'
-  displayName: 'Publish build as artifacts'
+  - stage: collect_final_code_coverage
+    dependsOn: 
+      - unit_tests
+      - e2e_tests
+    jobs:
+      - template: templates/merge-cov-reports.yml

From d83305e1f9166c324a9e6a375cc04d39f8878604 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 21:21:13 -0500
Subject: [PATCH 14/20] Run azure tests

---
 devops/pr-gate-os.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 46849bd..0ee6412 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -17,7 +17,7 @@ stages:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6']
-          testTypes: ['unit']
+          testTypes: ['unit', 'io']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
   - stage: e2e_tests
@@ -26,7 +26,7 @@ stages:
       - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6']
-          testTypes: ['io']
+          testTypes: ['azure']
           imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
   - stage: collect_final_code_coverage

From c7e1b62cdceef7e801579cccdb48dcbc195da3a7 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 21:36:44 -0500
Subject: [PATCH 15/20] Restructure templates

---
 devops/nightly.yml                            | 53 +++++++++++++---
 devops/pr-gate-os.yml                         | 23 +------
 devops/pr-gate.yml                            | 62 +++++++++----------
 .../{ => base}/publish-test-results.yml       |  0
 devops/templates/{ => base}/run-linter.yml    |  0
 devops/templates/{ => base}/run-tests.yml     |  0
 .../templates/run-tests-on-multiple-os-py.yml |  4 +-
 7 files changed, 78 insertions(+), 64 deletions(-)
 rename devops/templates/{ => base}/publish-test-results.yml (100%)
 rename devops/templates/{ => base}/run-linter.yml (100%)
 rename devops/templates/{ => base}/run-tests.yml (100%)

diff --git a/devops/nightly.yml b/devops/nightly.yml
index 7a79ac6..f7c1f9d 100644
--- a/devops/nightly.yml
+++ b/devops/nightly.yml
@@ -1,12 +1,47 @@
 name: $(Date:yyyyMMdd).$(Rev:r)
 
-stages:
-  - template: pr-gate-os.yml
+trigger: none # nightly build is scheduled once per day
 
-  - stage: publish_artifacts
-    jobs:
-      - job: archive_wheel_and_sdist
-        pool:
-          vmImage: 'ubuntu-latest'
-        steps:
-          - template: templates/build_wheel_n_sdist.yml
+stages:
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+      - template: templates/base/run-linter.yml
+        parameters:
+          pyVersion: '3.6'
+
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+    - template: templates/run-tests-on-multiple-os-py.yml
+      parameters:
+        pyVersions: ['3.6']
+        testTypes: ['unit', 'io']
+        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+    - template: templates/run-tests-on-multiple-os-py.yml
+      parameters:
+        pyVersions: ['3.6']
+        testTypes: ['azure']
+        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: collect_final_code_coverage
+  dependsOn: 
+    - unit_tests
+    - e2e_tests
+  jobs:
+    - template: templates/merge-cov-reports.yml
+
+- stage: publish_artifacts
+  jobs:
+    - job: archive_wheel_and_sdist
+      pool:
+        vmImage: 'ubuntu-latest'
+      steps:
+        - template: templates/build_wheel_n_sdist.yml
diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 0ee6412..7b075b9 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -7,7 +7,7 @@ stages:
       pool:
         vmImage: 'ubuntu-latest'
       steps:
-        - template: templates/run-linter.yml
+        - template: templates/base/run-linter.yml
           parameters:
             pyVersion: '3.6'
 
@@ -35,24 +35,3 @@ stages:
       - e2e_tests
     jobs:
       - template: templates/merge-cov-reports.yml
-
-
-# - template: templates/publish-cov.yml
-  
-# - bash: |
-#     python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
-#   workingDirectory: $(Build.SourcesDirectory)
-#   displayName: 'Building wheel package'
-
-# - bash: |
-#     mkdir $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-#     cp -r dist/* $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-#   workingDirectory: $(Build.SourcesDirectory)
-#   displayName: 'Copying builds to artifact staging dir'
-
-# - task: PublishBuildArtifacts@1
-#   inputs:
-#     PathtoPublish: $(Build.ArtifactStagingDirectory)
-#     ArtifactName: genalog
-#     publishLocation: 'Container'
-#   displayName: 'Publish build as artifacts'
diff --git a/devops/pr-gate.yml b/devops/pr-gate.yml
index d7e826f..cded71a 100644
--- a/devops/pr-gate.yml
+++ b/devops/pr-gate.yml
@@ -4,37 +4,37 @@ pr:
 - main
 
 stages:
-  - stage: static_analysis
-    jobs:
-    - job: flake8_linux_py36
-      pool:
-        vmImage: 'ubuntu-latest'
-      steps:
-        - template: templates/run-linter.yml
-          parameters:
-            pyVersion: '3.6'
-
-  - stage: unit_tests
-    dependsOn: static_analysis
-    jobs:
-      - template: templates/run-tests-on-multiple-os-py.yml
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+      - template: templates/base/run-linter.yml
         parameters:
-          pyVersions: ['3.6']
-          testTypes: ['unit']
-          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+          pyVersion: '3.6'
 
-  - stage: e2e_tests
-    dependsOn: static_analysis
-    jobs:
-      - template: templates/run-tests-on-multiple-os-py.yml
-        parameters:
-          pyVersions: ['3.6']
-          testTypes: ['io']
-          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+    - template: templates/run-tests-on-multiple-os-py.yml
+      parameters:
+        pyVersions: ['3.6']
+        testTypes: ['unit']
+        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-  - stage: collect_final_code_coverage
-    dependsOn: 
-      - unit_tests
-      - e2e_tests
-    jobs:
-      - template: templates/merge-cov-reports.yml
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+    - template: templates/run-tests-on-multiple-os-py.yml
+      parameters:
+        pyVersions: ['3.6']
+        testTypes: ['io']
+        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: collect_final_code_coverage
+  dependsOn: 
+    - unit_tests
+    - e2e_tests
+  jobs:
+    - template: templates/merge-cov-reports.yml
diff --git a/devops/templates/publish-test-results.yml b/devops/templates/base/publish-test-results.yml
similarity index 100%
rename from devops/templates/publish-test-results.yml
rename to devops/templates/base/publish-test-results.yml
diff --git a/devops/templates/run-linter.yml b/devops/templates/base/run-linter.yml
similarity index 100%
rename from devops/templates/run-linter.yml
rename to devops/templates/base/run-linter.yml
diff --git a/devops/templates/run-tests.yml b/devops/templates/base/run-tests.yml
similarity index 100%
rename from devops/templates/run-tests.yml
rename to devops/templates/base/run-tests.yml
diff --git a/devops/templates/run-tests-on-multiple-os-py.yml b/devops/templates/run-tests-on-multiple-os-py.yml
index ea5b8c7..02624e7 100644
--- a/devops/templates/run-tests-on-multiple-os-py.yml
+++ b/devops/templates/run-tests-on-multiple-os-py.yml
@@ -28,10 +28,10 @@ jobs:
         - template: install-dependencies.yml
 
         - ${{ each testType in parameters.testTypes }}:
-          - template: run-tests.yml
+          - template: base/run-tests.yml
             parameters:
               testType: ${{testType}}
-          - template: publish-test-results.yml
+          - template: base/publish-test-results.yml
             parameters:
               pyVersion: ${{pyVersion}}
 

From cac5394a69fc87a8160f5d19c94d770c1e478ec4 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 21:56:36 -0500
Subject: [PATCH 16/20] Use variable group

---
 devops/nightly.yml                             | 3 +++
 devops/pr-gate-os.yml                          | 3 +++
 devops/pr-gate.yml                             | 5 +++++
 devops/templates/base/publish-test-results.yml | 2 +-
 devops/templates/base/run-tests.yml            | 4 ++--
 5 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/devops/nightly.yml b/devops/nightly.yml
index f7c1f9d..5a54e8b 100644
--- a/devops/nightly.yml
+++ b/devops/nightly.yml
@@ -2,6 +2,9 @@ name: $(Date:yyyyMMdd).$(Rev:r)
 
 trigger: none # nightly build is scheduled once per day
 
+variables:
+  - group: azureResourceKeys
+
 stages:
 - stage: static_analysis
   jobs:
diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
index 7b075b9..0cb575b 100644
--- a/devops/pr-gate-os.yml
+++ b/devops/pr-gate-os.yml
@@ -1,5 +1,8 @@
 name: $(Date:yyyyMMdd).$(Rev:r)
 
+variables:
+  - group: azureResourceKeys
+
 stages:
   - stage: static_analysis
     jobs:
diff --git a/devops/pr-gate.yml b/devops/pr-gate.yml
index cded71a..a85f3b8 100644
--- a/devops/pr-gate.yml
+++ b/devops/pr-gate.yml
@@ -1,8 +1,13 @@
 name: $(Date:yyyyMMdd).$(Rev:r)
 
+trigger: none # trigger only via pr
+
 pr:
 - main
 
+variables:
+  - group: azureResourceKeys
+
 stages:
 - stage: static_analysis
   jobs:
diff --git a/devops/templates/base/publish-test-results.yml b/devops/templates/base/publish-test-results.yml
index 8ac2bb9..55b2a3d 100644
--- a/devops/templates/base/publish-test-results.yml
+++ b/devops/templates/base/publish-test-results.yml
@@ -12,4 +12,4 @@ steps:
     testRunTitle: $(Agent.OS) py$(pyVersion) Build
     buildPlatform: $(Agent.OS)
   condition: always() # Always publish test results
-  displayName: 'Publish unit test report'
\ No newline at end of file
+  displayName: 'Publish test report'
\ No newline at end of file
diff --git a/devops/templates/base/run-tests.yml b/devops/templates/base/run-tests.yml
index cff3a15..2a4751e 100644
--- a/devops/templates/base/run-tests.yml
+++ b/devops/templates/base/run-tests.yml
@@ -1,5 +1,4 @@
 # Template for running tests on multiple Python versions and platforms
-
 parameters:
 - name: testType
   type: string
@@ -23,7 +22,8 @@ steps:
       tox -e py -- -m "${{parameters.testType}}"
     fi
   env:
-    BLOB_KEY : $(BLOB_KEY)
+    # These keys come from azureResourceKeys variable group
+    BLOB_KEY : $(BLOB_KEY) 
     SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
     COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
     COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY)

From 8d8b12dbc5d13ad34d4e87e6103abb831493096e Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 22:08:29 -0500
Subject: [PATCH 17/20] Small format change

---
 devops/nightly.yml | 42 +++++++++++++++++++++---------------------
 devops/pr-gate.yml | 28 ++++++++++++++--------------
 2 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/devops/nightly.yml b/devops/nightly.yml
index 5a54e8b..2dabaad 100644
--- a/devops/nightly.yml
+++ b/devops/nightly.yml
@@ -12,39 +12,39 @@ stages:
     pool:
       vmImage: 'ubuntu-latest'
     steps:
-      - template: templates/base/run-linter.yml
-        parameters:
-          pyVersion: '3.6'
+    - template: templates/base/run-linter.yml
+      parameters:
+        pyVersion: '3.6'
 
 - stage: unit_tests
   dependsOn: static_analysis
   jobs:
-    - template: templates/run-tests-on-multiple-os-py.yml
-      parameters:
-        pyVersions: ['3.6']
-        testTypes: ['unit', 'io']
-        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['unit', 'io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: e2e_tests
   dependsOn: static_analysis
   jobs:
-    - template: templates/run-tests-on-multiple-os-py.yml
-      parameters:
-        pyVersions: ['3.6']
-        testTypes: ['azure']
-        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['azure']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: collect_final_code_coverage
   dependsOn: 
-    - unit_tests
-    - e2e_tests
+  - unit_tests
+  - e2e_tests
   jobs:
-    - template: templates/merge-cov-reports.yml
+  - template: templates/merge-cov-reports.yml
 
 - stage: publish_artifacts
   jobs:
-    - job: archive_wheel_and_sdist
-      pool:
-        vmImage: 'ubuntu-latest'
-      steps:
-        - template: templates/build_wheel_n_sdist.yml
+  - job: archive_wheel_and_sdist
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+    - template: templates/build_wheel_n_sdist.yml
diff --git a/devops/pr-gate.yml b/devops/pr-gate.yml
index a85f3b8..1de4a38 100644
--- a/devops/pr-gate.yml
+++ b/devops/pr-gate.yml
@@ -6,7 +6,7 @@ pr:
 - main
 
 variables:
-  - group: azureResourceKeys
+- group: azureResourceKeys
 
 stages:
 - stage: static_analysis
@@ -22,24 +22,24 @@ stages:
 - stage: unit_tests
   dependsOn: static_analysis
   jobs:
-    - template: templates/run-tests-on-multiple-os-py.yml
-      parameters:
-        pyVersions: ['3.6']
-        testTypes: ['unit']
-        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['unit']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: e2e_tests
   dependsOn: static_analysis
   jobs:
-    - template: templates/run-tests-on-multiple-os-py.yml
-      parameters:
-        pyVersions: ['3.6']
-        testTypes: ['io']
-        imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: collect_final_code_coverage
   dependsOn: 
-    - unit_tests
-    - e2e_tests
+  - unit_tests
+  - e2e_tests
   jobs:
-    - template: templates/merge-cov-reports.yml
+  - template: templates/merge-cov-reports.yml

From 4456fcaffa0263d11169c3049e288c1da3326aac Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 22:10:02 -0500
Subject: [PATCH 18/20] Update test matrix

---
 devops/nightly.yml | 4 ++--
 devops/pr-gate.yml | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/devops/nightly.yml b/devops/nightly.yml
index 2dabaad..1f0b22d 100644
--- a/devops/nightly.yml
+++ b/devops/nightly.yml
@@ -21,7 +21,7 @@ stages:
   jobs:
   - template: templates/run-tests-on-multiple-os-py.yml
     parameters:
-      pyVersions: ['3.6']
+      pyVersions: ['3.6', '3.7', '3.8']
       testTypes: ['unit', 'io']
       imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
@@ -31,7 +31,7 @@ stages:
   - template: templates/run-tests-on-multiple-os-py.yml
     parameters:
       pyVersions: ['3.6']
-      testTypes: ['azure']
+      testTypes: ['azure', 'slow']
       imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: collect_final_code_coverage
diff --git a/devops/pr-gate.yml b/devops/pr-gate.yml
index 1de4a38..497572f 100644
--- a/devops/pr-gate.yml
+++ b/devops/pr-gate.yml
@@ -24,8 +24,8 @@ stages:
   jobs:
   - template: templates/run-tests-on-multiple-os-py.yml
     parameters:
-      pyVersions: ['3.6']
-      testTypes: ['unit']
+      pyVersions: ['3.6', '3.7', '3.8']
+      testTypes: ['unit', 'io']
       imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: e2e_tests
@@ -34,7 +34,7 @@ stages:
   - template: templates/run-tests-on-multiple-os-py.yml
     parameters:
       pyVersions: ['3.6']
-      testTypes: ['io']
+      testTypes: ['azure', 'slow']
       imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 - stage: collect_final_code_coverage

From e074a1d44fb8d5a562b0c2b95104522bebc979ac Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 22:10:35 -0500
Subject: [PATCH 19/20] Remove test pipeline

---
 devops/pr-gate-os.yml | 40 ----------------------------------------
 1 file changed, 40 deletions(-)
 delete mode 100644 devops/pr-gate-os.yml

diff --git a/devops/pr-gate-os.yml b/devops/pr-gate-os.yml
deleted file mode 100644
index 0cb575b..0000000
--- a/devops/pr-gate-os.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-name: $(Date:yyyyMMdd).$(Rev:r)
-
-variables:
-  - group: azureResourceKeys
-
-stages:
-  - stage: static_analysis
-    jobs:
-    - job: flake8_linux_py36
-      pool:
-        vmImage: 'ubuntu-latest'
-      steps:
-        - template: templates/base/run-linter.yml
-          parameters:
-            pyVersion: '3.6'
-
-  - stage: unit_tests
-    dependsOn: static_analysis
-    jobs:
-      - template: templates/run-tests-on-multiple-os-py.yml
-        parameters:
-          pyVersions: ['3.6']
-          testTypes: ['unit', 'io']
-          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
-
-  - stage: e2e_tests
-    dependsOn: static_analysis
-    jobs:
-      - template: templates/run-tests-on-multiple-os-py.yml
-        parameters:
-          pyVersions: ['3.6']
-          testTypes: ['azure']
-          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
-
-  - stage: collect_final_code_coverage
-    dependsOn: 
-      - unit_tests
-      - e2e_tests
-    jobs:
-      - template: templates/merge-cov-reports.yml

From 003bf3154a637783e2017720089d7e1105c2e6a8 Mon Sep 17 00:00:00 2001
From: "Jianjie Liu (MAIDAP)" <jianjl@microsoft.com>
Date: Wed, 27 Jan 2021 22:43:00 -0500
Subject: [PATCH 20/20] Add publish artifacts step

---
 devops/nightly.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/devops/nightly.yml b/devops/nightly.yml
index 1f0b22d..4174752 100644
--- a/devops/nightly.yml
+++ b/devops/nightly.yml
@@ -48,3 +48,10 @@ stages:
       vmImage: 'ubuntu-latest'
     steps:
     - template: templates/build_wheel_n_sdist.yml
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: $(Build.SourcesDirectory)/dist
+        ArtifactName: distribution_artifacts
+        publishLocation: 'Container'
+      displayName: 'Publish wheel and sdist'