Merge pull request #15 from microsoft/laserprec/ci_templates

Add the follow CI pipelines and use template to construct all pipelines:

1. Nightly Build
2. PR Gate with full test matrix for unit and e2e tests

devops/nightly.yml

@@ -0,0 +1,57 @@
+name: $(Date:yyyyMMdd).$(Rev:r)
+
+trigger: none # nightly build is scheduled once per day
+
+variables:
+  - group: azureResourceKeys
+
+stages:
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+    - template: templates/base/run-linter.yml
+      parameters:
+        pyVersion: '3.6'
+
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6', '3.7', '3.8']
+      testTypes: ['unit', 'io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['azure', 'slow']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: collect_final_code_coverage
+  dependsOn: 
+  - unit_tests
+  - e2e_tests
+  jobs:
+  - template: templates/merge-cov-reports.yml
+
+- stage: publish_artifacts
+  jobs:
+  - job: archive_wheel_and_sdist
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+    - template: templates/build_wheel_n_sdist.yml
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: $(Build.SourcesDirectory)/dist
+        ArtifactName: distribution_artifacts
+        publishLocation: 'Container'
+      displayName: 'Publish wheel and sdist'

devops/pr-gate.yml

@@ -1,93 +1,45 @@
-trigger:
-  branches:
-    include:
-      - main
-  
-strategy:
-  matrix:
-    linux_x64_py3.6:
-      imageName: 'ubuntu-18.04'
-      python.version: '3.6'
-
-    # windows_x64_py3.6:
-    #   imageName: 'windows-2019'
-    #   python.version: '3.6'
-
-    # macos_x64_py3.6:
-    #   imageName: 'macOS-10.14'
-    #   python.version: '3.6'
-
-    # To build using more python versions follow the syntax below:
-    # linux_x64_py3.5:
-    #   imageName: 'ubuntu-16.04'
-    #   python.version: '3.5'
-
-pool:
-  vmImage: '$(imageName)'
-  
 name: $(Date:yyyyMMdd).$(Rev:r)
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-    addToPath: true
-    architecture: 'x64'
-  displayName: 'Use Python $(python.version)'
 
-- bash: |
-    python -m pip install --upgrade pip 
-    python -m pip install setuptools wheel
-    python -m pip install -r requirements.txt 
-    python -m pip install -r requirements-dev.txt
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Install dependencies'
+trigger: none # trigger only via pr
 
-- bash: |
-    tox -e flake8
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Run Linter (flake8)'
+pr:
+- main
 
-- bash: |
-    tox -e py
-  env:
-    BLOB_KEY : $(BLOB_KEY)
-    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
-    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
-    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY) 
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Running unit tests'
+variables:
+- group: azureResourceKeys
 
+stages:
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+      - template: templates/base/run-linter.yml
+        parameters:
+          pyVersion: '3.6'
 
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'JUnit' 
-    testResultsFiles: 'junit/*.xml' 
-    searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName) Build
-    buildPlatform: $(imageName)
-  displayName: 'Publish unit test report'
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6', '3.7', '3.8']
+      testTypes: ['unit', 'io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: $(Build.SourcesDirectory)/coverage.xml
-    reportDirectory: $(Build.SourcesDirectory)/htmlcov
-  displayName: 'Publish test coverage'
-  
-- bash: |
-    python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Building wheel package'
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['azure', 'slow']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
-- bash: |
-    mkdir $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-    cp -r dist/* $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Copying builds to artifact staging dir'
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)
-    ArtifactName: genalog
-    publishLocation: 'Container'
-  displayName: 'Publish build as artifacts'
+- stage: collect_final_code_coverage
+  dependsOn: 
+  - unit_tests
+  - e2e_tests
+  jobs:
+  - template: templates/merge-cov-reports.yml

devops/templates/base/publish-test-results.yml

@@ -0,0 +1,15 @@
+# Template for publishing test result report
+parameters:
+- name: pyVersion
+  type: string
+
+steps:
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: 'JUnit' 
+    testResultsFiles: 'junit/*.xml' 
+    searchFolder: $(Build.SourcesDirectory)
+    testRunTitle: $(Agent.OS) py$(pyVersion) Build
+    buildPlatform: $(Agent.OS)
+  condition: always() # Always publish test results
+  displayName: 'Publish test report'

devops/templates/base/run-linter.yml

@@ -0,0 +1,24 @@
+# Template for running linter and other static analysis tools on the code
+parameters:
+- name: pyVersion
+  type: string
+  default: '3.6'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pyVersion }}
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install flake8 and other dev dependencies'
+
+- bash: |
+    tox -e flake8
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Run Linter (flake8)'

devops/templates/base/run-tests.yml

@@ -0,0 +1,31 @@
+# Template for running tests on multiple Python versions and platforms
+parameters:
+- name: testType
+  type: string
+  default: all
+  values:
+    - slow
+    - azure
+    - unit
+    - io
+    - all
+
+steps:
+- bash: |
+    if [[ '${{parameters.testType}}' == 'all' ]]
+    then
+      tox -e py 
+    elif [[ '${{parameters.testType}}' == 'unit' ]]
+    then
+      tox -e py -- tests/unit
+    else
+      tox -e py -- -m "${{parameters.testType}}"
+    fi
+  env:
+    # These keys come from azureResourceKeys variable group
+    BLOB_KEY : $(BLOB_KEY) 
+    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
+    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
+    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY)
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Running (${{parameters.testType}}) Tests'

devops/templates/build_wheel_n_sdist.yml

@@ -0,0 +1,31 @@
+# Template to create wheel and source distribution
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pyVersion }}
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+  displayName: 'Update pip and setuptools'
+
+- bash: |
+    python setup.py bdist_wheel
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build wheel'
+
+- bash: |
+    python setup.py sdist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build source distribution'
+
+- bash: |
+    ls dist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Show artifacts in folder'

devops/templates/install-dependencies.yml

@@ -0,0 +1,7 @@
+# Assume a python version is enabled with "UsePythonVersion@0" task
+steps:
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install dependencies'

devops/templates/merge-cov-reports.yml

@@ -0,0 +1,43 @@
+# Template to merge several code coverage reports (.coverage*)
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+jobs:
+  - job:
+    displayName: Merge cov reports
+    pool:
+        vmImage: 'ubuntu-latest'
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.pyVersion }}
+        addToPath: true
+        architecture: 'x64'
+      displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+    - bash: |
+        python -m pip --upgrade pip setuptools
+        python -m pip install coverage
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: 'Install coverage'
+         
+    # See https://docs.microsoft.com/en-us/azure/devops/pipelines/artifacts/pipeline-artifacts?view=azure-devops&tabs=yaml#multiple-artifacts
+    - download: current
+      patterns: '**/.coverage*'
+
+    - bash: |
+        python -m coverage combine $(Pipeline.Workspace)/**/.coverage*
+        python -m coverage report
+        python -m coverage xml
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: Show and merge cached coverage report
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: Cobertura
+        summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
+      displayName: 'Publish merged code coverage report'
+
+

devops/templates/run-tests-on-multiple-os-py.yml

@@ -0,0 +1,45 @@
+parameters:
+- name: pyVersions
+  type: object
+  default: ['3.6', '3.7', '3.8']
+- name: testTypes
+  type: object
+  default: ['fast', 'slow']
+- name: imageOSs
+  type: object
+  default: ['ubuntu-latest']
+
+jobs:
+  - ${{ each imageOS in parameters.imageOSs }}:
+    - ${{ each pyVersion in parameters.pyVersions }}:
+      - job:
+        displayName: ${{imageOS}} py${{pyVersion}}
+        pool:
+          vmImage: ${{imageOS}}
+        steps:
+
+        - task: UsePythonVersion@0
+          inputs:
+            versionSpec: ${{pyVersion}}
+            addToPath: true
+            architecture: 'x64'
+          displayName: 'Use Python ${{pyVersion}}'
+
+        - template: install-dependencies.yml
+
+        - ${{ each testType in parameters.testTypes }}:
+          - template: base/run-tests.yml
+            parameters:
+              testType: ${{testType}}
+          - template: base/publish-test-results.yml
+            parameters:
+              pyVersion: ${{pyVersion}}
+
+        - bash: |
+            mv .coverage .coverage_$(System.StageName)_${{imageOS}}_${{pyVersion}}
+            ls .coverage*
+          workingDirectory: $(Build.SourcesDirectory)
+          displayName: 'Rename coverage report'
+        # Cache the coverage report
+        - publish: $(Build.SourcesDirectory)/.coverage_$(System.StageName)_${{imageOS}}_${{pyVersion}}
+          artifact: cov_report_$(System.StageName)_${{imageOS}}_${{pyVersion}}

tests/e2e/test_image_channel.py

@@ -15,6 +15,7 @@ def doc_generator():
     return DocumentGenerator(template_path=TEMPLATE_PATH)
 
 
+@pytest.mark.io
 def test_red_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -25,6 +26,7 @@ def test_red_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "red.png", img_array)
 
 
+@pytest.mark.io
 def test_green_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -35,6 +37,7 @@ def test_green_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "green.png", img_array)
 
 
+@pytest.mark.io
 def test_blue_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:

tox.ini

@@ -14,7 +14,13 @@ passenv =
 # https://tox.readthedocs.io/en/latest/example/basic.html#depending-on-requirements-txt-or-defining-constraints
 deps = -rrequirements-dev.txt
 commands = 
-    pytest
+    # {posargs} will be substituded by arguments after the `--` when running.
+    # This will allow running subset of the test suite via tox. 
+    #
+    # EX: tox -- -m "not azure and not slow" 
+    #     will pass {-m "not azure and not slow"} to `pytest`
+    # See https://tox.readthedocs.io/en/latest/example/general.html for more details
+    pytest {posargs}
 
 
 [testenv:flake8]
@@ -32,13 +38,16 @@ junit_family = xunit2
 markers = 
     # These two markers allow to us to run faster subset of the test:
     # EX: pytest -m "not slow and not azure"
+    # See https://docs.pytest.org/en/stable/example/markers.html#registering-markers
     slow: marks tests as slow (deselect with '-m "not slow"')
     azure: marks as integration tests that require azure resource
     io: marks integration tests involving some form of I/O operations (disk, internet, etc)
 testpaths =
     tests
-addopts = 
-    -rsx --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
+addopts =
+    # reports all (except passed tests). See https://docs.pytest.org/en/latest/usage.html#detailed-summary-report
+    -ra 
+    --cov-append --cov=genalog --cov-report=html --cov-report=term-missing --cov-report=xml --junitxml=junit/test-results.xml
 
 
 [flake8]