Merge pull request #44 from microsoft/chenhui/integration_test_pipeline

added integration test

tests/ci/cpu_integration_tests_linux.yml

@@ -0,0 +1,59 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Pull request against these branches will trigger this build
+pr:
+- master
+- staging
+
+# Any commit to these branches will trigger the build.
+trigger:
+- staging
+- master
+
+
+jobs:
+- job: cpu_integration_tests_linux
+  timeoutInMinutes: 10 # how long to run the job before automatically cancelling
+  pool:
+    # vmImage: 'ubuntu-16.04' # hosted machine 
+    name: ForecastingAgents
+
+  steps:
+  - bash: |
+      echo "##vso[task.prependpath]/data/anaconda/bin"
+      export PATH="/data/anaconda/bin:$PATH"
+      conda env list
+    displayName: Add conda to PATH
+
+  - bash: |
+      echo "Initializing setup ..."
+      pwd
+      ls
+      uname -ra
+      echo "Running cleanup ... "
+      conda env remove -n forecast_cpu
+      echo "Cleanup done."
+    displayName: 'Cleanup existing conda environment'
+
+  - bash: |
+      echo "Running conda env setup ..."
+      yes | conda env create -n forecast_cpu -f tools/environment.yml
+      eval "$(conda shell.bash hook)" && conda activate forecast_cpu
+      pip install -e forecasting_lib
+      echo "Conda env installed."
+    displayName: 'Creating conda environment with dependencies'
+
+  - bash: |
+      sudo ln -s /usr/lib/R/modules/lapack.so /usr/lib/libRlapack.so
+      eval "$(conda shell.bash hook)" && conda activate forecast_cpu
+      python -m ipykernel install --user --name forecast_cpu
+      pytest --durations=0 tests/integration -m "not gpu and not azureml" --junitxml=report/test-integration.xml
+    displayName: 'Run integration tests'
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-integration.xml'
+      testRunTitle: 'Test results for PyTest'
+
+

tests/conftest.py

@@ -0,0 +1,14 @@
+import os
+import pytest
+from forecasting_lib.common.utils import git_repo_path
+
+
+@pytest.fixture(scope="module")
+def notebooks():
+    repo_path = git_repo_path()
+    examples_path = os.path.join(repo_path, "examples")
+    quick_start_path = os.path.join(examples_path, "00_quick_start")
+
+    # Path for the notebooks
+    paths = {"lightgbm_quick_start": os.path.join(quick_start_path, "lightgbm_point_forecast.ipynb")}
+    return paths

tests/integration/test_notebooks_python.py

@@ -0,0 +1,25 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import os
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import pytest
+import papermill as pm
+import scrapbook as sb
+
+ABS_TOL = 5.0
+
+
+@pytest.mark.integration
+def test_lightgbm_quick_start(notebooks):
+    notebook_path = notebooks["lightgbm_quick_start"]
+    output_notebook_path = os.path.join(os.path.dirname(notebook_path), "output.ipynb")
+    pm.execute_notebook(notebook_path, output_notebook_path, kernel_name="forecast_cpu")
+    nb = sb.read_notebook(output_notebook_path)
+    df = nb.scraps.dataframe
+    assert df.shape[0] == 1
+    mape = df.loc[df.name == "MAPE"]["data"][0]
+    assert mape == pytest.approx(35.60, abs=ABS_TOL)

tests/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+markers = 
+    notebooks
+    integration