Generate code coverage report

Signed-off-by: Keith Battocchi <kebatt@microsoft.com>

.github/workflows/ci.yml

@@ -97,14 +97,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        kind: [except customer scenarios, customer scenarios]
+        kind: [except-customer-scenarios, customer-scenarios]
         include:
-        - kind: "except customer scenarios"
+        - kind: "except-customer-scenarios"
           extras: "[tf,plt]"
           pattern: "(?!CustomerScenarios)"
           install_graphviz: true
           version: 3.8 # no supported version of tensorflow for 3.9
-        - kind: "customer scenarios"
+        - kind: "customer-scenarios"
           extras: "[plt,dowhy]"
           pattern: "CustomerScenarios"
           version: 3.9
@@ -127,16 +127,28 @@ jobs:
     # Add verbose flag to pip installation if in debug mode
     - run: pip install -e .${{ matrix.extras }} ${{ fromJSON('["","-v"]')[runner.debug] }}
       name: Install econml
-    - run: pip install pytest pytest-runner jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm
+    - run: pip install pytest pytest-runner coverage jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm
       name: Install test and notebook requirements
     - run: pip list
       name: List installed packages
     - run: python setup.py pytest
       name: Run notebook tests
+      id: run_tests
       env:
         PYTEST_ADDOPTS: '-m "notebook"'
         NOTEBOOK_DIR_PATTERN: ${{ matrix.pattern }}
-    
+        COVERAGE_PROCESS_START: 'setup.cfg'
+    - run: mv .coverage .coverage.${{ matrix.kind }}
+      # Run whether or not the tests passed, but only if they ran at all
+      if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
+      name: Make coverage filename unique
+    - uses: actions/upload-artifact@v3
+      name: Upload coverage report
+      if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
+      with:
+        name: coverage
+        path: .coverage.${{ matrix.kind }}    
+
   tests:
     name: "Run tests"
     needs: [eval]
@@ -191,10 +203,53 @@ jobs:
       name: Install pytest
     - run: python setup.py pytest
       name: Run tests
+      id: run_tests
       env:
         PYTEST_ADDOPTS: ${{ matrix.opts }}
         COVERAGE_PROCESS_START: 'setup.cfg'
-    # todo: publish test results, coverage info
+    - run: mv .coverage .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
+      # Run whether or not the tests passed, but only if they ran at all
+      if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
+      name: Make coverage filename unique
+    - uses: actions/upload-artifact@v3
+      name: Upload coverage report
+      if: success() || failure() && contains(fromJSON('["success", "failure"]'), steps.run_tests.outcome)
+      with:
+        name: coverage
+        path: .coverage.${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.kind }}
+
+  coverage-report:
+    name: "Coverage report"
+    needs: [tests, notebooks]
+    if: success() || failure()
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      name: Checkout repository
+      with:
+        ref: ${{ env.ref }}
+    - uses: actions/download-artifact@v3
+      name: Get coverage reports
+      with:
+        name: coverage
+        path: coverage
+    - uses: actions/setup-python@v4
+      name: Setup Python
+      with:
+        python-version: 3.8
+    - run: pip install coverage
+      name: Install coverage
+    - run: coverage combine coverage/
+      name: Combine coverage reports
+    - run: coverage report -m --format=markdown > $GITHUB_STEP_SUMMARY
+      name: Generate coverage report
+    - run: coverage html
+      name: Generate coverage html --fail-under=86
+    - uses: actions/upload-artifact@v3
+      name: Upload coverage report
+      with:
+        name: coverage
+        path: htmlcov
       
   build:
     name: Build package

econml/tests/test_notebooks.py

@@ -7,6 +7,7 @@ import html
 import os
 
 _nbdir = os.path.join(os.path.dirname(__file__), '..', '..', 'notebooks')
+_maindir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
 
 _nbsubdirs = ['.', 'CustomerScenarios', 'Solutions']  # TODO: add AutoML notebooks
 
@@ -28,6 +29,15 @@ def test_notebook(file):
     import nbconvert
 
     nb = nbformat.read(os.path.join(_nbdir, file), as_version=4)
+
+    # make sure that coverage outputs reflect notebook contents
+    nb.cells.insert(0, nbformat.v4.new_code_cell(f"""
+    import os, coverage
+    cwd = os.getcwd()
+    os.chdir({_maindir!r}) # change to the root directory, so that setup.cfg is found
+    coverage.process_startup()
+    os.chdir(cwd) # change back to the original directory"""))
+
     # require all cells to complete within 15 minutes, which will help prevent us from
     # creating notebooks that are annoying for our users to actually run themselves
     ep = nbconvert.preprocessors.ExecutePreprocessor(

pyproject.toml

@@ -9,7 +9,7 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
-addopts = "--junitxml=junit/test-results.xml -n auto --strict-markers --cov-config=setup.cfg --cov=econml --cov-report=xml"
+addopts = "--junitxml=junit/test-results.xml -n auto --strict-markers --cov-config=setup.cfg --cov=econml"
 markers = [    
     "slow",
     "notebook",

setup.cfg

@@ -92,9 +92,11 @@ exclude =
 
 ; coverage configuration
 [coverage:run]
-omit = econml/tests/*
 branch = True
 ; need to explicitly add support for multiprocessing for OrthoForest
 concurrency =
     thread
     multiprocessing
+source = econml
+omit = econml/tests/*
+relative_files = True