Full e2e coverage (#95)

.vsts-ci-templates/build_and_test.yml

@@ -1,3 +1,7 @@
+parameters:
+  cmake_args: ''
+  suite_name_suffix: ''
+
 steps:
 - checkout: self
   clean: true
@@ -6,7 +10,7 @@ steps:
 - task: CMake@1
   displayName: CMake
   inputs:
-    cmakeArgs: '-GNinja -DEND_TO_END_COVERAGE=ON ..'
+    cmakeArgs: '-GNinja .. ${{ parameters.cmake_args }}'
 
 - script: ninja
   displayName: Ninja
@@ -17,7 +21,7 @@ steps:
   displayName: CTest
   workingDirectory: build
 
-- script: 'xsltproc --stringparam suiteName "$(Agent.MachineName)" ../tests/infra/ctest_to_junit.xslt Testing/*/Test.xml > JUnit.xml'
+- script: 'xsltproc --stringparam suiteName "$(Agent.MachineName) ${{ parameters.suite_name_suffix }}" ../tests/infra/ctest_to_junit.xslt Testing/*/Test.xml > JUnit.xml'
   displayName: XSLT
   workingDirectory: build
   condition: succeededOrFailed()
@@ -26,15 +30,4 @@ steps:
   inputs:
     testResultsFiles: '**/JUnit.xml'
     publishRunAttachments: true
-  condition: succeededOrFailed()
-
-- script: |
-    ls *_test | xargs -n1 -I '{}' llvm-profdata-7 merge {}.profraw -o {}.profdata
-    ls *_test | xargs -n1 -I '{}' sh -c 'printf "\n*** Binary: {} ***\n "; llvm-cov-7 report -object {} -instr-profile {}.profdata -ignore-filename-regex="(boost|openenclave|3rdparty|/test/)"'
-  displayName: Coverage
-  workingDirectory: build
-
-- script: |
-    CODECOV_TOKEN=$(codecov) ../tests/coverage/generate_coverage.sh
-  displayName: CoverageGeneration
-  workingDirectory: build
+  condition: succeededOrFailed()

.vsts-ci-templates/coverage.yml

@@ -0,0 +1,11 @@
+steps:
+- script: |
+    ls *_test | xargs -n1 -I '{}' llvm-profdata-7 merge {}.profraw -o {}.profdata
+    ls *_test | xargs -n1 -I '{}' sh -c 'printf "\n*** Binary: {} ***\n "; llvm-cov-7 report -object {} -instr-profile {}.profdata -ignore-filename-regex="(boost|openenclave|3rdparty|/test/)"'
+  displayName: Coverage
+  workingDirectory: build
+
+- script: |
+    CODECOV_TOKEN=$(codecov) ../tests/coverage/generate_coverage.sh
+  displayName: CoverageGeneration
+  workingDirectory: build

.vsts-ci.yml

@@ -10,6 +10,15 @@ jobs:
     - template: .vsts-ci-templates/build_and_test.yml
     - template: .vsts-ci-templates/push_perf_data.yml
 
+- job: ACC_1804_virtual_build
+  pool: Ubuntu-1804-SGX-Azure
+  steps:
+    - template: .vsts-ci-templates/build_and_test.yml
+      parameters:
+        cmake_args: '-DVIRTUAL_ONLY=ON -DBUILD_SMALLBANK=OFF'
+        suite_name_suffix: ' Virtual Coverage'
+    - template: .vsts-ci-templates/coverage.yml
+
 - job: static_checks
   pool: Ubuntu-1804-SGX-Azure
   steps:

CMakeLists.txt

@@ -273,6 +273,15 @@ if(BUILD_TESTS)
       NAME quote_verification_test
       COMMAND ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/quote_verification.py . ./libloggingenc.so.signed
     )
+
+    add_test(
+      NAME governance_tests
+      COMMAND
+        ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/governance.py
+        -b .
+        ${CCF_NETWORK_TEST_ARGS}
+        --oesign ${OESIGN}
+    )
   endif()
 
   add_test(
@@ -312,15 +321,6 @@ if(BUILD_TESTS)
       ${RECOVERY_ARGS}
   )
 
-  add_test(
-    NAME governance_tests
-    COMMAND
-      ${PYTHON} ${CMAKE_SOURCE_DIR}/tests/governance.py
-      -b .
-      ${CCF_NETWORK_TEST_ARGS}
-      --oesign ${OESIGN}
-  )
-
   if(BUILD_SMALLBANK)
     include(${CMAKE_CURRENT_SOURCE_DIR}/samples/apps/smallbank/smallbank.cmake)
   endif()

tests/coverage/generate_coverage.sh

@@ -22,9 +22,7 @@ llvm-cov-7 show -instr-profile coverage.profdata "${objects[@]}" -ignore-filenam
 bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f codecov.txt -F unit
 
 for e2e in *.virtual.so; do
-    if [ -f 0_"$e2e".profraw  ]; then
-        llvm-profdata-7 merge -sparse ./*_"$e2e".profraw -o "$e2e".profdata
-        llvm-cov-7 show -instr-profile "$e2e".profdata -object cchost.virtual -object "$e2e" -ignore-filename-regex="(boost|openenclave|3rdparty|/test/)" > "$e2e".txt
-        bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f "$e2e".txt -F "$(echo $"e2e" | cut -d. -f1)"
-    fi
+    llvm-profdata-7 merge -sparse ./*_"$e2e".profraw -o "$e2e".profdata
+    llvm-cov-7 show -instr-profile "$e2e".profdata -object cchost.virtual -object "$e2e" -ignore-filename-regex="(boost|openenclave|3rdparty|/test/)" > "$e2e".txt
+    bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f "$e2e".txt -F "$(echo $"e2e" | cut -d. -f1)"
 done

tests/infra/remote.py

@@ -10,6 +10,7 @@ import getpass
 from contextlib import contextmanager
 import infra.path
 import json
+import uuid
 
 from loguru import logger as LOG
 
@@ -456,7 +457,9 @@ class CCFRemote(object):
         env = {}
         self.profraw = None
         if enclave_type == "virtual":
-            self.profraw = f"{node_id}_{os.path.basename(lib_path)}.profraw"
+            self.profraw = (
+                f"{uuid.uuid4()}-{node_id}_{os.path.basename(lib_path)}.profraw"
+            )
             env["LLVM_PROFILE_FILE"] = self.profraw
 
         oe_log_level = CCF_TO_OE_LOG_LEVEL.get(log_level)