Enable perf regression tests for virtual builds (#4125)

.azure-pipelines-templates/cimetrics.yml

@@ -0,0 +1,14 @@
+jobs:
+  - job: cimetrics_render
+    displayName: "CI Metrics Render"
+    dependsOn: ${{ parameters.depends_on }}
+    condition: eq(dependencies.configure.outputs['setVarStep.docOnly'], 'false')
+
+    ${{ insert }}: ${{ parameters.env }}
+
+    steps:
+      - checkout: self
+        clean: true
+
+      - ${{ if eq(parameters.suffix, 'Perf') }}:
+          - template: metrics.yml

.azure-pipelines-templates/common.yml

@@ -46,7 +46,6 @@ jobs:
 
       - ${{ if eq(parameters.suffix, 'Perf') }}:
           - template: push_perf_data.yml
-          - template: metrics.yml
 
       - ${{ if and(eq(parameters.suffix, 'Release'), eq(parameters.target, 'SGX')) }}:
           - template: cg.yml

.azure-pipelines-templates/matrix.yml

@@ -40,6 +40,8 @@ parameters:
       ctest_args: '-L "benchmark|perf|vegeta"'
     release:
       ctest_args: '-LE "benchmark|perf"'
+    nosgx_perf:
+      ctest_args: '-L "perf|vegeta"'
 
 jobs:
   # Debug
@@ -68,18 +70,28 @@ jobs:
           fetch_v8: release
           depends_on: configure
 
-  # Performance (NoSGX)
-  # NOTE: CI doesn't support SGX and NoSGX perf tests at the same time.
-  # - ${{ if eq(parameters.perf_tests, 'run') }}:
-  #     - template: common.yml
-  #       parameters:
-  #         target: NoSGX
-  #         env: ${{ parameters.env.NoSGX }}
-  #         fetch_v8: release
-  #         cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.v8.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }}"
-  #         suffix: "Perf"
-  #         artifact_name: "NoSGX_Perf"
-  #         ctest_filter: "${{ parameters.test.perf.ctest_args }}"
+  - ${{ if eq(parameters.perf_tests, 'run') }}:
+      - template: common.yml
+        parameters:
+          target: NoSGX
+          env: ${{ parameters.env.NoSGX }}
+          fetch_v8: release
+          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.v8.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }}"
+          suffix: "Perf"
+          artifact_name: "NoSGX_Perf"
+          ctest_filter: "${{ parameters.test.nosgx_perf.ctest_args }}"
+          depends_on: configure
+
+  - ${{ if eq(parameters.perf_tests, 'run') }}:
+      - template: cimetrics.yml
+        parameters:
+          target: NoSGX
+          env: ${{ parameters.env.NoSGX }}
+          suffix: "Perf"
+          depends_on:
+            - configure
+            - SGX_Perf
+            - NoSGX_Perf
 
   # Release
   - ${{ if eq(parameters.perf_or_release, 'release') }}:

cmake/common.cmake

@@ -599,59 +599,66 @@ function(add_perf_test)
     unset(VERIFICATION_ARG)
   endif()
 
-  set(TESTS_SUFFIX "")
-  if("sgx" IN_LIST COMPILE_TARGETS)
-    set(TESTS_SUFFIX "${TESTS_SUFFIX}_sgx")
-  endif()
+  foreach(COMPILE_TARGET ${COMPILE_TARGETS})
+    set(TESTS_SUFFIX "")
+    set(ENCLAVE_TYPE "")
+    if("sgx" STREQUAL ${COMPILE_TARGET})
+      set(TESTS_SUFFIX "${TESTS_SUFFIX}_sgx")
+      set(ENCLAVE_TYPE "release")
+    elseif("virtual" STREQUAL ${COMPILE_TARGET})
+      set(TESTS_SUFFIX "${TESTS_SUFFIX}_virtual")
+      set(ENCLAVE_TYPE "virtual")
+    endif()
 
-  if("cft" STREQUAL ${PARSED_ARGS_CONSENSUS})
-    set(TESTS_SUFFIX "${TESTS_SUFFIX}_cft")
-  elseif("bft" STREQUAL ${PARSED_ARGS_CONSENSUS})
-    set(TESTS_SUFFIX "${TESTS_SUFFIX}_bft")
-  endif()
+    if("cft" STREQUAL ${PARSED_ARGS_CONSENSUS})
+      set(TESTS_SUFFIX "${TESTS_SUFFIX}_cft")
+    elseif("bft" STREQUAL ${PARSED_ARGS_CONSENSUS})
+      set(TESTS_SUFFIX "${TESTS_SUFFIX}_bft")
+    endif()
 
-  set(TEST_NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}")
+    set(TEST_NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}")
 
-  if(PARSED_ARGS_LABEL)
-    set(LABEL_ARG "${TEST_NAME}^")
-  else()
-    set(LABEL_ARG "${TEST_NAME}^")
-  endif()
+    if(PARSED_ARGS_LABEL)
+      set(LABEL_ARG "${TEST_NAME}^")
+    else()
+      set(LABEL_ARG "${TEST_NAME}^")
+    endif()
 
-  string(TOUPPER ${PARSED_ARGS_CONSENSUS} CONSENSUS)
-  add_test(
-    NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}"
-    COMMAND
-      ${PYTHON} ${PARSED_ARGS_PYTHON_SCRIPT} -b . -c ${PARSED_ARGS_CLIENT_BIN}
-      ${CCF_NETWORK_TEST_ARGS} --consensus ${CONSENSUS}
-      ${PARSED_ARGS_CONSTITUTION} --write-tx-times ${VERIFICATION_ARG} --label
-      ${LABEL_ARG} --snapshot-tx-interval 10000 ${PARSED_ARGS_ADDITIONAL_ARGS}
-      ${NODES}
-  )
+    string(TOUPPER ${PARSED_ARGS_CONSENSUS} CONSENSUS)
+    add_test(
+      NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}"
+      COMMAND
+        ${PYTHON} ${PARSED_ARGS_PYTHON_SCRIPT} -b . -c
+        ${PARSED_ARGS_CLIENT_BIN} ${CCF_NETWORK_TEST_ARGS} --consensus
+        ${CONSENSUS} ${PARSED_ARGS_CONSTITUTION} --write-tx-times
+        ${VERIFICATION_ARG} --label ${LABEL_ARG} --snapshot-tx-interval 10000
+        ${PARSED_ARGS_ADDITIONAL_ARGS} -e ${ENCLAVE_TYPE} ${NODES}
+    )
 
-  # Make python test client framework importable
-  set_property(
-    TEST ${TEST_NAME}
-    APPEND
-    PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
-  )
-  if(DEFINED DEFAULT_ENCLAVE_TYPE)
+    # Make python test client framework importable
     set_property(
       TEST ${TEST_NAME}
       APPEND
-      PROPERTY ENVIRONMENT "DEFAULT_ENCLAVE_TYPE=${DEFAULT_ENCLAVE_TYPE}"
+      PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
     )
-  endif()
-  set_property(
-    TEST ${TEST_NAME}
-    APPEND
-    PROPERTY LABELS perf
-  )
-  set_property(
-    TEST ${TEST_NAME}
-    APPEND
-    PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
-  )
+    if(DEFINED DEFAULT_ENCLAVE_TYPE)
+      set_property(
+        TEST ${TEST_NAME}
+        APPEND
+        PROPERTY ENVIRONMENT "DEFAULT_ENCLAVE_TYPE=${DEFAULT_ENCLAVE_TYPE}"
+      )
+    endif()
+    set_property(
+      TEST ${TEST_NAME}
+      APPEND
+      PROPERTY LABELS perf
+    )
+    set_property(
+      TEST ${TEST_NAME}
+      APPEND
+      PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
+    )
+  endforeach()
 endfunction()
 
 # Picobench wrapper

metrics.yml

@@ -10,4 +10,5 @@ monitoring_span: 150
 groups:
   "V8": ".*_v8_.*"
   "SGX": ".*_sgx_.*"
+  "Virtual": ".*_virtual_.*"
   "Others": ".*"