Test Suite on the CI (#541)

.azure-pipelines-templates/matrix.yml

@@ -31,7 +31,16 @@ parameters:
     san:
       cmake_args: '-DSAN=ON'
 
-  perf_test_filter: '"benchmark|perf"'
+  test:
+    NoSGX:
+      ctest_args: '-LE "benchmark|perf|suite"'
+    SGX:
+      ctest_args: '-LE "benchmark|perf"'
+    perf:
+      ctest_args: '-L "benchmark|perf"'
+    san:
+      ctest_args: '-LE "benchmark|perf|suite"'
+
   static_check_job_name: 'Formatting_and_License_Checks'
 
   san_target: 'NoSGX'
@@ -52,7 +61,7 @@ jobs:
           consensus: ${{ consensus }}
           env: ${{ parameters.env[target] }}
           cmake_args: '${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build[target].cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
-          ctest_filter: '-LE ${{ parameters.perf_test_filter }}'
+          ctest_filter: '${{ parameters.test[target].ctest_args }}'
           suffix: ''
           depends_on: ${{ parameters.static_check_job_name }}
 
@@ -64,7 +73,7 @@ jobs:
         consensus: ${{ consensus }}
         env: '${{ parameters.env.NoSGX_SAN }}'
         cmake_args: '${{ parameters.build.common.cmake_args }} ${{ parameters.build.san.cmake_args }} ${{ parameters.build[parameters.san_target].cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
-        ctest_filter: '-LE ${{ parameters.perf_test_filter }}'
+        ctest_filter: '${{ parameters.test.san.ctest_args }}'
         suffix: 'SAN'
         depends_on: ${{ parameters.static_check_job_name }}
 
@@ -76,6 +85,6 @@ jobs:
         consensus: ${{ consensus }}
         env: ${{ parameters.env.SGX }}
         cmake_args: '${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
-        ctest_filter: '-L ${{ parameters.perf_test_filter }}'
+        ctest_filter: '${{ parameters.test.perf.ctest_args }}'
         suffix: 'Perf'
         depends_on: ${{ parameters.static_check_job_name }}

CMakeLists.txt

@@ -374,6 +374,15 @@ if(BUILD_TESTS)
         --schema-dir ${CMAKE_SOURCE_DIR}/sphinx/source/schemas
         )
 
+      add_e2e_test(
+        NAME test_suite
+        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
+        IS_SUITE TRUE
+        ADDITIONAL_ARGS
+        --test-duration 100
+        --enforce-reqs
+      )
+
       if (BUILD_SMALLBANK)
         include(${CMAKE_CURRENT_SOURCE_DIR}/samples/apps/smallbank/smallbank.cmake)
       endif()

cmake/common.cmake

@@ -609,7 +609,7 @@ endfunction()
 function(add_e2e_test)
   cmake_parse_arguments(PARSE_ARGV 0 PARSED_ARGS
     ""
-    "NAME;PYTHON_SCRIPT;"
+    "NAME;PYTHON_SCRIPT;IS_SUITE"
     "ADDITIONAL_ARGS"
   )
 
@@ -630,12 +630,21 @@ function(add_e2e_test)
       PROPERTY
         ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
     )
-    set_property(
-      TEST ${PARSED_ARGS_NAME}
-      APPEND
-      PROPERTY
-        LABELS end_to_end
-    )
+    if (${PARSED_ARGS_IS_SUITE})
+      set_property(
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
+        PROPERTY
+          LABELS suite
+      )
+    else()
+      set_property(
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
+        PROPERTY
+          LABELS end_to_end
+      )
+    endif()
     if (HTTP)
       set_property(
         TEST ${PARSED_ARGS_NAME}