The rest of the daily build (#1027)

.azure-pipelines-templates/daily-matrix.yml

@@ -6,8 +6,6 @@ parameters:
     SGX:
       container: sgx
       pool: Ubuntu-1804-DC8_v2
-    NoSGX_host:
-      pool: Ubuntu-1804-D16s_v3
 
   build:
     NoSGX:
@@ -28,22 +26,14 @@ jobs:
       cmake_args: '${{ parameters.build.debug.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }}'
       suffix: 'Coverage'
       artifact_name: 'NoSGX_Coverage'
-      ctest_filter: '-LE "perf|tlstest"'
-
-  - template: common.yml
-    parameters:
-      target: NoSGX
-      env: '${{ parameters.env.NoSGX_host }}'
-      cmake_args: '${{ parameters.build.debug.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }}'
-      suffix: 'Host'
-      artifact_name: 'NoSGX_Host'
-      ctest_filter: '-L "tlstest"'
+      ctest_filter: '-LE "perf|suite"'
 
   - template: common.yml
     parameters:
       target: SGX
       env: '${{ parameters.env.SGX }}'
-      cmake_args: '${{ parameters.build.install.cmake_args }} ${{ parameters.build.SGX.cmake_args }}'
+      cmake_args: '${{ parameters.build.install.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DTLS_TEST=ON -DSHUFFLE_SUITE=ON'
       suffix: 'Release'
       artifact_name: 'SGX_Release'
-      ctest_filter: '-LE "benchmark|perf|tlstest"'
+      ctest_filter: '-LE "benchmark|perf"'
+      install_prefix: '${{ parameters.build.install.install_prefix }}'

.azure-pipelines-templates/publish_build.yml

@@ -7,7 +7,12 @@ steps:
   displayName: Compress build artifact
   workingDirectory: build
 
-- task: PublishPipelineArtifact@0
+- task: PublishPipelineArtifact@1
   inputs:
     artifactName: ${{ parameters.artifact_name }}
     targetPath: build/artifact.tar.gz
+
+- task: PublishPipelineArtifact@1
+  inputs:
+    artifactName: '${{ parameters.artifact_name }} TLS report'
+    targetPath: build/tls_report.html

.daily.yml

@@ -1,4 +1,11 @@
-pr: none
+pr:
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - .daily.yml
+      - .azure-pipelines-templates/*
 
 trigger: none
 

CMakeLists.txt

@@ -38,6 +38,7 @@ set(CONSENSUSES raft pbft)
 
 option(BUILD_TESTS "Build tests" ON)
 option(BUILD_UNIT_TESTS "Build unit tests" ON)
+option(TLS_TEST "TLS Test using https://github.com/drwetter/testssl.sh" OFF)
 option(BUILD_SMALLBANK "Build SmallBank sample app and clients" ON)
 
 # Build common library for CCF enclaves
@@ -546,6 +547,15 @@ if(BUILD_TESTS)
     include(${CMAKE_CURRENT_SOURCE_DIR}/samples/apps/smallbank/smallbank.cmake)
   endif()
 
+  if(TLS_TEST)
+    add_custom_target(
+      testssl ALL
+      COMMAND
+        test -d testssl || git clone https://github.com/drwetter/testssl.sh
+        ${CMAKE_CURRENT_BINARY_DIR}/testssl
+    )
+  endif()
+
   foreach(CONSENSUS ${CONSENSUSES})
     add_e2e_test(
       NAME member_client_test_${CONSENSUS}
@@ -567,11 +577,13 @@ if(BUILD_TESTS)
       CONSENSUS ${CONSENSUS}
     )
 
-    add_e2e_test(
-      NAME tlstest_${CONSENSUS}
-      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/tlstest.py
-      CONSENSUS ${CONSENSUS} LABEL tlstest
-    )
+    if(TLS_TEST AND ${CONSENSUS} STREQUAL raft)
+      add_e2e_test(
+        NAME tlstest_${CONSENSUS}
+        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/tlstest.py
+        CONSENSUS ${CONSENSUS} LABEL tlstest
+      )
+    endif()
 
     add_e2e_test(
       NAME schema_tests_${CONSENSUS}

cmake/common.cmake

@@ -71,6 +71,7 @@ option(SAN "Enable Address and Undefined Behavior Sanitizers" OFF)
 option(DISABLE_QUOTE_VERIFICATION "Disable quote verification" OFF)
 option(BUILD_END_TO_END_TESTS "Build end to end tests" ON)
 option(COVERAGE "Enable coverage mapping" OFF)
+option(SHUFFLE_SUITE "Shuffle end to end test suite" OFF)
 
 option(DEBUG_CONFIG "Enable non-production options options to aid debugging"
        OFF
@@ -401,6 +402,13 @@ function(add_e2e_test)
       TEST ${PARSED_ARGS_NAME} APPEND
       PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
     )
+
+    if(SHUFFLE_SUITE)
+      set_property(
+        TEST ${PARSED_ARGS_NAME} APPEND PROPERTY ENVIRONMENT "SHUFFLE_SUITE=1"
+      )
+    endif()
+
     set_property(TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS end_to_end)
     set_property(
       TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS ${PARSED_ARGS_LABEL}

docker/Dockerfile

@@ -7,10 +7,10 @@ RUN mkdir -p /etc/init
 
 COPY getting_started/setup_vm/ /setup_vm/
 RUN apt update \
-    && apt install -y ansible software-properties-common \
+    && apt install -y ansible software-properties-common bsdmainutils dnsutils \
     && cd setup_vm \
     && ansible-playbook ccf-dependencies-no-driver.yml ci-dependencies.yml \
     && rm -rf /tmp/* \
-    && apt remove -y software-properties-common \
+    && apt remove -y ansible software-properties-common \
     && apt -y autoremove \
-    && apt -y clean
+    && apt -y clean

tests/e2e_suite.py

@@ -10,6 +10,8 @@ import time
 import json
 import sys
 from enum import Enum
+import random
+import os
 
 from loguru import logger as LOG
 
@@ -22,6 +24,11 @@ class TestStatus(Enum):
 
 def run(args):
 
+    if os.getenv("SHUFFLE_SUITE"):
+        seed = os.getenv("SHUFFLE_SUITE_SEED", int(time.time()))
+        LOG.info(f"Shuffling suite with seed: {seed}")
+        random.seed(seed)
+        random.shuffle(s.tests)
     s.validate_tests_signature(s.tests)
 
     if args.enforce_reqs is False:

tests/infra/ccf.py

@@ -530,11 +530,14 @@ class Network:
         while time.time() < end_time:
             rekeyed_nodes = []
             for node in self.get_joined_nodes():
-                max_sealed_version = int(
-                    max(node.get_sealed_secrets(), key=lambda x: int(x))
-                )
-                if max_sealed_version >= version:
-                    rekeyed_nodes.append(node)
+                try:
+                    max_sealed_version = int(
+                        max(node.get_sealed_secrets(), key=lambda x: int(x))
+                    )
+                    if max_sealed_version >= version:
+                        rekeyed_nodes.append(node)
+                except IndexError:
+                    pass
             if len(rekeyed_nodes) == len(self.get_joined_nodes()):
                 break
             time.sleep(0.1)

tests/suite/test_suite.py

@@ -5,6 +5,7 @@ import e2e_logging
 import reconfiguration
 import recovery
 import rekey
+import random
 
 from inspect import signature, Parameter
 

tests/tlstest.py

@@ -21,9 +21,8 @@ from loguru import logger as LOG
 def test(network, args, notifications_queue=None):
     node = network.nodes[0]
     endpoint = f"https://{node.host}:{node.rpc_port}"
-    r = subprocess.run(
-        ["docker", "run", "--rm", "-it", "--net=host", "drwetter/testssl.sh", endpoint]
-    )
+    r = subprocess.run(["testssl/testssl.sh", "--outfile", "tls_report", endpoint])
+    assert r.returncode == 0
 
 
 def run(args):