This commit is contained in:
Amaury Chamayou 2019-10-13 06:58:20 +01:00 коммит произвёл GitHub
Родитель 8ab167f386
Коммит 2306bd8d4f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 157 добавлений и 353 удалений

Просмотреть файл

@ -1,161 +0,0 @@
trigger:
batch: true
branches:
include:
- "master"
- "ci/*"
paths:
exclude:
- 'README.md'
- 'CCF-TECHNICAL-REPORT.pdf'
- 'Dockerfile'
- 'Doxyfile'
- 'THIRD_PARTY_NOTICES.txt'
- 'getting_started/'
- 'sphinx/'
- '.github/'
- '.azure-pipelines-gh-pages.yml'
- 'LICENSE'
pr:
autoCancel: true
branches:
include:
- master
paths:
include:
- '*'
exclude:
- 'README.md'
- 'CCF-TECHNICAL-REPORT.pdf'
- 'Dockerfile'
- 'Doxyfile'
- 'THIRD_PARTY_NOTICES.txt'
- 'getting_started/'
- 'sphinx/'
- '.github/'
- '.azure-pipelines-gh-pages.yml'
- 'LICENSE'
schedules:
- cron: "0 3 * * Mon-Fri"
displayName: Daily morning build
branches:
include:
- master
always: true
jobs:
- job: Coverage
pool: Ubuntu-1804-D8s_v3
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-nosgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all -v $(log.path):$(log.path)
steps:
- checkout: self
clean: true
submodules: true
# Container initialization is expensive, so don't start separate container
# for static_checks. Piggy back them here
- script: find . -regex ".*\.sh$" | xargs shellcheck -s bash -e SC2044,SC2002,SC1091
displayName: 'Shell Check'
- script: python3.7 notice-check.py
displayName: 'Check CCF copyright notices'
- script: python3.7 notice-check.py
workingDirectory: ePBFT
displayName: 'Check ePBFT copyright notices'
- script: ./check-format.sh src samples ePBFT/src
displayName: 'Check C++ code format'
- script: |
python3.7 -m venv env
source env/bin/activate
pip install black
black --check sphinx/ tests/ notice-check.py
displayName: 'Check Python code format'
# Actual coverage build steps starts here
- template: .azure-pipelines-templates/build.yml
parameters:
cmake_args: '-DTARGET=virtual -DBUILD_SMALLBANK=OFF -DCOVERAGE=ON'
- template: .azure-pipelines-templates/test.yml
parameters:
suite_name_suffix: 'coverage'
- script: ../tests/coverage/generate_coverage.sh
displayName: 'Push coverage'
workingDirectory: build
env:
CODECOV_TOKEN: $(codecov.token)
- job: SAN
pool: Ubuntu-1804-D8s_v3
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-nosgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --cap-add SYS_PTRACE enables ptrace and thus LeakSanitizer
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --cap-add SYS_PTRACE -v $(log.path):$(log.path)
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/build.yml
parameters:
cmake_args: '-DTARGET=virtual -DBUILD_SMALLBANK=OFF -DSAN=ON'
- template: .azure-pipelines-templates/test.yml
parameters:
suite_name_suffix: 'san'
- job: ePBFT
pool: Ubuntu-1804-D8s_v3
container:
image: ccfciteam/ccf-ci-18.04-oe-0.6:latest
# --publish-all enables end-to-end tests to communicate over ports
# --cap-add SYS_PTRACE enables ptrace and thus LeakSanitizer
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --cap-add SYS_PTRACE -v $(log.path):$(log.path)
steps:
- checkout: self
clean: true
submodules: true
- script: mkdir build
displayName: 'Make a build directory'
- script: cmake -DTARGET=virtual -GNinja -DPBFT=ON -DBUILD_SMALLBANK=OFF ..
displayName: cmake
workingDirectory: build
- script: ninja
displayName: Ninja
workingDirectory: build
# TODO: The -E argument should be removed once this test passes consistently.
# Current investigation is in PR #424
- script: |
./tests.sh -VV --timeout 240 --no-compress-output -T Test -E test_TCP_with_delay
displayName: CTest
workingDirectory: build
- script: 'xsltproc --stringparam suiteName "$(Agent.MachineName) SGX" ../tests/infra/ctest_to_junit.xslt Testing/*/Test.xml > JUnit.xml'
displayName: XSLT
workingDirectory: build
condition: succeededOrFailed()
- task: PublishTestResults@2
inputs:
testResultsFiles: '**/JUnit.xml'
publishRunAttachments: true
condition: succeededOrFailed()

Просмотреть файл

@ -0,0 +1,30 @@
jobs:
- job: Formatting_and_License_Checks
displayName: 'Formatting and License Checks'
${{ insert }}: ${{ parameters.env }}
steps:
- checkout: self
clean: true
submodules: true
- script: find . -regex ".*\.sh$" | xargs shellcheck -s bash -e SC2044,SC2002,SC1091
displayName: 'Shell Check'
- script: python3.7 notice-check.py
displayName: 'Check CCF copyright notices'
- script: python3.7 notice-check.py
workingDirectory: ePBFT
displayName: 'Check ePBFT copyright notices'
- script: ./check-format.sh src samples ePBFT/src
displayName: 'Check C++ code format'
- script: |
python3.7 -m venv env
source env/bin/activate
pip install black
black --check sphinx/ tests/ notice-check.py
displayName: 'Check Python code format'

Просмотреть файл

@ -0,0 +1,25 @@
jobs:
- job: BuildAndTest_${{ parameters.target }}_${{ parameters.consensus }}_${{ parameters.suffix }}
displayName: 'Build and Test ${{ parameters.target }} ${{ parameters.consensus }} ${{ parameters.suffix }}'
${{ insert }}: ${{ parameters.env }}
steps:
- checkout: self
clean: true
submodules: true
- template: build.yml
parameters:
cmake_args: '${{ parameters.cmake_args }}'
- template: test.yml
parameters:
ctest_filter: ${{ parameters.ctest_filter }}
suite_name_suffix: '${{ parameters.target }} ${{ parameters.consensus }} ${{ parameters.suffix }} tests'
- ${{ if and(eq(parameters.target, 'NoSGX'), eq(parameters.suffix, '')) }}:
- template: coverage.yml
parameters:
consensus: ${{ parameters.consensus }}
- ${{ if and(eq(parameters.suffix, 'Perf'), eq(parameters.consensus, 'CFT')) }}:
- template: push_perf_data.yml
parameters:
consensus: ${{ parameters.consensus }}

Просмотреть файл

@ -0,0 +1,6 @@
steps:
- script: ../tests/coverage/generate_coverage.sh ${{ parameters.consensus }}
displayName: 'Push coverage'
workingDirectory: build
env:
CODECOV_TOKEN: $(codecov.token)

Просмотреть файл

@ -0,0 +1,69 @@
parameters:
consensus: ['CFT', 'BFT']
target: ['NoSGX', 'SGX']
env:
NoSGX:
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-nosgx:latest
options: --publish-all --cap-add SYS_PTRACE
pool: Ubuntu-1804-D8s_v3
SGX:
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
options: --publish-all --device /dev/sgx:/dev/sgx
pool: Ubuntu-1804-DC4s
build:
NoSGX:
cmake_args: '-DTARGET=virtual -DCOVERAGE=ON'
SGX:
cmake_args: ''
BFT:
cmake_args: '-DPBFT=ON'
CFT:
cmake_args: ''
debug:
cmake_args: '-DCMAKE_BUILD_TYPE=Debug -DBUILD_SMALLBANK=OFF'
perf:
cmake_args: '-DSERVICE_IDENTITY_CURVE_CHOICE=secp256k1_bitcoin'
san:
cmake_args: '-DSAN=ON'
perf_test_filter: '"benchmark|perf"'
jobs:
- template: checks.yml
parameters:
env: ${{ parameters.env.NoSGX }}
- ${{ each target in parameters.target }}:
- ${{ each consensus in parameters.consensus }}:
- template: common.yml
parameters:
target: ${{ target }}
consensus: ${{ consensus }}
env: ${{ parameters.env[target] }}
cmake_args: '${{ parameters.build.debug.cmake_args }} ${{ parameters.build[target].cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
ctest_filter: '-LE ${{ parameters.perf_test_filter }}'
suffix: ''
- ${{ each consensus in parameters.consensus }}:
- template: common.yml
parameters:
target: NoSGX
consensus: ${{ consensus }}
env: ${{ parameters.env.NoSGX }}
cmake_args: '${{ parameters.build.san.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
ctest_filter: '-LE ${{ parameters.perf_test_filter }}'
suffix: 'SAN'
- ${{ each consensus in parameters.consensus }}:
- template: common.yml
parameters:
target: SGX
consensus: ${{ consensus }}
env: ${{ parameters.env.SGX }}
cmake_args: '${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }} ${{ parameters.build[consensus].cmake_args }}'
ctest_filter: '-L ${{ parameters.perf_test_filter }}'
suffix: 'Perf'

Просмотреть файл

@ -1,9 +1,10 @@
parameters:
artifact_name: ''
steps:
- script: |
tar -cavf artifact.tar.gz cchost CTestTestfile.cmake json_schema keygenerator merkle_mem raft_driver tests.sh *_test *_bench *client *.so.signed
tar -cavf artifact.tar.gz `find . -maxdepth 1 -type f | grep -v CMakeCache.txt`
displayName: Compress build artifact
workingDirectory: build

Просмотреть файл

@ -3,12 +3,10 @@ trigger:
branches:
include:
- "master"
- "ci/*"
paths:
exclude:
- 'README.md'
- 'CCF-TECHNICAL-REPORT.pdf'
- 'Dockerfile'
- 'Doxyfile'
- 'THIRD_PARTY_NOTICES.txt'
- 'getting_started/'
@ -28,7 +26,6 @@ pr:
exclude:
- 'README.md'
- 'CCF-TECHNICAL-REPORT.pdf'
- 'Dockerfile'
- 'Doxyfile'
- 'THIRD_PARTY_NOTICES.txt'
- 'getting_started/'
@ -46,171 +43,4 @@ schedules:
always: true
jobs:
- job: ACC_1804_SGX_build
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/build.yml
parameters:
cmake_args: '-DBUILD_SMALLBANK=OFF'
- template: .azure-pipelines-templates/publish_build.yml
parameters:
artifact_name: build_results
- job: ACC_1804_SGX_quick_tests
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
dependsOn:
- ACC_1804_SGX_build
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/download_build.yml
parameters:
artifact_name: build_results
- template: .azure-pipelines-templates/test.yml
parameters:
ctest_filter: '-LE "perf|end_to_end"'
suite_name_suffix: ' SGX quick tests'
- job: ACC_1804_SGX_e2e_tests_A
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
dependsOn:
- ACC_1804_SGX_build
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/download_build.yml
parameters:
artifact_name: build_results
- template: .azure-pipelines-templates/test.yml
parameters:
ctest_filter: '-L end_to_end -I 0,,2'
suite_name_suffix: ' SGX end to end tests A'
- job: ACC_1804_SGX_e2e_tests_B
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
dependsOn:
- ACC_1804_SGX_build
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/download_build.yml
parameters:
artifact_name: build_results
- template: .azure-pipelines-templates/test.yml
parameters:
ctest_filter: '-L end_to_end -I 1,,2'
suite_name_suffix: ' SGX end to end tests B'
- job: ACC_1804_SGX_perf_build
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/build.yml
parameters:
cmake_args: '-DBUILD_SMALLBANK=ON -DSERVICE_IDENTITY_CURVE_CHOICE=secp256k1_bitcoin'
- template: .azure-pipelines-templates/publish_build.yml
parameters:
artifact_name: perf_build_results
- job: ACC_1804_SGX_perf_tests
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
dependsOn:
- ACC_1804_SGX_perf_build
steps:
- checkout: self
clean: true
submodules: true
- template: .azure-pipelines-templates/download_build.yml
parameters:
artifact_name: perf_build_results
- template: .azure-pipelines-templates/test.yml
parameters:
suite_name_suffix: ' SGX Performance'
ctest_filter: '-L perf'
- template: .azure-pipelines-templates/push_perf_data.yml
- job: ePBFT
pool: Ubuntu-1804-DC4s
container:
image: ccfciteam/ccf-ci-18.04-oe-0.7.0-rc1-sgx:latest
# --publish-all enables end-to-end tests to communicate over ports
# --device /dev/sgx:/dev/sgx makes sgx available to the container
# -v $(log.path):$(log.path) mounts the local directory to the same path in the container
options: --publish-all --device /dev/sgx:/dev/sgx -v $(log.path):$(log.path)
dependsOn:
- ACC_1804_SGX_build
steps:
- checkout: self
clean: true
submodules: true
- script: mkdir build
displayName: 'Make a build directory'
- script: cmake -GNinja -DPBFT=ON -DBUILD_SMALLBANK=OFF ..
displayName: cmake
workingDirectory: build
- script: ninja
displayName: Ninja
workingDirectory: build
- script: |
./tests.sh -VV --timeout 240 --no-compress-output -T Test
displayName: CTest
workingDirectory: build
- script: 'xsltproc --stringparam suiteName "$(Agent.MachineName) SGX" ../tests/infra/ctest_to_junit.xslt Testing/*/Test.xml > JUnit.xml'
displayName: XSLT
workingDirectory: build
condition: succeededOrFailed()
- task: PublishTestResults@2
inputs:
testResultsFiles: '**/JUnit.xml'
publishRunAttachments: true
condition: succeededOrFailed()
- template: .azure-pipelines-templates/matrix.yml

Просмотреть файл

@ -68,21 +68,23 @@ if(BUILD_TESTS)
target_link_libraries(ds_test PRIVATE
${CMAKE_THREAD_LIBS_INIT})
add_unit_test(raft_test
${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/raft/test/main.cpp)
target_link_libraries(raft_test PRIVATE
${CRYPTO_LIBRARY})
add_unit_test(ledger_test
${CMAKE_CURRENT_SOURCE_DIR}/src/host/test/ledger.cpp)
add_unit_test(raft_enclave_test
${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/raft/test/enclave.cpp)
target_include_directories(raft_enclave_test PRIVATE
${CCFCRYPTO_INC})
target_link_libraries(raft_enclave_test PRIVATE
${CRYPTO_LIBRARY}
secp256k1.host)
if (NOT PBFT)
add_unit_test(raft_test
${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/raft/test/main.cpp)
target_link_libraries(raft_test PRIVATE
${CRYPTO_LIBRARY})
add_unit_test(raft_enclave_test
${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/raft/test/enclave.cpp)
target_include_directories(raft_enclave_test PRIVATE
${CCFCRYPTO_INC})
target_link_libraries(raft_enclave_test PRIVATE
${CRYPTO_LIBRARY}
secp256k1.host)
endif()
add_unit_test(crypto_test
${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/test/crypto.cpp)

Просмотреть файл

@ -1,8 +1,7 @@
| | Link |
|- |- |
| Community Chat | [![Gitter](https://badges.gitter.im/MSRC-CCF/community.svg)](https://gitter.im/MSRC-CCF/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) |
| SGX CI | [![Build Status](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20Github%20CI?branchName=master)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=3&branchName=master) |
| Non-SGX CI | [![Build Status](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20Github%20CI%20-%20No%20SGX?branchName=master)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=2&branchName=master) |
| Continuous Integration | [![Build Status](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20Github%20CI?branchName=master)](https://dev.azure.com/MSRC-CCF/CCF/_build/latest?definitionId=3&branchName=master) |
| Code coverage | [![codecov](https://codecov.io/gh/microsoft/CCF/branch/master/graph/badge.svg)](https://codecov.io/gh/microsoft/CCF) |
| Documentation | [![docs](https://dev.azure.com/MSRC-CCF/CCF/_apis/build/status/CCF%20GitHub%20Pages?branchName=master)](https://microsoft.github.io/CCF/) |

Просмотреть файл

@ -383,7 +383,6 @@ namespace raft
{
if (timeout_elapsed >= request_timeout)
{
LOG_TRACE_FMT("Sending periodic updates to followers");
using namespace std::chrono_literals;
timeout_elapsed = 0ms;

Просмотреть файл

@ -214,7 +214,7 @@ namespace logger
static inline Level& level()
{
static Level the_level =
#if !defined(NDEBUG) || defined(VERBOSE_LOGGING)
#ifdef VERBOSE_LOGGING
Level::TRACE
#else
Level::INFO

Просмотреть файл

@ -4,6 +4,8 @@
set -ex
suffix=$1
objects=()
for f in *_test; do
objects+=( -object "$f")
@ -19,10 +21,12 @@ llvm-cov-7 export -instr-profile coverage.profdata -format=text "${objects[@]}"
# Generate and upload combined coverage report for Codecov
llvm-cov-7 show -instr-profile coverage.profdata "${objects[@]}" -ignore-filename-regex="(openenclave|3rdparty|/test/)" > codecov.txt
bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f codecov.txt -F unit
bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f codecov.txt -F "unit_${suffix}"
for e2e in *.virtual.so; do
llvm-profdata-7 merge -sparse ./*_"$e2e".profraw -o "$e2e".profdata
llvm-cov-7 show -instr-profile "$e2e".profdata -object cchost.virtual -object "$e2e" -ignore-filename-regex="(openenclave|3rdparty|/test/)" > "$e2e".txt
bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f "$e2e".txt -F "$(echo $"e2e" | cut -d. -f1)"
if compgen -G "./*_$e2e.profraw"; then
llvm-profdata-7 merge -sparse ./*_"$e2e".profraw -o "$e2e".profdata
llvm-cov-7 show -instr-profile "$e2e".profdata -object cchost.virtual -object "$e2e" -ignore-filename-regex="(openenclave|3rdparty|/test/)" > "$e2e".txt
bash <(curl -s https://codecov.io/bash) -t "${CODECOV_TOKEN}" -f "$e2e".txt -F "$(echo $"e2e" | cut -d. -f1)_$suffix"
fi
done