Continuous benchmark tests as part of the CI (#1174)

.github/workflows/benchmark.yml

@@ -0,0 +1,72 @@
+name: OpenTelemetry-cpp benchmarks
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run OpenTelemetry-cpp benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Mount Bazel Cache
+        uses: actions/cache@v2
+        env:
+          cache-name: bazel_cache
+        with:
+          path: /home/runner/.cache/bazel
+          key: bazel_benchmark
+      - name: setup
+        run: |
+          sudo ./ci/setup_cmake.sh
+          sudo ./ci/setup_ci_environment.sh
+      - name: Run benchmark
+        id: run_benchmarks
+        run: |
+          ci/do_ci.sh bazel.benchmark
+          mkdir -p benchmarks
+          mv api-benchmark_result.json benchmarks
+          mv sdk-benchmark_result.json benchmarks
+          mv exporters-benchmark_result.json benchmarks
+      - uses: actions/upload-artifact@master
+        with:
+          name: benchmark_results
+          path: benchmarks
+  store_benchmark:
+    needs: benchmark
+    strategy:
+      matrix:
+        components: ["api", "sdk", "exporters"]
+    name: Store benchmark result
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@master
+        with:
+          name: benchmark_results
+          path: benchmarks
+      - name: Print json files
+        id: print_json
+        run: |
+          cat benchmarks/*
+      - name: Push benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: OpenTelemetry-cpp ${{ matrix.components }} Benchmark
+          tool: 'googlecpp'
+          output-file-path: benchmarks/${{ matrix.components }}-benchmark_result.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks

bazel/otel_cc_benchmark.bzl

@@ -24,16 +24,17 @@ def otel_cc_benchmark(name, srcs, deps, tags = [""]):
         srcs = srcs,
         deps = deps + ["@com_github_google_benchmark//:benchmark"],
         tags = tags + ["manual"],
+        defines = ["BAZEL_BUILD"],
     )
 
     # The result of running the benchmark, captured into a text file.
     native.genrule(
         name = name + "_result",
-        outs = [name + "_result.txt"],
+        outs = [name + "_result.json"],
         tools = [":" + name],
         tags = tags + ["benchmark_result", "manual"],
         testonly = True,
-        cmd = "$(location :" + name + (") --benchmark_color=false --benchmark_min_time=.1 &> $@"),
+        cmd = "$(location :" + name + (") --benchmark_format=json --benchmark_color=false --benchmark_min_time=.1 &> $@"),
     )
 
     # This is run as part of "bazel test ..." to smoke-test benchmarks. It's
@@ -44,4 +45,5 @@ def otel_cc_benchmark(name, srcs, deps, tags = [""]):
         deps = deps + ["@com_github_google_benchmark//:benchmark"],
         args = ["--benchmark_min_time=0"],
         tags = tags + ["benchmark"],
+        defines = ["BAZEL_BUILD"],
     )

ci/do_ci.sh

@@ -17,6 +17,42 @@ function install_prometheus_cpp_client
   popd
 }
 
+function run_benchmarks
+{
+  docker run -d --rm -it -p 4317:4317 -p 4318:4318 -v \
+    $(pwd)/examples/otlp:/cfg otel/opentelemetry-collector:0.38.0 \
+    --config=/cfg/opentelemetry-collector-config/config.dev.yaml
+
+  [ -z "${BENCHMARK_DIR}" ] && export BENCHMARK_DIR=$HOME/benchmark
+  mkdir -p $BENCHMARK_DIR
+  bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS -c opt -- \
+    $(bazel query 'attr("tags", "benchmark_result", ...)')
+  echo ""
+  echo "Benchmark results in $BENCHMARK_DIR:"
+  (
+    cd bazel-bin
+    find . -name \*_result.json -exec bash -c \
+      'echo "$@" && mkdir -p "$BENCHMARK_DIR/$(dirname "$@")" && \
+       cp "$@" "$BENCHMARK_DIR/$@" && chmod +w "$BENCHMARK_DIR/$@"' _ {} \;
+  )
+
+  # collect benchmark results into one array
+  pushd $BENCHMARK_DIR
+  components=(api sdk exporters)
+  for component in "${components[@]}"
+  do
+    out=$component-benchmark_result.json
+    find ./$component -type f -name "*_result.json" -exec cat {} \; > $component_tmp_bench.json
+    cat $component_tmp_bench.json | docker run -i --rm itchyny/gojq:0.12.6 -s \
+      '.[0].benchmarks = ([.[].benchmarks] | add) |
+      if .[0].benchmarks == null then null else .[0] end' > $BENCHMARK_DIR/$out
+  done
+
+  mv *benchmark_result.json ${SRC_DIR}
+  popd
+  docker kill $(docker ps -q)
+}
+
 [ -z "${SRC_DIR}" ] && export SRC_DIR="`pwd`"
 [ -z "${BUILD_DIR}" ] && export BUILD_DIR=$HOME/build
 mkdir -p "${BUILD_DIR}"
@@ -124,6 +160,10 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then
   make -j $(nproc)
   cd exporters/otlp && make test
   exit 0
+elif [[ "$1" == "bazel.with_abseil" ]]; then
+  bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS --//api:with_abseil=true //...
+  bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS --//api:with_abseil=true //...
+  exit 0
 elif [[ "$1" == "cmake.test_example_plugin" ]]; then
   # Build the plugin
   cd "${BUILD_DIR}"
@@ -162,9 +202,8 @@ elif [[ "$1" == "bazel.test" ]]; then
   bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS //...
   bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS //...
   exit 0
-elif [[ "$1" == "bazel.with_abseil" ]]; then
-  bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS --//api:with_abseil=true //...
-  bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS --//api:with_abseil=true //...
+elif [[ "$1" == "bazel.benchmark" ]]; then
+  run_benchmarks
   exit 0
 elif [[ "$1" == "bazel.macos.test" ]]; then
   bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_MACOS_OPTIONS //...
@@ -200,7 +239,7 @@ elif [[ "$1" == "benchmark" ]]; then
   echo "Benchmark results in $BENCHMARK_DIR:"
   (
     cd bazel-bin
-    find . -name \*_result.txt -exec bash -c \
+    find . -name \*_result.json -exec bash -c \
       'echo "$@" && mkdir -p "$BENCHMARK_DIR/$(dirname "$@")" && \
        cp "$@" "$BENCHMARK_DIR/$@" && chmod +w "$BENCHMARK_DIR/$@"' _ {} \;
   )

exporters/otlp/BUILD

@@ -279,5 +279,6 @@ otel_cc_benchmark(
     ],
     deps = [
         ":otlp_grpc_exporter",
+        "//examples/common/foo_library:common_foo_library",
     ],
 )

exporters/otlp/test/otlp_grpc_exporter_benchmark.cc

@@ -4,6 +4,22 @@
 #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h"
 #include "opentelemetry/exporters/otlp/otlp_recordable.h"
 
+#include <benchmark/benchmark.h>
+#include "opentelemetry/sdk/trace/simple_processor.h"
+#include "opentelemetry/sdk/trace/tracer_provider.h"
+#include "opentelemetry/trace/provider.h"
+
+#ifdef BAZEL_BUILD
+#  include "examples/common/foo_library/foo_library.h"
+#else
+#  include "foo_library/foo_library.h"
+#endif
+
+namespace trace     = opentelemetry::trace;
+namespace nostd     = opentelemetry::nostd;
+namespace trace_sdk = opentelemetry::sdk::trace;
+namespace otlp      = opentelemetry::exporter::otlp;
+
 #include <benchmark/benchmark.h>
 
 OPENTELEMETRY_BEGIN_NAMESPACE
@@ -171,4 +187,30 @@ BENCHMARK(BM_OtlpExporterDenseSpans);
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE
 
+namespace
+{
+opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts;
+void InitTracer()
+{
+  // Create OTLP exporter instance
+  auto exporter  = std::unique_ptr<trace_sdk::SpanExporter>(new otlp::OtlpGrpcExporter(opts));
+  auto processor = std::unique_ptr<trace_sdk::SpanProcessor>(
+      new trace_sdk::SimpleSpanProcessor(std::move(exporter)));
+  auto provider =
+      nostd::shared_ptr<trace::TracerProvider>(new trace_sdk::TracerProvider(std::move(processor)));
+  // Set the global trace provider
+  trace::Provider::SetTracerProvider(provider);
+}
+
+void BM_otlp_grpc_with_collector(benchmark::State &state)
+{
+  InitTracer();
+  while (state.KeepRunning())
+  {
+    foo_library();
+  }
+}
+BENCHMARK(BM_otlp_grpc_with_collector);
+}  // namespace
+
 BENCHMARK_MAIN();