Benchmarks: Build Pipeline - Add nccl-tests as a submodule and add build logic. (#128)

Benchmarks: Build Pipeline - Add nccl-tests as a submodule and add build logic.

.gitmodules

@@ -2,3 +2,6 @@
 	path = third_party/cutlass
 	url = https://github.com/NVIDIA/cutlass.git
 	branch = v2.4.0
+[submodule "third_party/nccl-tests"]
+	path = third_party/nccl-tests
+	url = https://github.com/NVIDIA/nccl-tests.git

dockerfile/cuda11.1.1.dockerfile

@@ -92,11 +92,8 @@ RUN git clone -b v4.5-0.2 https://github.com/linux-rdma/perftest.git /usr/local/
     ./configure CUDA_H_PATH=/usr/local/cuda/include/cuda.h && \
     make -j && \
     make install
-RUN git clone https://github.com/nvidia/nccl-tests /usr/local/nccl-tests && \
-    cd /usr/local/nccl-tests && \
-    make MPI=1 MPI_HOME=/usr/local/mpi/ -j
 
-ENV PATH="/usr/local/nccl-tests/build:${PATH}" \
+ENV PATH="${PATH}" \
     LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" \
     SB_HOME="/opt/superbench" \
     SB_MICRO_PATH="/opt/superbench"

third_party/Makefile

@@ -1,25 +1,29 @@
+
 # Copyright (c) Microsoft Corporation - All rights reserved
 # Licensed under the MIT License
 
 
 SB_MICRO_PATH ?= "/usr/local"
+MPI_HOME_PATH ?= "/usr/local/mpi"
 
-.PHONY: all cutlass bandwidthTest
+.PHONY: all cutlass bandwidthTest nccl_tests
 
 # Build all targets.
-all: cutlass bandwidthTest
+all: cutlass bandwidthTest nccl_tests
 
 # Create $(SB_MICRO_PATH)/bin and $(SB_MICRO_PATH)/lib, no error if existing, make parent directories as needed.
 sb_micro_path:
 	mkdir -p $(SB_MICRO_PATH)/bin
 	mkdir -p $(SB_MICRO_PATH)/lib
+
 # Build cutlass.
-cutlass:
+cutlass: 
 ifneq (,$(wildcard cutlass/CMakeLists.txt))
 	cmake -DCMAKE_INSTALL_BINDIR=$(SB_MICRO_PATH)/bin -DCMAKE_INSTALL_LIBDIR=$(SB_MICRO_PATH)/lib -DCMAKE_BUILD_TYPE=Release \
 		-DCUTLASS_NVCC_ARCHS='70;80' -DCUTLASS_ENABLE_EXAMPLES=OFF -DCUTLASS_ENABLE_TESTS=OFF -S ./cutlass -B ./cutlass/build
 	cmake --build ./cutlass/build -j 8 --target install
 endif
+
 # Build cuda-samples/Samples/bandwidthTest.
 # cuda-samples is released together with CUDA, they have the exact same version. Like v10.0, v11.1 and so on.
 # The version we use is the released tag of cuda-samples which is consistent with the cuda version in the environment or docker.
@@ -29,3 +33,11 @@ bandwidthTest: sb_micro_path
 	git clone -b v$(shell nvcc --version | grep 'release' | awk '{print $$6}' | cut -c2- | cut -d '.' -f1-2) https://github.com/NVIDIA/cuda-samples.git ./cuda-samples
 	cd ./cuda-samples/Samples/bandwidthTest && make clean && make TARGET_ARCH=x86_64 SMS="70 75 80 86"
 	cp -v ./cuda-samples/Samples/bandwidthTest/bandwidthTest $(SB_MICRO_PATH)/bin/
+
+# Build nccl-tests.
+# The version we use is commit 44df0bf from master branch, since it didn't update release tag for long time.
+nccl_tests: sb_micro_path
+ifneq (,$(wildcard nccl-tests/Makefile))
+	cd ./nccl-tests && make MPI=1 MPI_HOME=$(MPI_HOME_PATH) -j
+	cp -v ./nccl-tests/build/* $(SB_MICRO_PATH)/bin/
+endif

third_party/nccl-tests

@@ -0,0 +1 @@
+Subproject commit 44df0bf010dcc95e840ca0fb7466c67cff3f1f0f