Fix cmake and build issues (#360)

**Description**

Fix cmake and build issues.

**Major Revision**

* Remove unnecessary boost build
* Remove user-agent for mlc
* Remove -j for third party to build each project in sequence
* Fix ansible collections installation path

dockerfile/cuda11.1.1.dockerfile

@@ -103,23 +103,22 @@ RUN cd /tmp && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    wget --user-agent="Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0" https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 ENV PATH="${PATH}" \
     LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN make -j ${NUM_MAKE_JOBS} -C third_party cuda
+RUN make -C third_party cuda
 
 ADD . .
 RUN python3 -m pip install .[nvidia,torch,ort] && \

dockerfile/rocm4.0-pytorch1.7.0.dockerfile

@@ -28,6 +28,7 @@ RUN wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | APT_KEY_DONT_WARN_ON_D
     git \
     jq \
     libaio-dev \
+    libboost-program-options-dev \
     libcap2 \
     libnuma-dev \
     libpci-dev \
@@ -92,23 +93,22 @@ RUN cd /opt && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    wget --user-agent="Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0" https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 ENV PATH="${PATH}" \
     LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN ROCM_VERSION=rocm-4.0.0 make -j -C third_party -o rocm_rocblas rocm
+RUN ROCM_VERSION=rocm-4.0.0 make -C third_party -o rocm_rocblas rocm
 
 # Workaround for image having package installed in user path
 RUN mv /root/.local/bin/* /opt/conda/bin/ && \

dockerfile/rocm4.2-pytorch1.7.0.dockerfile

@@ -28,6 +28,7 @@ RUN wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | APT_KEY_DONT_WARN_ON_D
     git \
     jq \
     libaio-dev \
+    libboost-program-options-dev \
     libcap2 \
     libnuma-dev \
     libpci-dev \
@@ -92,23 +93,22 @@ RUN cd /opt && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    wget --user-agent="Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0" https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 ENV PATH="${PATH}" \
     LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN ROCM_VERSION=rocm-4.2.0 make -j -C third_party rocm
+RUN ROCM_VERSION=rocm-4.2.0 make -C third_party rocm
 
 ADD . .
 RUN python3 -m pip install .[torch,ort] && \

dockerfile/rocm5.0-pytorch1.9.0.dockerfile

@@ -28,6 +28,7 @@ RUN apt-get update && \
     hipify-clang \
     jq \
     libaio-dev \
+    libboost-program-options-dev \
     libcap2 \
     libpci-dev \
     libtinfo5 \
@@ -88,13 +89,10 @@ RUN cd /tmp && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 # Install rccl-rdma-sharp-plugins
 ENV SHARP_VERSION=5.0
@@ -105,13 +103,15 @@ RUN cd /opt/rocm && \
 
 ENV PATH="${PATH}:/opt/rocm/hip/bin/" \
     LD_LIBRARY_PATH="/usr/local/lib/:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN ROCM_VERSION=rocm-5.0.0 make -j ${NUM_MAKE_JOBS} -C third_party rocm
+RUN ROCM_VERSION=rocm-5.0.0 make -C third_party rocm
 
 ADD . .
 RUN python3 -m pip install .[torch,ort]  && \

dockerfile/rocm5.0.1-pytorch1.9.0.dockerfile

@@ -28,6 +28,7 @@ RUN apt-get update && \
     hipify-clang \
     jq \
     libaio-dev \
+    libboost-program-options-dev \
     libcap2 \
     libpci-dev \
     libtinfo5 \
@@ -88,13 +89,10 @@ RUN cd /tmp && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 # Install rccl-rdma-sharp-plugins
 ENV SHARP_VERSION=5.0
@@ -105,13 +103,15 @@ RUN cd /opt/rocm && \
 
 ENV PATH="${PATH}:/opt/rocm/hip/bin/" \
     LD_LIBRARY_PATH="/usr/local/lib/:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN ROCM_VERSION=rocm-5.0.1 make -j ${NUM_MAKE_JOBS} -C third_party rocm
+RUN ROCM_VERSION=rocm-5.0.1 make -C third_party rocm
 
 ADD . .
 RUN python3 -m pip install .[torch,ort]  && \

dockerfile/rocm5.1.1-pytorch1.10.0.dockerfile

@@ -28,6 +28,7 @@ RUN apt-get update && \
     hipify-clang \
     jq \
     libaio-dev \
+    libboost-program-options-dev \
     libcap2 \
     libpci-dev \
     libtinfo5 \
@@ -102,13 +103,10 @@ RUN cd /tmp && \
 
 # Install Intel MLC
 RUN cd /tmp && \
-    mkdir -p mlc && \
-    cd mlc && \
-    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc_v3.9a.tgz && \
-    tar xvf mlc_v3.9a.tgz && \
+    curl https://www.intel.com/content/dam/develop/external/us/en/documents/mlc_v3.9a.tgz -o mlc.tgz && \
+    tar xzvf mlc.tgz Linux/mlc && \
     cp ./Linux/mlc /usr/local/bin/ && \
-    cd /tmp && \
-    rm -rf mlc
+    rm -rf ./Linux mlc.tgz
 
 # Install rccl with commitid 6707a27
 RUN cd /tmp && \
@@ -128,13 +126,15 @@ RUN cd /opt/rocm && \
 
 ENV PATH="${PATH}:/opt/rocm/hip/bin/" \
     LD_LIBRARY_PATH="/usr/local/lib/:${LD_LIBRARY_PATH}" \
-    SB_HOME="/opt/superbench" \
-    SB_MICRO_PATH="/opt/superbench"
+    SB_HOME=/opt/superbench \
+    SB_MICRO_PATH=/opt/superbench \
+    ANSIBLE_DEPRECATION_WARNINGS=FALSE \
+    ANSIBLE_COLLECTIONS_PATH=/usr/share/ansible/collections
 
 WORKDIR ${SB_HOME}
 
 ADD third_party third_party
-RUN ROCM_VERSION=release/rocm-rel-5.1 make -j ${NUM_MAKE_JOBS} -C third_party rocm
+RUN ROCM_VERSION=release/rocm-rel-5.1 make -C third_party rocm
 
 ADD . .
 RUN python3 -m pip install .[torch,ort]  && \

superbench/benchmarks/micro_benchmarks/ib_validation_performance/CMakeLists.txt

@@ -16,13 +16,8 @@ add_executable(ib_validation ib_validation_performance.cc)
 find_package(MPI REQUIRED)
 
 # Boost
-execute_process(COMMAND wget -O boost_1_71_0.tar.gz https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
-execute_process(COMMAND tar xzvf boost_1_71_0.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
-execute_process(COMMAND ./bootstrap.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/boost_1_71_0/)
-execute_process(COMMAND ./b2 --with-program_options install WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/boost_1_71_0/)
-list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/boost_1_71_0/build/)
-
 find_package(Boost REQUIRED COMPONENTS program_options)
+
 include_directories(${Boost_INCLUDE_DIRS})
 target_link_libraries(ib_validation PUBLIC MPI::MPI_CXX ${Boost_LIBRARIES})
 install(TARGETS ib_validation RUNTIME DESTINATION bin)

third_party/Makefile

@@ -1,4 +1,3 @@
-
 # Copyright (c) Microsoft Corporation - All rights reserved
 # Licensed under the MIT License
 
@@ -28,7 +27,7 @@ cuda_cutlass:
 ifneq (,$(wildcard cutlass/CMakeLists.txt))
 	cmake -DCMAKE_INSTALL_BINDIR=$(SB_MICRO_PATH)/bin -DCMAKE_INSTALL_LIBDIR=$(SB_MICRO_PATH)/lib -DCMAKE_BUILD_TYPE=Release \
 		-DCUTLASS_NVCC_ARCHS='70;75;80;86' -DCUTLASS_ENABLE_EXAMPLES=OFF -DCUTLASS_ENABLE_TESTS=OFF -S ./cutlass -B ./cutlass/build
-	cmake --build ./cutlass/build -j 8 --target install
+	cmake --build ./cutlass/build -j $(shell nproc --ignore=2) --target install
 endif
 
 # Build cuda-samples/Samples/bandwidthTest.