Install Boost at /usr/local/boost-x.y.z on Docker

Build unit tests only when Boost is available Configure Boost path in configure Address CR comments fix #678 This is a combination of 10 commits. Adapt makefile to work with Docker container on Linux This is a combination of 7 commits. enable unit tests build remove -liomp5 from math unit test enable openmp add -ldl fix space error add -dl for reader test change order of -l remove gdk installation use /usr/local/lib as boost library path Adapt path in unit tests on Linux adapt path for Linux adapt path on other test projects for Linux remove extra blank line add BOOST_PATH,and build unit tests based on BOOST_PATH configure boost path install Boost 1.60.0; use version variable instead of hardcoding. add comments for different paths on Linux than on Windows. restore installation of gdk, because the removal of gdk needs more changes and will be done in a separate check-in use ifdef, fix typos
2016-07-19 09:34:31 +02:00 · 2016-07-19 09:34:31 +02:00 · 3c3862a20a
--- a/44
+++ b/44
@ -29,10 +29,12 @@
 #     If not specified, CNTK will be be built without cuDNN.
 #   KALDI_PATH= Path to Kaldi
 #     If not specified, Kaldi plugins will not be built
-#   OPENCV_PATH= path to OpenCV 3.0.0 installation, so $(OPENCV_PATH) exists
-#     defaults to /usr/local/opencv-3.0.0
+#   OPENCV_PATH= path to OpenCV 3.1.0 installation, so $(OPENCV_PATH) exists
+#     defaults to /usr/local/opencv-3.1.0
 #   LIBZIP_PATH= path to libzip installation, so $(LIBZIP_PATH) exists
 #     defaults to /usr/local/
+#   BOOST_PATH= path to Boost installation, so $(BOOST_PATH)/include/boost/test/unit_test.hpp
+#     defaults to /usr/local/boost-1.60.0
 # These can be overridden on the command line, e.g. make BUILDTYPE=debug

 # TODO: Build static libraries for common dependencies that are shared by multiple 
@ -806,9 +808,12 @@ $(CNTK_CORE_BS): $(SOURCEDIR)/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs
 # Unit Tests
 ########################################

-# use system pre-installed Boost libraries
-# Todo: use our own version of boost libraries 
-BOOSTLIB_PATH = /usr/lib/x86_64-linux-gnu
+# only build unit tests when Boost is available
+ifdef BOOST_PATH
+
+INCLUDEPATH += $(BOOST_PATH)/include
+
+BOOSTLIB_PATH = $(BOOST_PATH)/lib
 BOOSTLIBS := boost_unit_test_framework boost_filesystem boost_system

 UNITTEST_EVAL_SRC = \
@ -818,9 +823,9 @@ UNITTEST_EVAL_SRC = \
 UNITTEST_EVAL_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_EVAL_SRC))

 UNITTEST_EVAL := $(BINDIR)/evaltests
-# Temporarily not build unit tests as the docker image does not include boost.
-#ALL += $(UNITTEST_EVAL)
-#SRC += $(UNITTEST_EVAL_SRC)
+
+ALL += $(UNITTEST_EVAL)
+SRC += $(UNITTEST_EVAL_SRC)

 $(UNITTEST_EVAL) : $(UNITTEST_EVAL_OBJ) | $(EVAL_LIB) $(CNTKMATH_LIB)
 	@echo $(SEPARATOR)
@ -844,15 +849,15 @@ UNITTEST_READER_SRC = \
 UNITTEST_READER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_READER_SRC))

 UNITTEST_READER := $(BINDIR)/readertests
-# Temporarily not build unit tests as the docker image does not include boost.
-#ALL += $(UNITTEST_READER)
-#SRC += $(UNITTEST_READER_SRC)
+
+ALL += $(UNITTEST_READER)
+SRC += $(UNITTEST_READER_SRC)

 $(UNITTEST_READER): $(UNITTEST_READER_OBJ) | $(HTKMLFREADER) $(HTKDESERIALIZERS) $(UCIFASTREADER) $(COMPOSITEDATAREADER) $(IMAGEREADER) $(CNTKMATH_LIB)
 	@echo $(SEPARATOR)
 	@mkdir -p $(dir $@)
 	@echo building $@ for $(ARCH) with build type $(BUILDTYPE)
-	$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(LIBDIR) $(BOOSTLIB_PATH)) -o $@ $^ $(patsubst %, -l%, $(BOOSTLIBS))  -l$(CNTKMATH) 
+	$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(LIBDIR) $(BOOSTLIB_PATH)) -o $@ $^ $(patsubst %, -l%, $(BOOSTLIBS)) -l$(CNTKMATH) -ldl 

 UNITTEST_NETWORK_SRC = \
 	$(SOURCEDIR)/../Tests/UnitTests/NetworkTests/OperatorEvaluation.cpp \
@ -878,9 +883,9 @@ UNITTEST_NETWORK_SRC += $(SGDLIB_SRC)
 UNITTEST_NETWORK_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_NETWORK_SRC)))

 UNITTEST_NETWORK := $(BINDIR)/networktests
-# Temporarily not build unit tests as the docker image does not include boost.
-#ALL += $(UNITTEST_NETWORK)
-#SRC += $(UNITTEST_NETWORK_SRC)
+
+ALL += $(UNITTEST_NETWORK)
+SRC += $(UNITTEST_NETWORK_SRC)

 $(UNITTEST_NETWORK): $(UNITTEST_NETWORK_OBJ) | $(CNTKMATH_LIB) $(CNTKTEXTFORMATREADER)
 	@echo $(SEPARATOR)
@ -910,18 +915,19 @@ UNITTEST_MATH_SRC = \
 UNITTEST_MATH_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_MATH_SRC))

 UNITTEST_MATH := $(BINDIR)/mathtests
-# Temporarily not build unit tests as the docker image does not include boost.
-#ALL += $(UNITTEST_MATH)
-#SRC += $(UNITTEST_MATH_SRC)
+
+ALL += $(UNITTEST_MATH)
+SRC += $(UNITTEST_MATH_SRC)

 $(UNITTEST_MATH): $(UNITTEST_MATH_OBJ) | $(CNTKMATH_LIB) 
 	@echo $(SEPARATOR)
 	@mkdir -p $(dir $@)
 	@echo building $@ for $(ARCH) with build type $(BUILDTYPE)
-	$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(LIBDIR) $(LIBPATH) $(BOOSTLIB_PATH)) -o $@ $^ $(patsubst %, -l%, $(BOOSTLIBS)) $(LIBS) -l$(CNTKMATH) -liomp5
+	$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(LIBDIR) $(LIBPATH) $(BOOSTLIB_PATH)) -o $@ $^ $(patsubst %, -l%, $(BOOSTLIBS)) $(LIBS) -l$(CNTKMATH) -ldl -fopenmp

 unittests: $(UNITTEST_EVAL) $(UNITTEST_READER) $(UNITTEST_NETWORK) $(UNITTEST_MATH)

+endif

 ########################################
 # General compile and dependency rules
--- a/Tests/UnitTests/EvalTests/EvalTestHelper.h
+++ b/Tests/UnitTests/EvalTests/EvalTestHelper.h
@ -27,7 +27,13 @@ struct EvalFixture
        m_parentPath = boost::filesystem::canonical(path.parent_path()).generic_string();
        fprintf(stderr, "Executable path: %s\n", m_parentPath.c_str());

+#ifdef _WIN32
+	// The executable path on Windows is e.g. <cntk>/x64/Debug/Unittests/
        m_testDataPath = m_parentPath + "/../../../Tests/UnitTests/EvalTests";
+#else
+	// The executable path on Linux is e.g. <cntk>/build/cpu/release/bin/
+        m_testDataPath = m_parentPath + "/../../../../Tests/UnitTests/EvalTests";
+#endif
        boost::filesystem::path absTestPath(m_testDataPath);
        absTestPath = boost::filesystem::canonical(absTestPath);
        m_testDataPath = absTestPath.generic_string();
@ -112,4 +118,4 @@ struct EvalFixture
        return boost::filesystem::current_path().generic_string();
    }
 };
-}}}}
+}}}}
--- a/Tests/UnitTests/NetworkTests/Common/NetworkTestHelper.h
+++ b/Tests/UnitTests/NetworkTests/Common/NetworkTestHelper.h
@ -34,7 +34,13 @@ struct DataFixture
        m_parentPath = boost::filesystem::canonical(path.parent_path()).generic_string();
        fprintf(stderr, "Executable path: %s\n", m_parentPath.c_str());

+#ifdef _WIN32
+	// The executable path on Windows is e.g. <cntk>/x64/Debug/Unittests/
        m_testDataPath = m_parentPath + "/../../../Tests/UnitTests/NetworkTests";
+#else
+	// The executable path on Linux is e.g. <cntk>/build/cpu/release/bin/
+        m_testDataPath = m_parentPath + "/../../../../Tests/UnitTests/NetworkTests";
+#endif
        boost::filesystem::path absTestPath(m_testDataPath);
        absTestPath = boost::filesystem::canonical(absTestPath);
        m_testDataPath = absTestPath.generic_string();
--- a/Tests/UnitTests/ReaderTests/Common/ReaderTestHelper.h
+++ b/Tests/UnitTests/ReaderTests/Common/ReaderTestHelper.h
@ -28,7 +28,13 @@ struct ReaderFixture
        m_parentPath = boost::filesystem::canonical(path.parent_path()).generic_string();
        fprintf(stderr, "Executable path: %s\n", m_parentPath.c_str());

+#ifdef _WIN32
+	// The executable path on Windows is e.g. <cntk>/x64/Debug/Unittests/
        m_testDataPath = m_parentPath + "/../../../Tests/UnitTests/ReaderTests";
+#else
+	// The executable path on Linux is e.g. <cntk>/build/cpu/release/bin/
+        m_testDataPath = m_parentPath + "/../../../../Tests/UnitTests/ReaderTests";
+#endif
        boost::filesystem::path absTestPath(m_testDataPath);
        absTestPath = boost::filesystem::canonical(absTestPath);
        m_testDataPath = absTestPath.generic_string();
--- a/Tools/docker/Dockerfile-CPU
+++ b/Tools/docker/Dockerfile-CPU
@ -63,14 +63,14 @@ RUN OPENBLAS_VERSION=0.2.18 && \
    rm -rf /OpenBLAS-${OPENBLAS_VERSION}

 # Install Boost
-RUN BOOST_VERSION=1_59_0 && \
-    wget -q -O - https://sourceforge.net/projects/boost/files/boost/1.59.0/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
+RUN BOOST_VERSION=1_60_0 && \
+    BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
+    wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
    cd boost_${BOOST_VERSION} && \
-    ./bootstrap.sh --prefix=/usr/local --with-libraries=filesystem,system,test  && \
-    ./b2 install && \
+    ./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test  && \
+    ./b2 -d0 -j"$(nproc)" install  && \
    rm -rf /boost_${BOOST_VERSION}

-
 # Install CNTK custom MKL, version 1
 RUN mkdir /usr/local/CNTKCustomMKL && \
    wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-1.tgz | \
--- a/Tools/docker/Dockerfile-GPU
+++ b/Tools/docker/Dockerfile-GPU
@ -59,11 +59,12 @@ RUN OPENBLAS_VERSION=0.2.18 && \
    rm -rf /OpenBLAS-${OPENBLAS_VERSION}

 # Install Boost
-RUN BOOST_VERSION=1_59_0 && \
-    wget -q -O - https://sourceforge.net/projects/boost/files/boost/1.59.0/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
+RUN BOOST_VERSION=1_60_0 && \
+    BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
+    wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
    cd boost_${BOOST_VERSION} && \
-    ./bootstrap.sh --prefix=/usr/local --with-libraries=filesystem,system,test  && \
-    ./b2 install && \
+    ./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test  && \
+    ./b2 -d0 -j"$(nproc)" install  && \
    rm -rf /boost_${BOOST_VERSION}

 # Install CNTK custom MKL, version 1
--- a/47
+++ b/47
@ -31,6 +31,10 @@ have_openblas=no
 openblas_path=
 openblas_check=include/openblas_config.h

+# Boost library
+boost_path=
+boost_check=include/boost/test/unit_test.hpp
+
 have_kaldi=no
 kaldi_path=
 kaldi_check=src/kaldi.mk
@ -75,6 +79,8 @@ default_acmls="acml5.3.1/ifort64_mp"
 default_mkls="CNTKCustomMKL"
 default_openblas=""

+default_boost="boost-1.60.0"
+
 # NOTE: Will get compilation errors with cuda-6.0
 default_cudas="cuda-7.5 cuda-7.0 cuda-6.5"
 default_kaldis="kaldi-trunk kaldi-c024e8aa"
@ -135,6 +141,11 @@ function find_openblas ()
    find_dir "$default_openblas" "$openblas_check"
 }

+function find_boost ()
+{
+    find_dir "$default_boost" "$boost_check"
+}
+
 function find_cuda ()
 {
    find_dir "$default_cudas" "$cuda_check"
@ -224,6 +235,7 @@ function show_help ()
    echo "  --with-opencv[=directory] $(show_default $(find_opencv))"
    echo "  --with-libzip[=directory] $(show_default $(find_libzip))"
    echo "  --with-code-coverage[=(yes|no)] $(show_default ${default_use_code_coverage})"
+    echo "  --with-boost[=directory] $(show_default $(find_boost))"
    echo "Libraries search path:"
    for head in $(default_paths)
    do
@ -448,6 +460,26 @@ do
                fi
            fi
            ;;
+        --with-boost*)
+            if test x$optarg = x
+            then
+                boost_path=$(find_boost)
+                if test x$boost_path = x
+                then 
+                    echo "Cannot find Boost directory"
+                    echo "Please specify a value for --with-boost"
+                    exit 1
+                fi
+            else
+                if test $(check_dir $optarg $boost_check) = yes
+                then 
+                    boost_path=$optarg
+                else
+                    echo "Invalid Boost directory $optarg"
+                    exit 1
+                fi
+             fi
+             ;;
        --with-buildtype*)
            have_buildtype=yes
            case $optarg in
@ -641,6 +673,18 @@ then
    fi
 fi

+if test x$boost_path = x
+then
+    boost_path=$(find_boost)
+    if test x$boost_path = x
+    then
+        echo Cannot locate Boost libraries
+        echo Unit tests will NOT be built.
+    else
+        echo Found Boost at $boost_path
+    fi
+fi
+
 config=$build_top/Config.make
 echo Generating $config
 echo "#Configuration file for cntk" > $config
@ -680,6 +724,9 @@ fi
 if test $enable_code_coverage = yes ; then
    echo CNTK_CODE_COVERAGE=true >> $config
 fi
+if test x$boost_path != x; then
+    echo BOOST_PATH=$boost_path >> $config
+fi

 # If we are not in the configure directory, generate a trampoline Makefile
 makefile=$build_top/Makefile