update GPU Dockerfile to cuda 7.5 and add test dependencies to both CPU and GPU Dockerfiles

2016-08-03 15:22:23 -07:00 · 2016-08-03 15:22:23 -07:00 · 0f9eadf7c5
--- a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile
+++ b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile
@ -26,7 +26,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
        python-pip \
        python-yaml \
        wget \
-        zlib1g-dev && \
+        zlib1g-dev \
+        #For Kaldi
+        automake \
+        libtool \
+        autoconf \
+        subversion \
+        #For Kaldi's dependencies
+        libapr1 libaprutil1 libltdl-dev libltdl7 libserf-1-1 libsigsegv2 libsvn1 m4 && \
    rm -rf /var/lib/apt/lists/*

 RUN OPENMPI_VERSION=1.10.3 && \
@ -78,17 +85,62 @@ RUN mkdir /usr/local/CNTKCustomMKL && \
    wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-1.tgz | \
    tar -xzf - -C /usr/local/CNTKCustomMKL

+RUN pip install pytest sphinx-rtd-theme Pillow
+
+ENV BLAS=/usr/local/openblas/lib/libopenblas.so
+ENV LAPACK=/usr/local/openblas/lib/libopenblas.so
+ENV LD_LIBRARY_PATH=/usr/local/openblas/lib:/usr/local/lib:$LD_LIBRARY_PATH
+
+RUN SCIPY_VERSION=0.18.0 && \
+    wget -q -O - https://github.com/scipy/scipy/releases/download/v${SCIPY_VERSION}/scipy-${SCIPY_VERSION}.tar.gz | tar -zxf - && \
+    cd scipy-${SCIPY_VERSION} && \
+    python setup.py build && \
+    python setup.py install && \
+    cd .. && \
+    rm -rf scipy-${SCIPY_VERSION}
+
+# Install Kaldi
+ENV KALDI_VERSION=c024e8aa
+ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
+    
+RUN mv /bin/sh /bin/sh.orig && \
+   ln -s -f /bin/bash /bin/sh && \
+   mkdir $KALDI_PATH && \
+   wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
+   cd $KALDI_PATH && \
+   cd tools && \
+   perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
+   ./extras/check_dependencies.sh && \
+   make -j $(nproc) all && \
+   cd ../src && \
+   ./configure --openblas-root=/usr/local/openblas --shared && \
+   make -j $(nproc) depend && \
+   make -j $(nproc) all && \
+# Remove some unneeded stuff in $KALDI_PATH to reduce size
+   find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
+   for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
+   mv -f /bin/sh.orig /bin/sh
+
 WORKDIR /cntk

 RUN git clone --depth=1 -b master https://github.com/Microsoft/CNTK.git . && \
    mkdir -p build/cpu/release && \
    cd build/cpu/release && \
-    ../../../configure --with-openblas=/usr/local/openblas && \
+    ../../../configure --with-openblas=/usr/local/openblas --with-kaldi=${KALDI_PATH} && \
    make -j"$(nproc)" all && \
    cd ../../.. && \
    mkdir -p build-mkl/cpu/release && \
    cd build-mkl/cpu/release && \
-    ../../../configure --with-mkl=/usr/local/CNTKCustomMKL && \
+    ../../../configure --with-mkl=/usr/local/CNTKCustomMKL --with-kaldi=${KALDI_PATH} && \
    make -j"$(nproc)" all

+RUN cd Examples/Image/Miscellaneous/CIFAR-10 && \
+    python CIFAR_convert.py && \
+    python CifarConverter.py cifar-10-batches-py && \
+    cd ../../../..
+
+RUN cd Examples/Image/MNIST/AdditionalFiles && \
+    python mnist_convert.py && \
+    cd ../../../..
+
 ENV PATH=/cntk/build/cpu/release/bin:$PATH
--- a/Tools/docker/CNTK-GPU-Image/Dockerfile
+++ b/Tools/docker/CNTK-GPU-Image/Dockerfile
@ -1,4 +1,4 @@
-FROM nvidia/cuda:7.0-cudnn4-devel
+FROM nvidia/cuda:7.5-cudnn4-devel

 RUN apt-get update && apt-get install -y --no-install-recommends \
        autotools-dev \
@ -20,7 +20,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
        python-pip \
        python-yaml \
        wget \
-        zlib1g-dev && \
+        zlib1g-dev \
+        #For Kaldi
+        automake \
+        libtool \
+        autoconf \
+        subversion \
+        #For Kaldi's dependencies
+        libapr1 libaprutil1 libltdl-dev libltdl7 libserf-1-1 libsigsegv2 libsvn1 m4 && \
    rm -rf /var/lib/apt/lists/*

 RUN OPENMPI_VERSION=1.10.3 && \
@ -79,6 +86,42 @@ RUN mkdir -p /usr/local/cudnn/cuda/include && \
    mkdir -p /usr/local/cudnn/cuda/lib64 && \
    ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn/cuda/lib64/libcudnn.so

+RUN pip install pytest sphinx-rtd-theme Pillow
+
+ENV BLAS=/usr/local/openblas/lib/libopenblas.so
+ENV LAPACK=/usr/local/openblas/lib/libopenblas.so
+ENV LD_LIBRARY_PATH=/usr/local/openblas/lib:/usr/local/lib:$LD_LIBRARY_PATH
+
+RUN SCIPY_VERSION=0.18.0 && \
+    wget -q -O - https://github.com/scipy/scipy/releases/download/v${SCIPY_VERSION}/scipy-${SCIPY_VERSION}.tar.gz | tar -zxf - && \
+    cd scipy-${SCIPY_VERSION} && \
+    python setup.py build && \
+    python setup.py install && \
+    cd .. && \
+    rm -rf scipy-${SCIPY_VERSION}
+
+# Install Kaldi
+ENV KALDI_VERSION=c024e8aa
+ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
+
+RUN mv /bin/sh /bin/sh.orig && \
+   ln -s -f /bin/bash /bin/sh && \
+   mkdir $KALDI_PATH && \
+   wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
+   cd $KALDI_PATH && \
+   cd tools && \
+   perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
+   ./extras/check_dependencies.sh && \
+   make -j $(nproc) all && \
+   cd ../src && \
+   ./configure --openblas-root=/usr/local/openblas --shared && \
+   make -j $(nproc) depend && \
+   make -j $(nproc) all && \
+# Remove some unneeded stuff in $KALDI_PATH to reduce size
+   find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
+   for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
+   mv -f /bin/sh.orig /bin/sh
+
 WORKDIR /cntk

 RUN git clone --depth=1 -b master https://github.com/Microsoft/CNTK.git . && \
@ -86,6 +129,7 @@ RUN git clone --depth=1 -b master https://github.com/Microsoft/CNTK.git . && \
      --with-cuda=/usr/local/cuda \
      --with-gdk-include=/usr/local/cuda/include \
      --with-gdk-nvml-lib=/usr/local/cuda/lib64/stubs \
+      --with-kaldi=${KALDI_PATH} \
      --with-cudnn=/usr/local/cudnn" && \
    mkdir -p build/gpu/release && \
    cd build/gpu/release && \
@ -97,4 +141,13 @@ RUN git clone --depth=1 -b master https://github.com/Microsoft/CNTK.git . && \
    ../../../configure $CONFIGURE_OPTS --with-mkl=/usr/local/CNTKCustomMKL && \
    make -j"$(nproc)" all

-ENV PATH=/cntk/build/gpu/release/bin:$PATH
+RUN cd Examples/Image/Miscellaneous/CIFAR-10 && \
+    python CIFAR_convert.py && \
+    python CifarConverter.py cifar-10-batches-py && \
+    cd ../../../..
+
+RUN cd Examples/Image/MNIST/AdditionalFiles && \
+    python mnist_convert.py && \
+    cd ../../../..
+
+ENV PATH=/cntk/build/gpu/release/bin:/usr/local/mpi/bin:$PATH