Cache CNTK dependencies at CNTK Azure Blob

This commit is contained in:
Thiago Crepaldi 2019-01-29 12:59:40 -08:00
Родитель 4ef44170ab
Коммит 9093a4def1
4 изменённых файлов: 177 добавлений и 165 удалений

Просмотреть файл

@ -38,10 +38,10 @@ fi
SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
cd "$SCRIPT_DIR"
wget -q http://prdownloads.sourceforge.net/swig/$SWIG_PATH.tar.gz -O - | tar xvfz -
wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/$SWIG_PATH.tar.gz -O - | tar xvfz -
cd swig-$SWIG_VERSION
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
./configure --prefix=$SWIG_PREFIX --without-alllang
./configure --prefix=$SWIG_PREFIX --without-perl5 --without-alllang
make -j $(nproc)
$AS_ROOT make install
cd ..

Просмотреть файл

@ -11,7 +11,7 @@ function OpAnaconda3411(
$prodName = "Anaconda3-4.1.1"
$prodFile = "Anaconda3-4.1.1-Windows-x86_64.exe"
$targetPath = join-path $targetFolder $prodSubDir
$downloadSource = "https://repo.continuum.io/archive/Anaconda3-4.1.1-Windows-x86_64.exe"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/4.1.1/Anaconda3-4.1.1-Windows-x86_64.exe"
$expectedHash = "B4889513DC574F9D6F96DB089315D69D293F8B17635DA4D2E6EEE118DC105F38"
@( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -62,7 +62,7 @@ function OpBoost160VS15(
$prodFile = "boost_1_60_0-msvc-14.0-64.exe"
$prodSubDir = "boost_1_60_0-msvc-14.0"
$targetPath = join-path $targetFolder $prodSubDir
$downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-14.0-64.exe/download"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/1.60.0/boost_1_60_0-msvc-14.0-64.exe"
$envVar = "BOOST_INCLUDE_PATH"
$envVarLib = "BOOST_LIB_PATH"
$envContentLib = "$targetPath\lib64-msvc-14.0"
@ -85,7 +85,7 @@ function OpCMake362(
$prodName = "CMake 3.6.2"
$targetPath = join-path $env:ProgramFiles "cmake\bin"
$cmakeName = "cmake-3.6.2-win64-x64.msi"
$downloadSource = "https://cmake.org/files/v3.6/cmake-3.6.2-win64-x64.msi"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/3.6.2/cmake-3.6.2-win64-x64.msi"
$expectedHash = "5EB7C09C23B13742161076401BB2F4EDABD75ECAFE8916C7A401532BC3794DD5"
@( @{ShortName = "CMake362"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -107,7 +107,7 @@ function OpMKLDNN012(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "MKL_PATH"
$envValue = $targetPath
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/mkldnn/mklml-mkldnn-0.12.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/0.12/mklml-mkldnn-0.12.zip"
$expectedHash = "13C3D485CF96C216B6460188CE6E120847F1BB16B9F66A4134E56EB5D3A37857"
@( @{ShortName = "MKLDNN012"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -124,7 +124,7 @@ function OpMSMPI70([parameter(
{
$remoteFilename = "MSMpiSetup.exe"
$localFilename = "MSMpiSetup70.exe"
$downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename";
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/msmpi/70/$remoteFilename";
$expectedHash = "7DB377051524EE64D0551735A7A9E9A82402068DC529C0D4CF296E2A616C22AF"
@( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70";
@ -139,7 +139,7 @@ function OpMSMPI70SDK(
{
$remoteFilename = "msmpisdk.msi"
$localFilename = "msmpisdk70.msi"
$downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename";
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/msmpisdk/70/$remoteFilename";
$expectedHash = "C28FB6121FE7A5102ED8B011992708039EE878B9F58A34B84AF41AA3622B8F4D"
@( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Installing MSMPI 70 SDK";
@ -159,7 +159,7 @@ function OpNvidiaCub180(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "CUB_PATH";
$envValue = $targetPath
$downloadSource = "https://github.com/NVlabs/cub/archive/1.8.0.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cub/1.8.0/cub-1.8.0.zip"
@( @{ShortName = "CUB180"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" },
@ -181,7 +181,7 @@ function OpNVidiaCudnn73100(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "CUDNN_PATH"
$envValue = join-path $targetPath "cuda"
$downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v7.3.1"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cudnn/7.3.1"
@( @{ShortName = "CUDNN73100"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath },
@ -203,7 +203,7 @@ function OpOpenCV31(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "OPENCV_PATH_V31";
$envValue = "$targetPath\build"
$downloadSource = "https://sourceforge.net/projects/opencvlibrary/files/opencv-win/3.1.0/opencv-3.1.0.exe/download"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/3.1.0/opencv-3.1.0.exe"
$expectedHash = "0CBB10FAB967111B5B699A44CB224F5D729F8D852D2720CBD5CDB56D8770B7B3"
$archiveSubTree = "opencv"
@ -235,7 +235,7 @@ function OpProtoBuf310VS17(
$targetPath = Join-Path $protoSourceDir $prodSrcSubdir
$scriptDirectory = join-path $targetFolder "script"
$buildDir = join-path $targetFolder $prodSubDir
$downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/3.1.0/protobuf-3.1.0.zip"
$expectedHash = "C07629F666312E43A4C2415AF77F6442178605A8658D975299C793CB89999212"
@( @{ShortName = "PROTO310"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -258,7 +258,7 @@ function OpProtoBuf310VS17Prebuild(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "PROTOBUF_PATH"
$envValue = $targetPath
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/protobuf/protobuf-3.1.0-vs17.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/3.1.0/protobuf-3.1.0-vs17.zip"
$expectedHash = "ED0F3215AC60E6AE29B21CBFF53F8876E4CF8B4767FEC525CEF0DA6FDF6A4A73"
@( @{ShortName = "PROTO310PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -287,7 +287,7 @@ function OpSwig3010(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "SWIG_PATH"
$envValue = $targetPath
$downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/3.0.10/swigwin-3.0.10.zip"
$expectedHash = "68A202EBFC62647495074A190A115B629E84C56D74D3017CCB43E56A4B9B83F6"
@( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
@ -332,12 +332,12 @@ function OpZlibVS17(
$zlibProdName = "zlib-1.2.8"
$zlibFilename = "zlib128.zip"
# $zlibDownloadSource = "https://netix.dl.sourceforge.net/project/libpng/zlib/1.2.8/zlib128.zip"
$zlibDownloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib128.zip"
$zlibDownloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/zlib/1.2.8/zlib128.zip"
$expectedHashZlib = "879D73D8CD4D155F31C1F04838ECD567D34BEBDA780156F0E82A20721B3973D5"
$libzipProdName = "libzip-1.1.3"
$libzipFilename = "libzip-1.1.3.tar.gz"
$libzipDownloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz"
$libzipDownloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/1.1.3/libzip-1.1.3.tar.gz"
$downloadeSizeLibzip = "1FAA5A524DD4A12C43B6344E618EDCE1BF8050DFDB9D0F73F3CC826929A002B0"
$prodSubDir = "zlib-vs17"
@ -374,7 +374,7 @@ function OpZlibVS17Prebuild(
$targetPath = join-path $targetFolder $prodSubDir
$envVar = "ZLIB_PATH"
$envValue = $targetPath
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib-vs17.zip"
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/zlib/vs17/zlib-vs17.zip"
$expectedHash = "40A79007EC792756370C35E6C8585C0C5E8750A44BD2F60DB1EA542AAF398A7B"
@( @{ShortName = "ZLIBPRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";

Просмотреть файл

@ -55,64 +55,69 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ARG CMAKE_DOWNLOAD_VERSION=3.11
ARG CMAKE_BUILD_VERSION=4
RUN DEBIAN_FRONTEND=noninteractive && \
wget --no-verbose https://cmake.org/files/v$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
wget --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
cd cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION && \
./bootstrap --system-curl -- -DCMAKE_USE_OPENSSL=ON && \
make -j $(nproc) install && \
cd .. && \
cd .. && \
rm -rf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION
RUN OPENMPI_VERSION=1.10.3 && \
wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
ARG OPENMPI_VERSION=1.10.7
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openmpi/$OPENMPI_VERSION/openmpi-$OPENMPI_VERSION.tar.gz | tar -xzf - && \
cd openmpi-${OPENMPI_VERSION} && \
./configure --prefix=/usr/local/mpi && \
make -j"$(nproc)" install && \
rm -rf /openmpi-${OPENMPI_VERSION}
apt-get -y update && \
apt-get -y -f install && \
apt-get -y install libsysfs2 libsysfs-dev && \
./configure --with-verbs --with-cuda=/usr/local/cuda --prefix=/usr/local/mpi && \
make -j $(nproc) install && \
cd .. && \
rm -rf openmpi-${OPENMPI_VERSION}
ENV PATH /usr/local/mpi/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/mpi/lib:$LD_LIBRARY_PATH
RUN LIBZIP_VERSION=1.1.2 && \
wget -q -O - https://libzip.org/download/libzip-${LIBZIP_VERSION}.tar.gz | tar -xzf - && \
ARG LIBZIP_VERSION=1.1.2
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/$LIBZIP_VERSION/libzip-$LIBZIP_VERSION.tar.gz | tar -xzf - && \
cd libzip-${LIBZIP_VERSION} && \
./configure && \
make -j"$(nproc)" install && \
rm -rf /libzip-${LIBZIP_VERSION}
make -j $(nproc) install && \
cd .. && \
rm -rf libzip-${LIBZIP_VERSION}
ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
RUN OPENCV_VERSION=3.1.0 && \
wget -q -O - https://github.com/Itseez/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
ARG OPENCV_VERSION=3.1.0
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/$OPENCV_VERSION/opencv-$OPENCV_VERSION.tar.gz | tar -xzf - && \
cd opencv-${OPENCV_VERSION} && \
cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
make -j"$(nproc)" install && \
rm -rf /opencv-${OPENCV_VERSION}
cmake -DWITH_CUDA=OFF -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
make -j $(nproc) install && \
cd .. && \
rm -rf opencv-${OPENCV_VERSION}
RUN OPENBLAS_VERSION=0.2.18 && \
wget -q -O - https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz | tar -xzf - && \
ARG OPENBLAS_VERSION=0.2.18
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openblas/$OPENBLAS_VERSION/OpenBLAS-$OPENBLAS_VERSION.tar.gz | tar -xzf - && \
cd OpenBLAS-${OPENBLAS_VERSION} && \
make -j"$(nproc)" USE_OPENMP=1 | tee make.log && \
make -j 2 MAKE_NB_JOBS=0 USE_OPENMP=1 | tee make.log && \
grep -qF 'OpenBLAS build complete. (BLAS CBLAS LAPACK LAPACKE)' make.log && \
grep -qF 'Use OpenMP in the multithreading.' make.log && \
make PREFIX=/usr/local/openblas install && \
rm -rf /OpenBLAS-${OPENBLAS_VERSION}
cd .. && \
rm -rf OpenBLAS-${OPENBLAS_VERSION}
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
# Install Boost
RUN BOOST_VERSION=1_60_0 && \
BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
cd boost_${BOOST_VERSION} && \
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test && \
./b2 -d0 -j"$(nproc)" install && \
rm -rf /boost_${BOOST_VERSION}
ARG BOOST_VERSION=1.60.0
RUN BOOST_UNDERSCORE_VERSION=$(echo $BOOST_VERSION | tr . _) && \
wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/$BOOST_VERSION/boost_$BOOST_UNDERSCORE_VERSION.tar.gz | tar -xzf - && \
cd boost_${BOOST_UNDERSCORE_VERSION} && \
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_VERSION} && \
./b2 -d0 -j $(nproc) install && \
cd .. && \
rm -rf boost_${BOOST_UNDERSCORE_VERSION}
# Install Protobuf
RUN PROTOBUF_VERSION=3.1.0 \
PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
wget -O - --no-verbose https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
ARG PROTOBUF_VERSION=3.1.0
RUN PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
wget -O - --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz | tar -xzf - && \
cd $PROTOBUF_STRING && \
./autogen.sh && \
./configure CFLAGS=-fPIC CXXFLAGS=-fPIC --disable-shared --prefix=/usr/local/$PROTOBUF_STRING && \
@ -124,10 +129,10 @@ RUN PROTOBUF_VERSION=3.1.0 \
ARG MKLDNN_VERSION=0.14
ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
RUN mkdir /usr/local/mklml && \
wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/$MKLDNN_LONG_VERSION.tgz | \
tar -xzf - -C /usr/local/mklml && \
MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/mkl-dnn-$MKLDNN_VERSION.tar.gz | tar -xzf - && \
cd ${MKLDNN_STRING} && \
ln -s /usr/local external && \
mkdir -p build && \
@ -140,28 +145,27 @@ RUN mkdir /usr/local/mklml && \
rm -rf ${MKLDNN_STRING}
# Install Kaldi
ENV KALDI_VERSION=c024e8aa
ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
ARG KALDI_VERSION=c024e8aa
ARG KALDI_PATH=/usr/local/kaldi-$KALDI_VERSION
RUN mv /bin/sh /bin/sh.orig && \
ln -s -f /bin/bash /bin/sh && \
mkdir $KALDI_PATH && \
wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
cd $KALDI_PATH && \
cd tools && \
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
./extras/check_dependencies.sh && \
make -j $(nproc) all && \
cd ../src && \
# remove Fermi support as CUDA 9 no longer works on it
perl -pi -e 's/-gencode arch=compute_20,code=sm_20//' cudamatrix/Makefile && \
./configure --openblas-root=/usr/local/openblas --shared && \
make -j $(nproc) depend && \
make -j $(nproc) all && \
# Remove some unneeded stuff in $KALDI_PATH to reduce size
find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
mv -f /bin/sh.orig /bin/sh
ln -s -f /bin/bash /bin/sh && \
mkdir $KALDI_PATH && \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/kaldi/$KALDI_VERSION/kaldi-$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
cd $KALDI_PATH && \
cd tools && \
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
./extras/check_dependencies.sh && \
make -j $(nproc) all && \
cd ../src && \
# remove Fermi support as CUDA 9 no longer works on it
perl -pi -e 's/-gencode arch=compute_20,code=sm_20//' cudamatrix/Makefile && \
./configure --openblas-root=/usr/local/openblas --shared && \
make -j $(nproc) depend && \
make -j $(nproc) all && \
# Remove some unneeded stuff in $KALDI_PATH to reduce size
find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
mv -f /bin/sh.orig /bin/sh
## PYTHON
@ -169,32 +173,33 @@ RUN mv /bin/sh /bin/sh.orig && \
ARG COMMIT=master
# Swig
RUN cd /root && \
wget -q http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz -O - | tar xvfz - && \
cd swig-3.0.10 && \
ARG SWIG_VERSION=3.0.10
ARG CACHEBUST=1
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/swig-$SWIG_VERSION.tar.gz -O - | tar xvfz - && \
cd swig-$SWIG_VERSION && \
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
./configure --without-alllang && \
./configure --prefix=/usr/local/swig-$SWIG_VERSION --without-perl5 --without-alllang && \
make -j $(nproc) && \
make install
make install && \
cd .. && \
rm -rf swig-$SWIG_VERSION
COPY ./Patches /tmp/patches
RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
rm -rfd /tmp/patches
# .NET Core SDK
RUN cd /tmp && \
(wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/packages-microsoft-prod/deb/packages-microsoft-prod.deb && \
dpkg -i packages-microsoft-prod.deb && \
apt-get update && \
apt-get --yes install dotnet-sdk-2.1.200 && \
rm /tmp/packages-microsoft-prod.deb && \
apt-get --yes autoremove && \
rm -rf /var/lib/apt/lists/*
apt-get -y -f install apt-transport-https && \
apt-get -y update && \
apt-get -y -f install dotnet-sdk-2.1 && \
rm ./packages-microsoft-prod.deb
# Anaconda
RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \
bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \
rm Anaconda3-4.2.0-Linux-x86_64.sh
ARG ANACONDA_VERSION=4.2.0
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/$ANACONDA_VERSION/Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh && \
bash Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh -b && \
rm Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh
RUN CONDA_ENV_PATH=/tmp/conda-linux-cntk-py35-environment.yml; \
wget -q https://raw.githubusercontent.com/Microsoft/CNTK/$COMMIT/Scripts/install/linux/conda-linux-cntk-py35-environment.yml -O "$CONDA_ENV_PATH" && \

Просмотреть файл

@ -56,66 +56,73 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ARG CMAKE_DOWNLOAD_VERSION=3.11
ARG CMAKE_BUILD_VERSION=4
RUN DEBIAN_FRONTEND=noninteractive && \
wget --no-verbose https://cmake.org/files/v$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
wget --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
cd cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION && \
./bootstrap --system-curl -- -DCMAKE_USE_OPENSSL=ON && \
make -j $(nproc) install && \
cd .. && \
cd .. && \
rm -rf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION
RUN OPENMPI_VERSION=1.10.3 && \
wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
ARG OPENMPI_VERSION=1.10.7
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openmpi/$OPENMPI_VERSION/openmpi-$OPENMPI_VERSION.tar.gz | tar -xzf - && \
cd openmpi-${OPENMPI_VERSION} && \
./configure --prefix=/usr/local/mpi && \
make -j"$(nproc)" install && \
rm -rf /openmpi-${OPENMPI_VERSION}
apt-get -y update && \
apt-get -y -f install && \
apt-get -y install libsysfs2 libsysfs-dev && \
./configure --with-verbs --with-cuda=/usr/local/cuda --prefix=/usr/local/mpi && \
make -j $(nproc) install && \
cd .. && \
rm -rf openmpi-${OPENMPI_VERSION}
ENV PATH /usr/local/mpi/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/mpi/lib:$LD_LIBRARY_PATH
RUN LIBZIP_VERSION=1.1.2 && \
wget -q -O - https://libzip.org/download/libzip-${LIBZIP_VERSION}.tar.gz | tar -xzf - && \
ARG LIBZIP_VERSION=1.1.2
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/$LIBZIP_VERSION/libzip-$LIBZIP_VERSION.tar.gz | tar -xzf - && \
cd libzip-${LIBZIP_VERSION} && \
./configure && \
make -j"$(nproc)" install && \
rm -rf /libzip-${LIBZIP_VERSION}
make -j $(nproc) install && \
cd .. && \
rm -rf libzip-${LIBZIP_VERSION}
ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
RUN wget -q -O - https://github.com/NVlabs/cub/archive/1.8.0.tar.gz | tar -C /usr/local -xzf -
ARG CUB_VERSION=1.8.0
RUN wget https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cub/$CUB_VERSION/cub-$CUB_VERSION.zip && \
unzip -d /usr/local cub-${CUB_VERSION}.zip && \
rm cub-${CUB_VERSION}.zip
RUN OPENCV_VERSION=3.1.0 && \
wget -q -O - https://github.com/Itseez/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
ARG OPENCV_VERSION=3.1.0
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/$OPENCV_VERSION/opencv-$OPENCV_VERSION.tar.gz | tar -xzf - && \
cd opencv-${OPENCV_VERSION} && \
cmake -DWITH_CUDA=OFF -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
make -j"$(nproc)" install && \
rm -rf /opencv-${OPENCV_VERSION}
make -j $(nproc) install && \
cd .. && \
rm -rf opencv-${OPENCV_VERSION}
RUN OPENBLAS_VERSION=0.2.18 && \
wget -q -O - https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz | tar -xzf - && \
ARG OPENBLAS_VERSION=0.2.18
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openblas/$OPENBLAS_VERSION/OpenBLAS-$OPENBLAS_VERSION.tar.gz | tar -xzf - && \
cd OpenBLAS-${OPENBLAS_VERSION} && \
make -j"$(nproc)" USE_OPENMP=1 | tee make.log && \
make -j 2 MAKE_NB_JOBS=0 USE_OPENMP=1 | tee make.log && \
grep -qF 'OpenBLAS build complete. (BLAS CBLAS LAPACK LAPACKE)' make.log && \
grep -qF 'Use OpenMP in the multithreading.' make.log && \
make PREFIX=/usr/local/openblas install && \
rm -rf /OpenBLAS-${OPENBLAS_VERSION}
cd .. && \
rm -rf OpenBLAS-${OPENBLAS_VERSION}
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
# Install Boost
RUN BOOST_VERSION=1_60_0 && \
BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
cd boost_${BOOST_VERSION} && \
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test && \
./b2 -d0 -j"$(nproc)" install && \
rm -rf /boost_${BOOST_VERSION}
ARG BOOST_VERSION=1.60.0
RUN BOOST_UNDERSCORE_VERSION=$(echo $BOOST_VERSION | tr . _) && \
wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/$BOOST_VERSION/boost_$BOOST_UNDERSCORE_VERSION.tar.gz | tar -xzf - && \
cd boost_${BOOST_UNDERSCORE_VERSION} && \
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_VERSION} && \
./b2 -d0 -j $(nproc) install && \
cd .. && \
rm -rf boost_${BOOST_UNDERSCORE_VERSION}
# Install Protobuf
RUN PROTOBUF_VERSION=3.1.0 \
PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
wget -O - --no-verbose https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
ARG PROTOBUF_VERSION=3.1.0
RUN PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
wget -O - --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz | tar -xzf - && \
cd $PROTOBUF_STRING && \
./autogen.sh && \
./configure CFLAGS=-fPIC CXXFLAGS=-fPIC --disable-shared --prefix=/usr/local/$PROTOBUF_STRING && \
@ -127,10 +134,10 @@ RUN PROTOBUF_VERSION=3.1.0 \
ARG MKLDNN_VERSION=0.14
ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
RUN mkdir /usr/local/mklml && \
wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/$MKLDNN_LONG_VERSION.tgz | \
tar -xzf - -C /usr/local/mklml && \
MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/mkl-dnn-$MKLDNN_VERSION.tar.gz | tar -xzf - && \
cd ${MKLDNN_STRING} && \
ln -s /usr/local external && \
mkdir -p build && \
@ -143,28 +150,27 @@ RUN mkdir /usr/local/mklml && \
rm -rf ${MKLDNN_STRING}
# Install Kaldi
ENV KALDI_VERSION=c024e8aa
ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
ARG KALDI_VERSION=c024e8aa
ARG KALDI_PATH=/usr/local/kaldi-$KALDI_VERSION
RUN mv /bin/sh /bin/sh.orig && \
ln -s -f /bin/bash /bin/sh && \
mkdir $KALDI_PATH && \
wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
cd $KALDI_PATH && \
cd tools && \
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
./extras/check_dependencies.sh && \
make -j $(nproc) all && \
cd ../src && \
# remove Fermi support as CUDA 9 no longer works on it
perl -pi -e 's/-gencode arch=compute_20,code=sm_20//' cudamatrix/Makefile && \
./configure --openblas-root=/usr/local/openblas --shared && \
make -j $(nproc) depend && \
make -j $(nproc) all && \
# Remove some unneeded stuff in $KALDI_PATH to reduce size
find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
mv -f /bin/sh.orig /bin/sh
ln -s -f /bin/bash /bin/sh && \
mkdir $KALDI_PATH && \
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/kaldi/$KALDI_VERSION/kaldi-$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
cd $KALDI_PATH && \
cd tools && \
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
./extras/check_dependencies.sh && \
make -j $(nproc) all && \
cd ../src && \
# remove Fermi support as CUDA 9 no longer works on it
perl -pi -e 's/-gencode arch=compute_20,code=sm_20//' cudamatrix/Makefile && \
./configure --openblas-root=/usr/local/openblas --shared && \
make -j $(nproc) depend && \
make -j $(nproc) all && \
# Remove some unneeded stuff in $KALDI_PATH to reduce size
find $KALDI_PATH -name '*.o' -print0 | xargs -0 rm && \
for dir in $KALDI_PATH/src/*bin; do make -C $dir clean; done && \
mv -f /bin/sh.orig /bin/sh
## PYTHON
@ -172,32 +178,33 @@ RUN mv /bin/sh /bin/sh.orig && \
ARG COMMIT=master
# Swig
RUN cd /root && \
wget -q http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz -O - | tar xvfz - && \
cd swig-3.0.10 && \
ARG SWIG_VERSION=3.0.10
ARG CACHEBUST=1
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/swig-$SWIG_VERSION.tar.gz -O - | tar xvfz - && \
cd swig-$SWIG_VERSION && \
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
./configure --without-alllang && \
./configure --prefix=/usr/local/swig-$SWIG_VERSION --without-perl5 --without-alllang && \
make -j $(nproc) && \
make install
make install && \
cd .. && \
rm -rf swig-$SWIG_VERSION
COPY ./Patches /tmp/patches
RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
rm -rfd /tmp/patches
# .NET Core SDK
RUN cd /tmp && \
(wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/packages-microsoft-prod/deb/packages-microsoft-prod.deb && \
dpkg -i packages-microsoft-prod.deb && \
apt-get update && \
apt-get --yes install dotnet-sdk-2.1.200 && \
rm /tmp/packages-microsoft-prod.deb && \
apt-get --yes autoremove && \
rm -rf /var/lib/apt/lists/*
apt-get -y -f install apt-transport-https && \
apt-get -y update && \
apt-get -y -f install dotnet-sdk-2.1 && \
rm ./packages-microsoft-prod.deb
# Anaconda
RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \
bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \
rm Anaconda3-4.2.0-Linux-x86_64.sh
ARG ANACONDA_VERSION=4.2.0
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/$ANACONDA_VERSION/Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh && \
bash Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh -b && \
rm Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh
RUN CONDA_ENV_PATH=/tmp/conda-linux-cntk-py35-environment.yml; \
wget -q https://raw.githubusercontent.com/Microsoft/CNTK/$COMMIT/Scripts/install/linux/conda-linux-cntk-py35-environment.yml -O "$CONDA_ENV_PATH" && \