Cache CNTK dependencies at CNTK Azure Blob
This commit is contained in:
Thiago Crepaldi
Родитель
4ef44170ab
Коммит
9093a4def1
|
|
@ -38,10 +38,10 @@ fi
|
|||
|
||||
SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
|
||||
cd "$SCRIPT_DIR"
|
||||
wget -q http://prdownloads.sourceforge.net/swig/$SWIG_PATH.tar.gz -O - | tar xvfz -
|
||||
wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/$SWIG_PATH.tar.gz -O - | tar xvfz -
|
||||
cd swig-$SWIG_VERSION
|
||||
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
|
||||
./configure --prefix=$SWIG_PREFIX --without-alllang
|
||||
./configure --prefix=$SWIG_PREFIX --without-perl5 --without-alllang
|
||||
make -j $(nproc)
|
||||
$AS_ROOT make install
|
||||
cd ..
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ function OpAnaconda3411(
|
|||
$prodName = "Anaconda3-4.1.1"
|
||||
$prodFile = "Anaconda3-4.1.1-Windows-x86_64.exe"
|
||||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$downloadSource = "https://repo.continuum.io/archive/Anaconda3-4.1.1-Windows-x86_64.exe"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/4.1.1/Anaconda3-4.1.1-Windows-x86_64.exe"
|
||||
$expectedHash = "B4889513DC574F9D6F96DB089315D69D293F8B17635DA4D2E6EEE118DC105F38"
|
||||
|
||||
@( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -62,7 +62,7 @@ function OpBoost160VS15(
|
|||
$prodFile = "boost_1_60_0-msvc-14.0-64.exe"
|
||||
$prodSubDir = "boost_1_60_0-msvc-14.0"
|
||||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-14.0-64.exe/download"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/1.60.0/boost_1_60_0-msvc-14.0-64.exe"
|
||||
$envVar = "BOOST_INCLUDE_PATH"
|
||||
$envVarLib = "BOOST_LIB_PATH"
|
||||
$envContentLib = "$targetPath\lib64-msvc-14.0"
|
||||
|
|
@ -85,7 +85,7 @@ function OpCMake362(
|
|||
$prodName = "CMake 3.6.2"
|
||||
$targetPath = join-path $env:ProgramFiles "cmake\bin"
|
||||
$cmakeName = "cmake-3.6.2-win64-x64.msi"
|
||||
$downloadSource = "https://cmake.org/files/v3.6/cmake-3.6.2-win64-x64.msi"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/3.6.2/cmake-3.6.2-win64-x64.msi"
|
||||
$expectedHash = "5EB7C09C23B13742161076401BB2F4EDABD75ECAFE8916C7A401532BC3794DD5"
|
||||
|
||||
@( @{ShortName = "CMake362"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -107,7 +107,7 @@ function OpMKLDNN012(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "MKL_PATH"
|
||||
$envValue = $targetPath
|
||||
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/mkldnn/mklml-mkldnn-0.12.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/0.12/mklml-mkldnn-0.12.zip"
|
||||
$expectedHash = "13C3D485CF96C216B6460188CE6E120847F1BB16B9F66A4134E56EB5D3A37857"
|
||||
|
||||
@( @{ShortName = "MKLDNN012"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -124,7 +124,7 @@ function OpMSMPI70([parameter(
|
|||
{
|
||||
$remoteFilename = "MSMpiSetup.exe"
|
||||
$localFilename = "MSMpiSetup70.exe"
|
||||
$downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename";
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/msmpi/70/$remoteFilename";
|
||||
$expectedHash = "7DB377051524EE64D0551735A7A9E9A82402068DC529C0D4CF296E2A616C22AF"
|
||||
|
||||
@( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70";
|
||||
|
|
@ -139,7 +139,7 @@ function OpMSMPI70SDK(
|
|||
{
|
||||
$remoteFilename = "msmpisdk.msi"
|
||||
$localFilename = "msmpisdk70.msi"
|
||||
$downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename";
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/msmpisdk/70/$remoteFilename";
|
||||
$expectedHash = "C28FB6121FE7A5102ED8B011992708039EE878B9F58A34B84AF41AA3622B8F4D"
|
||||
|
||||
@( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Installing MSMPI 70 SDK";
|
||||
|
|
@ -159,7 +159,7 @@ function OpNvidiaCub180(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "CUB_PATH";
|
||||
$envValue = $targetPath
|
||||
$downloadSource = "https://github.com/NVlabs/cub/archive/1.8.0.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cub/1.8.0/cub-1.8.0.zip"
|
||||
|
||||
@( @{ShortName = "CUB180"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" },
|
||||
|
|
@ -181,7 +181,7 @@ function OpNVidiaCudnn73100(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "CUDNN_PATH"
|
||||
$envValue = join-path $targetPath "cuda"
|
||||
$downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v7.3.1"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cudnn/7.3.1"
|
||||
|
||||
@( @{ShortName = "CUDNN73100"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath },
|
||||
|
|
@ -203,7 +203,7 @@ function OpOpenCV31(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "OPENCV_PATH_V31";
|
||||
$envValue = "$targetPath\build"
|
||||
$downloadSource = "https://sourceforge.net/projects/opencvlibrary/files/opencv-win/3.1.0/opencv-3.1.0.exe/download"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/3.1.0/opencv-3.1.0.exe"
|
||||
$expectedHash = "0CBB10FAB967111B5B699A44CB224F5D729F8D852D2720CBD5CDB56D8770B7B3"
|
||||
$archiveSubTree = "opencv"
|
||||
|
||||
|
|
@ -235,7 +235,7 @@ function OpProtoBuf310VS17(
|
|||
$targetPath = Join-Path $protoSourceDir $prodSrcSubdir
|
||||
$scriptDirectory = join-path $targetFolder "script"
|
||||
$buildDir = join-path $targetFolder $prodSubDir
|
||||
$downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/3.1.0/protobuf-3.1.0.zip"
|
||||
$expectedHash = "C07629F666312E43A4C2415AF77F6442178605A8658D975299C793CB89999212"
|
||||
|
||||
@( @{ShortName = "PROTO310"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -258,7 +258,7 @@ function OpProtoBuf310VS17Prebuild(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "PROTOBUF_PATH"
|
||||
$envValue = $targetPath
|
||||
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/protobuf/protobuf-3.1.0-vs17.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/3.1.0/protobuf-3.1.0-vs17.zip"
|
||||
$expectedHash = "ED0F3215AC60E6AE29B21CBFF53F8876E4CF8B4767FEC525CEF0DA6FDF6A4A73"
|
||||
|
||||
@( @{ShortName = "PROTO310PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -287,7 +287,7 @@ function OpSwig3010(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "SWIG_PATH"
|
||||
$envValue = $targetPath
|
||||
$downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/3.0.10/swigwin-3.0.10.zip"
|
||||
$expectedHash = "68A202EBFC62647495074A190A115B629E84C56D74D3017CCB43E56A4B9B83F6"
|
||||
|
||||
@( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
@ -332,12 +332,12 @@ function OpZlibVS17(
|
|||
$zlibProdName = "zlib-1.2.8"
|
||||
$zlibFilename = "zlib128.zip"
|
||||
# $zlibDownloadSource = "https://netix.dl.sourceforge.net/project/libpng/zlib/1.2.8/zlib128.zip"
|
||||
$zlibDownloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib128.zip"
|
||||
$zlibDownloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/zlib/1.2.8/zlib128.zip"
|
||||
$expectedHashZlib = "879D73D8CD4D155F31C1F04838ECD567D34BEBDA780156F0E82A20721B3973D5"
|
||||
|
||||
$libzipProdName = "libzip-1.1.3"
|
||||
$libzipFilename = "libzip-1.1.3.tar.gz"
|
||||
$libzipDownloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz"
|
||||
$libzipDownloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/1.1.3/libzip-1.1.3.tar.gz"
|
||||
$downloadeSizeLibzip = "1FAA5A524DD4A12C43B6344E618EDCE1BF8050DFDB9D0F73F3CC826929A002B0"
|
||||
|
||||
$prodSubDir = "zlib-vs17"
|
||||
|
|
@ -374,7 +374,7 @@ function OpZlibVS17Prebuild(
|
|||
$targetPath = join-path $targetFolder $prodSubDir
|
||||
$envVar = "ZLIB_PATH"
|
||||
$envValue = $targetPath
|
||||
$downloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib-vs17.zip"
|
||||
$downloadSource = "https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/zlib/vs17/zlib-vs17.zip"
|
||||
$expectedHash = "40A79007EC792756370C35E6C8585C0C5E8750A44BD2F60DB1EA542AAF398A7B"
|
||||
|
||||
@( @{ShortName = "ZLIBPRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName";
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
ARG CMAKE_DOWNLOAD_VERSION=3.11
|
||||
ARG CMAKE_BUILD_VERSION=4
|
||||
RUN DEBIAN_FRONTEND=noninteractive && \
|
||||
wget --no-verbose https://cmake.org/files/v$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
wget --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
cd cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION && \
|
||||
./bootstrap --system-curl -- -DCMAKE_USE_OPENSSL=ON && \
|
||||
|
|
@ -63,56 +63,61 @@ RUN DEBIAN_FRONTEND=noninteractive && \
|
|||
cd .. && \
|
||||
rm -rf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION
|
||||
|
||||
RUN OPENMPI_VERSION=1.10.3 && \
|
||||
wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENMPI_VERSION=1.10.7
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openmpi/$OPENMPI_VERSION/openmpi-$OPENMPI_VERSION.tar.gz | tar -xzf - && \
|
||||
cd openmpi-${OPENMPI_VERSION} && \
|
||||
./configure --prefix=/usr/local/mpi && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /openmpi-${OPENMPI_VERSION}
|
||||
|
||||
apt-get -y update && \
|
||||
apt-get -y -f install && \
|
||||
apt-get -y install libsysfs2 libsysfs-dev && \
|
||||
./configure --with-verbs --with-cuda=/usr/local/cuda --prefix=/usr/local/mpi && \
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf openmpi-${OPENMPI_VERSION}
|
||||
ENV PATH /usr/local/mpi/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH /usr/local/mpi/lib:$LD_LIBRARY_PATH
|
||||
|
||||
RUN LIBZIP_VERSION=1.1.2 && \
|
||||
wget -q -O - https://libzip.org/download/libzip-${LIBZIP_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG LIBZIP_VERSION=1.1.2
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/$LIBZIP_VERSION/libzip-$LIBZIP_VERSION.tar.gz | tar -xzf - && \
|
||||
cd libzip-${LIBZIP_VERSION} && \
|
||||
./configure && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /libzip-${LIBZIP_VERSION}
|
||||
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf libzip-${LIBZIP_VERSION}
|
||||
ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
|
||||
|
||||
RUN OPENCV_VERSION=3.1.0 && \
|
||||
wget -q -O - https://github.com/Itseez/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENCV_VERSION=3.1.0
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/$OPENCV_VERSION/opencv-$OPENCV_VERSION.tar.gz | tar -xzf - && \
|
||||
cd opencv-${OPENCV_VERSION} && \
|
||||
cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /opencv-${OPENCV_VERSION}
|
||||
cmake -DWITH_CUDA=OFF -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf opencv-${OPENCV_VERSION}
|
||||
|
||||
RUN OPENBLAS_VERSION=0.2.18 && \
|
||||
wget -q -O - https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENBLAS_VERSION=0.2.18
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openblas/$OPENBLAS_VERSION/OpenBLAS-$OPENBLAS_VERSION.tar.gz | tar -xzf - && \
|
||||
cd OpenBLAS-${OPENBLAS_VERSION} && \
|
||||
make -j"$(nproc)" USE_OPENMP=1 | tee make.log && \
|
||||
make -j 2 MAKE_NB_JOBS=0 USE_OPENMP=1 | tee make.log && \
|
||||
grep -qF 'OpenBLAS build complete. (BLAS CBLAS LAPACK LAPACKE)' make.log && \
|
||||
grep -qF 'Use OpenMP in the multithreading.' make.log && \
|
||||
make PREFIX=/usr/local/openblas install && \
|
||||
rm -rf /OpenBLAS-${OPENBLAS_VERSION}
|
||||
|
||||
cd .. && \
|
||||
rm -rf OpenBLAS-${OPENBLAS_VERSION}
|
||||
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
|
||||
|
||||
# Install Boost
|
||||
RUN BOOST_VERSION=1_60_0 && \
|
||||
BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
|
||||
wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
|
||||
cd boost_${BOOST_VERSION} && \
|
||||
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test && \
|
||||
./b2 -d0 -j"$(nproc)" install && \
|
||||
rm -rf /boost_${BOOST_VERSION}
|
||||
ARG BOOST_VERSION=1.60.0
|
||||
RUN BOOST_UNDERSCORE_VERSION=$(echo $BOOST_VERSION | tr . _) && \
|
||||
wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/$BOOST_VERSION/boost_$BOOST_UNDERSCORE_VERSION.tar.gz | tar -xzf - && \
|
||||
cd boost_${BOOST_UNDERSCORE_VERSION} && \
|
||||
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_VERSION} && \
|
||||
./b2 -d0 -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf boost_${BOOST_UNDERSCORE_VERSION}
|
||||
|
||||
# Install Protobuf
|
||||
RUN PROTOBUF_VERSION=3.1.0 \
|
||||
PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
|
||||
wget -O - --no-verbose https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG PROTOBUF_VERSION=3.1.0
|
||||
RUN PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
|
||||
wget -O - --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz | tar -xzf - && \
|
||||
cd $PROTOBUF_STRING && \
|
||||
./autogen.sh && \
|
||||
./configure CFLAGS=-fPIC CXXFLAGS=-fPIC --disable-shared --prefix=/usr/local/$PROTOBUF_STRING && \
|
||||
|
|
@ -124,10 +129,10 @@ RUN PROTOBUF_VERSION=3.1.0 \
|
|||
ARG MKLDNN_VERSION=0.14
|
||||
ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
|
||||
RUN mkdir /usr/local/mklml && \
|
||||
wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/$MKLDNN_LONG_VERSION.tgz | \
|
||||
tar -xzf - -C /usr/local/mklml && \
|
||||
MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
|
||||
wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/mkl-dnn-$MKLDNN_VERSION.tar.gz | tar -xzf - && \
|
||||
cd ${MKLDNN_STRING} && \
|
||||
ln -s /usr/local external && \
|
||||
mkdir -p build && \
|
||||
|
|
@ -140,13 +145,12 @@ RUN mkdir /usr/local/mklml && \
|
|||
rm -rf ${MKLDNN_STRING}
|
||||
|
||||
# Install Kaldi
|
||||
ENV KALDI_VERSION=c024e8aa
|
||||
ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
|
||||
|
||||
ARG KALDI_VERSION=c024e8aa
|
||||
ARG KALDI_PATH=/usr/local/kaldi-$KALDI_VERSION
|
||||
RUN mv /bin/sh /bin/sh.orig && \
|
||||
ln -s -f /bin/bash /bin/sh && \
|
||||
mkdir $KALDI_PATH && \
|
||||
wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/kaldi/$KALDI_VERSION/kaldi-$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
|
||||
cd $KALDI_PATH && \
|
||||
cd tools && \
|
||||
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
|
||||
|
|
@ -169,32 +173,33 @@ RUN mv /bin/sh /bin/sh.orig && \
|
|||
ARG COMMIT=master
|
||||
|
||||
# Swig
|
||||
RUN cd /root && \
|
||||
wget -q http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz -O - | tar xvfz - && \
|
||||
cd swig-3.0.10 && \
|
||||
ARG SWIG_VERSION=3.0.10
|
||||
ARG CACHEBUST=1
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/swig-$SWIG_VERSION.tar.gz -O - | tar xvfz - && \
|
||||
cd swig-$SWIG_VERSION && \
|
||||
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
|
||||
./configure --without-alllang && \
|
||||
./configure --prefix=/usr/local/swig-$SWIG_VERSION --without-perl5 --without-alllang && \
|
||||
make -j $(nproc) && \
|
||||
make install
|
||||
|
||||
make install && \
|
||||
cd .. && \
|
||||
rm -rf swig-$SWIG_VERSION
|
||||
COPY ./Patches /tmp/patches
|
||||
RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
|
||||
rm -rfd /tmp/patches
|
||||
|
||||
# .NET Core SDK
|
||||
RUN cd /tmp && \
|
||||
(wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/packages-microsoft-prod/deb/packages-microsoft-prod.deb && \
|
||||
dpkg -i packages-microsoft-prod.deb && \
|
||||
apt-get update && \
|
||||
apt-get --yes install dotnet-sdk-2.1.200 && \
|
||||
rm /tmp/packages-microsoft-prod.deb && \
|
||||
apt-get --yes autoremove && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
apt-get -y -f install apt-transport-https && \
|
||||
apt-get -y update && \
|
||||
apt-get -y -f install dotnet-sdk-2.1 && \
|
||||
rm ./packages-microsoft-prod.deb
|
||||
|
||||
# Anaconda
|
||||
RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \
|
||||
bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \
|
||||
rm Anaconda3-4.2.0-Linux-x86_64.sh
|
||||
ARG ANACONDA_VERSION=4.2.0
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/$ANACONDA_VERSION/Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh && \
|
||||
bash Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh -b && \
|
||||
rm Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh
|
||||
|
||||
RUN CONDA_ENV_PATH=/tmp/conda-linux-cntk-py35-environment.yml; \
|
||||
wget -q https://raw.githubusercontent.com/Microsoft/CNTK/$COMMIT/Scripts/install/linux/conda-linux-cntk-py35-environment.yml -O "$CONDA_ENV_PATH" && \
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
ARG CMAKE_DOWNLOAD_VERSION=3.11
|
||||
ARG CMAKE_BUILD_VERSION=4
|
||||
RUN DEBIAN_FRONTEND=noninteractive && \
|
||||
wget --no-verbose https://cmake.org/files/v$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
wget --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cmake/$CMAKE_DOWNLOAD_VERSION/cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
tar -xzvf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION.tar.gz && \
|
||||
cd cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION && \
|
||||
./bootstrap --system-curl -- -DCMAKE_USE_OPENSSL=ON && \
|
||||
|
|
@ -64,58 +64,65 @@ RUN DEBIAN_FRONTEND=noninteractive && \
|
|||
cd .. && \
|
||||
rm -rf cmake-$CMAKE_DOWNLOAD_VERSION.$CMAKE_BUILD_VERSION
|
||||
|
||||
RUN OPENMPI_VERSION=1.10.3 && \
|
||||
wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENMPI_VERSION=1.10.7
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openmpi/$OPENMPI_VERSION/openmpi-$OPENMPI_VERSION.tar.gz | tar -xzf - && \
|
||||
cd openmpi-${OPENMPI_VERSION} && \
|
||||
./configure --prefix=/usr/local/mpi && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /openmpi-${OPENMPI_VERSION}
|
||||
|
||||
apt-get -y update && \
|
||||
apt-get -y -f install && \
|
||||
apt-get -y install libsysfs2 libsysfs-dev && \
|
||||
./configure --with-verbs --with-cuda=/usr/local/cuda --prefix=/usr/local/mpi && \
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf openmpi-${OPENMPI_VERSION}
|
||||
ENV PATH /usr/local/mpi/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH /usr/local/mpi/lib:$LD_LIBRARY_PATH
|
||||
|
||||
RUN LIBZIP_VERSION=1.1.2 && \
|
||||
wget -q -O - https://libzip.org/download/libzip-${LIBZIP_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG LIBZIP_VERSION=1.1.2
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/libzip/$LIBZIP_VERSION/libzip-$LIBZIP_VERSION.tar.gz | tar -xzf - && \
|
||||
cd libzip-${LIBZIP_VERSION} && \
|
||||
./configure && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /libzip-${LIBZIP_VERSION}
|
||||
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf libzip-${LIBZIP_VERSION}
|
||||
ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
|
||||
|
||||
RUN wget -q -O - https://github.com/NVlabs/cub/archive/1.8.0.tar.gz | tar -C /usr/local -xzf -
|
||||
ARG CUB_VERSION=1.8.0
|
||||
RUN wget https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/cub/$CUB_VERSION/cub-$CUB_VERSION.zip && \
|
||||
unzip -d /usr/local cub-${CUB_VERSION}.zip && \
|
||||
rm cub-${CUB_VERSION}.zip
|
||||
|
||||
RUN OPENCV_VERSION=3.1.0 && \
|
||||
wget -q -O - https://github.com/Itseez/opencv/archive/${OPENCV_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENCV_VERSION=3.1.0
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/opencv/$OPENCV_VERSION/opencv-$OPENCV_VERSION.tar.gz | tar -xzf - && \
|
||||
cd opencv-${OPENCV_VERSION} && \
|
||||
cmake -DWITH_CUDA=OFF -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local/opencv-${OPENCV_VERSION} . && \
|
||||
make -j"$(nproc)" install && \
|
||||
rm -rf /opencv-${OPENCV_VERSION}
|
||||
make -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf opencv-${OPENCV_VERSION}
|
||||
|
||||
RUN OPENBLAS_VERSION=0.2.18 && \
|
||||
wget -q -O - https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG OPENBLAS_VERSION=0.2.18
|
||||
RUN wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/openblas/$OPENBLAS_VERSION/OpenBLAS-$OPENBLAS_VERSION.tar.gz | tar -xzf - && \
|
||||
cd OpenBLAS-${OPENBLAS_VERSION} && \
|
||||
make -j"$(nproc)" USE_OPENMP=1 | tee make.log && \
|
||||
make -j 2 MAKE_NB_JOBS=0 USE_OPENMP=1 | tee make.log && \
|
||||
grep -qF 'OpenBLAS build complete. (BLAS CBLAS LAPACK LAPACKE)' make.log && \
|
||||
grep -qF 'Use OpenMP in the multithreading.' make.log && \
|
||||
make PREFIX=/usr/local/openblas install && \
|
||||
rm -rf /OpenBLAS-${OPENBLAS_VERSION}
|
||||
|
||||
cd .. && \
|
||||
rm -rf OpenBLAS-${OPENBLAS_VERSION}
|
||||
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
|
||||
|
||||
# Install Boost
|
||||
RUN BOOST_VERSION=1_60_0 && \
|
||||
BOOST_DOTTED_VERSION=$(echo $BOOST_VERSION | tr _ .) && \
|
||||
wget -q -O - https://sourceforge.net/projects/boost/files/boost/${BOOST_DOTTED_VERSION}/boost_${BOOST_VERSION}.tar.gz/download | tar -xzf - && \
|
||||
cd boost_${BOOST_VERSION} && \
|
||||
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_DOTTED_VERSION} --with-libraries=filesystem,system,test && \
|
||||
./b2 -d0 -j"$(nproc)" install && \
|
||||
rm -rf /boost_${BOOST_VERSION}
|
||||
ARG BOOST_VERSION=1.60.0
|
||||
RUN BOOST_UNDERSCORE_VERSION=$(echo $BOOST_VERSION | tr . _) && \
|
||||
wget -q -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/boost/$BOOST_VERSION/boost_$BOOST_UNDERSCORE_VERSION.tar.gz | tar -xzf - && \
|
||||
cd boost_${BOOST_UNDERSCORE_VERSION} && \
|
||||
./bootstrap.sh --prefix=/usr/local/boost-${BOOST_VERSION} && \
|
||||
./b2 -d0 -j $(nproc) install && \
|
||||
cd .. && \
|
||||
rm -rf boost_${BOOST_UNDERSCORE_VERSION}
|
||||
|
||||
# Install Protobuf
|
||||
RUN PROTOBUF_VERSION=3.1.0 \
|
||||
PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
|
||||
wget -O - --no-verbose https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar -xzf - && \
|
||||
ARG PROTOBUF_VERSION=3.1.0
|
||||
RUN PROTOBUF_STRING=protobuf-$PROTOBUF_VERSION && \
|
||||
wget -O - --no-verbose https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/protobuf/$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz | tar -xzf - && \
|
||||
cd $PROTOBUF_STRING && \
|
||||
./autogen.sh && \
|
||||
./configure CFLAGS=-fPIC CXXFLAGS=-fPIC --disable-shared --prefix=/usr/local/$PROTOBUF_STRING && \
|
||||
|
|
@ -127,10 +134,10 @@ RUN PROTOBUF_VERSION=3.1.0 \
|
|||
ARG MKLDNN_VERSION=0.14
|
||||
ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
|
||||
RUN mkdir /usr/local/mklml && \
|
||||
wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/$MKLDNN_LONG_VERSION.tgz | \
|
||||
tar -xzf - -C /usr/local/mklml && \
|
||||
MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
|
||||
wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/mkl-dnn/$MKLDNN_VERSION/mkl-dnn-$MKLDNN_VERSION.tar.gz | tar -xzf - && \
|
||||
cd ${MKLDNN_STRING} && \
|
||||
ln -s /usr/local external && \
|
||||
mkdir -p build && \
|
||||
|
|
@ -143,13 +150,12 @@ RUN mkdir /usr/local/mklml && \
|
|||
rm -rf ${MKLDNN_STRING}
|
||||
|
||||
# Install Kaldi
|
||||
ENV KALDI_VERSION=c024e8aa
|
||||
ENV KALDI_PATH /usr/local/kaldi-$KALDI_VERSION
|
||||
|
||||
ARG KALDI_VERSION=c024e8aa
|
||||
ARG KALDI_PATH=/usr/local/kaldi-$KALDI_VERSION
|
||||
RUN mv /bin/sh /bin/sh.orig && \
|
||||
ln -s -f /bin/bash /bin/sh && \
|
||||
mkdir $KALDI_PATH && \
|
||||
wget --no-verbose -O - https://github.com/kaldi-asr/kaldi/archive/$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
|
||||
wget --no-verbose -O - https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/kaldi/$KALDI_VERSION/kaldi-$KALDI_VERSION.tar.gz | tar -xzf - --strip-components=1 -C $KALDI_PATH && \
|
||||
cd $KALDI_PATH && \
|
||||
cd tools && \
|
||||
perl -pi -e 's/^# (OPENFST_VERSION = 1.4.1)$/\1/' Makefile && \
|
||||
|
|
@ -172,32 +178,33 @@ RUN mv /bin/sh /bin/sh.orig && \
|
|||
ARG COMMIT=master
|
||||
|
||||
# Swig
|
||||
RUN cd /root && \
|
||||
wget -q http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz -O - | tar xvfz - && \
|
||||
cd swig-3.0.10 && \
|
||||
ARG SWIG_VERSION=3.0.10
|
||||
ARG CACHEBUST=1
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/swig/$SWIG_VERSION/swig-$SWIG_VERSION.tar.gz -O - | tar xvfz - && \
|
||||
cd swig-$SWIG_VERSION && \
|
||||
# Note: we specify --without-alllang to suppress building tests and examples for specific languages.
|
||||
./configure --without-alllang && \
|
||||
./configure --prefix=/usr/local/swig-$SWIG_VERSION --without-perl5 --without-alllang && \
|
||||
make -j $(nproc) && \
|
||||
make install
|
||||
|
||||
make install && \
|
||||
cd .. && \
|
||||
rm -rf swig-$SWIG_VERSION
|
||||
COPY ./Patches /tmp/patches
|
||||
RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
|
||||
rm -rfd /tmp/patches
|
||||
|
||||
# .NET Core SDK
|
||||
RUN cd /tmp && \
|
||||
(wget -q packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb || true) && \
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/packages-microsoft-prod/deb/packages-microsoft-prod.deb && \
|
||||
dpkg -i packages-microsoft-prod.deb && \
|
||||
apt-get update && \
|
||||
apt-get --yes install dotnet-sdk-2.1.200 && \
|
||||
rm /tmp/packages-microsoft-prod.deb && \
|
||||
apt-get --yes autoremove && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
apt-get -y -f install apt-transport-https && \
|
||||
apt-get -y update && \
|
||||
apt-get -y -f install dotnet-sdk-2.1 && \
|
||||
rm ./packages-microsoft-prod.deb
|
||||
|
||||
# Anaconda
|
||||
RUN wget -q https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh && \
|
||||
bash Anaconda3-4.2.0-Linux-x86_64.sh -b && \
|
||||
rm Anaconda3-4.2.0-Linux-x86_64.sh
|
||||
ARG ANACONDA_VERSION=4.2.0
|
||||
RUN wget -q https://cntkbuildstorage.blob.core.windows.net/cntk-ci-dependencies/anaconda3/$ANACONDA_VERSION/Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh && \
|
||||
bash Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh -b && \
|
||||
rm Anaconda3-$ANACONDA_VERSION-Linux-x86_64.sh
|
||||
|
||||
RUN CONDA_ENV_PATH=/tmp/conda-linux-cntk-py35-environment.yml; \
|
||||
wget -q https://raw.githubusercontent.com/Microsoft/CNTK/$COMMIT/Scripts/install/linux/conda-linux-cntk-py35-environment.yml -O "$CONDA_ENV_PATH" && \
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче