305 строки
8.0 KiB
Bash
Executable File
305 строки
8.0 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Copyright (c) Microsoft. All rights reserved.
|
|
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
#
|
|
# WARNING. This will run in Microsoft Internal Environment ONLY, to generate CNTK binary drops.
|
|
|
|
scriptName="$(basename "${BASH_SOURCE[0]}")"
|
|
scriptDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
|
|
|
|
parsedArgs=$(getopt -o '' --long target-configuration:,verbose,release-tag:,commit: -n "$scriptName" -- "$@")
|
|
|
|
[ $? != 0 ] && {
|
|
echo Terminating...
|
|
exit 1
|
|
}
|
|
|
|
eval set -- "$parsedArgs"
|
|
|
|
targetConfiguration=
|
|
verbose= # TODO
|
|
releaseTag=unknown
|
|
|
|
while true; do
|
|
case "$1" in
|
|
--target-configuration)
|
|
targetConfiguration="${2,,}"
|
|
shift 2
|
|
;;
|
|
--commit)
|
|
commit="${2,,}"
|
|
shift 2
|
|
;;
|
|
--release-tag)
|
|
releaseTag="${2,,}"
|
|
shift 2
|
|
;;
|
|
--verbose)
|
|
verbose=1
|
|
shift
|
|
;;
|
|
--)
|
|
shift
|
|
break
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[ $# = 0 ] || {
|
|
echo Extra parameters detected: $*
|
|
exit 1
|
|
}
|
|
|
|
[[ -n $commit ]] || {
|
|
echo Must specify --commit option.
|
|
exit 1
|
|
}
|
|
|
|
case $targetConfiguration in
|
|
cpu)
|
|
publicTargetConfiguration=CPU-Only
|
|
;;
|
|
gpu)
|
|
publicTargetConfiguration=GPU
|
|
;;
|
|
'')
|
|
echo Must specify --target-configuration option.
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo Unknown target configuration $targetConfiguration.
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
[[ $releaseTag = unknown ]] || [[ $releaseTag =~ ^[1-9][0-9a-z-]*$ ]] || {
|
|
echo \'--release-tag $releaseTag\' option does not match expected format.
|
|
exit 1
|
|
}
|
|
|
|
outputFile=CNTK-$releaseTag-Linux-64bit-$publicTargetConfiguration.tar.gz
|
|
|
|
# Stop on Error
|
|
set -e -o pipefail
|
|
|
|
# Enable verbose mode if needed
|
|
# stderr is NOT changed
|
|
if [[ -n $verbose ]]; then
|
|
exec 3>&1
|
|
else
|
|
exec 3>/dev/null
|
|
fi
|
|
|
|
# Define helper function
|
|
|
|
# File List Copy function
|
|
# usage: CopyFilesFromList source_path file_name_array destination_path
|
|
function CopyFilesFromList ()
|
|
{
|
|
declare -a fileNames=(${!2})
|
|
for fileName in "${fileNames[@]}"
|
|
do
|
|
cp -p "$1/$fileName" "$3"
|
|
done
|
|
}
|
|
|
|
# Main script
|
|
|
|
echo "Making binary drops..." >&3
|
|
|
|
cd "$scriptDir/.."
|
|
|
|
# Dependency files
|
|
|
|
# MKL
|
|
declare -a mklFiles=("libmklml_intel.so" "libiomp5.so" "libmkldnn.so.0")
|
|
|
|
# Open CV
|
|
declare -a opencvFiles=("libopencv_core.so.3.1" "libopencv_imgproc.so.3.1" "libopencv_imgproc.so.3.1" "libopencv_imgcodecs.so.3.1")
|
|
|
|
# libzip
|
|
declare -a libzipFiles=("libzip.so.4")
|
|
|
|
# CUDA
|
|
declare -a cudaFiles=("libcudart.so.9.0" "libcublas.so.9.0" "libcurand.so.9.0" "libcusparse.so.9.0")
|
|
|
|
# cuDNN
|
|
# Note: can be only a single file currently, see copy below.
|
|
declare -a cudnnFiles=("libcudnn.so")
|
|
|
|
# NCCL
|
|
declare -a ncclFiles=("libnccl.so.2.1.2")
|
|
|
|
# OpenBLAS (Needed by Kaldi)
|
|
declare -a openblasFiles=("libopenblas.so.0")
|
|
|
|
# Kaldi
|
|
declare -a kaldiFiles=("libkaldi-util.so" "libkaldi-matrix.so" "libkaldi-base.so" "libkaldi-hmm.so" "libkaldi-cudamatrix.so" "libkaldi-nnet.so" "libkaldi-lat.so" "libkaldi-tree.so")
|
|
|
|
# OpenFst (from Kaldi)
|
|
declare -a openfstFiles=("libfst.so.3")
|
|
|
|
# Include files
|
|
declare -a includeFiles=("Eval.h")
|
|
|
|
# Include files 2.0
|
|
declare -a includeFiles20=("CNTKLibrary.h" "CNTKLibraryInternals.h" "CNTKLibraryC.h" "HalfConverter.hpp")
|
|
declare -a includeInternalFiles20=("ComputationGraphAlgorithms.h" "EvaluatorWrapper.h" "PrimitiveFunctionAttribute.h" "PrimitiveFunction.h" "PrimitiveOpType.h")
|
|
|
|
# Set dependency sources paths
|
|
mklPath="/usr/local/lib"
|
|
opencvVersion="3.1.0"
|
|
opencvPath="/usr/local/opencv-$opencvVersion/lib"
|
|
libzipPath="/usr/local/lib"
|
|
cudaPath="/usr/local/cuda/lib64"
|
|
cudnnPath="/usr/local/cudnn-7.0/cuda/lib64"
|
|
ncclPath="/usr/lib/x86_64-linux-gnu"
|
|
openblasPath="/usr/local/openblas/lib"
|
|
kaldiVersion="c024e8aa"
|
|
kaldiPath="/usr/local/kaldi-$kaldiVersion/src/lib"
|
|
openfstPath="/usr/local/kaldi-$kaldiVersion/tools/openfst/lib"
|
|
|
|
# Set build paths
|
|
buildPath="build/$targetConfiguration/release"
|
|
basePath="BinaryDrops"
|
|
baseDropPath="$basePath/cntk"
|
|
baseBinariesPath="$baseDropPath/cntk"
|
|
baseDependenciesPath="$baseBinariesPath/dependencies/lib"
|
|
baseIncludePath="$baseDropPath/Include"
|
|
baseIncludeInternalPath="$baseIncludePath/Internals"
|
|
includePath="Source/Common/Include"
|
|
includePath20="Source/CNTKv2LibraryDll/API"
|
|
includeInternalPath20="$includePath20/Internals"
|
|
extrasPath="Tools/cntk-binary-drop/linux/$targetConfiguration"
|
|
|
|
# Make BinaryDrops directory
|
|
mkdir -p $baseBinariesPath
|
|
|
|
echo "Copying Python wheels..." >&3
|
|
mkdir $basePath/$publicTargetConfiguration
|
|
cp -p $buildPath/python/*.whl $basePath/$publicTargetConfiguration
|
|
|
|
echo "Copying build binaries..." >&3
|
|
cp -pr $buildPath/* $baseBinariesPath
|
|
# Remove unnecessary file(s) if exist(s)
|
|
#
|
|
# General TODO: Implement White List of Binary Drop contents.
|
|
# For the time being "cherry pick" removal of unneeded files
|
|
#
|
|
rm -f $baseBinariesPath/bin/brainscripttests
|
|
rm -f $baseBinariesPath/bin/cppevalclient
|
|
rm -f $baseBinariesPath/bin/cppevalextendedclient
|
|
rm -f $baseBinariesPath/bin/cppevalv2client
|
|
rm -f $baseBinariesPath/bin/evaltests
|
|
rm -f $baseBinariesPath/bin/mathtests
|
|
rm -f $baseBinariesPath/bin/multiversotests
|
|
rm -f $baseBinariesPath/bin/networktests
|
|
rm -f $baseBinariesPath/bin/readertests
|
|
rm -f $baseBinariesPath/bin/V2LibraryEndToEndTests
|
|
rm -f $baseBinariesPath/bin/v2librarytests
|
|
rm -rf $baseBinariesPath/python
|
|
rm -f $baseBinariesPath/lib/java/Main.class
|
|
|
|
# Make Include directory
|
|
mkdir -p $baseIncludePath
|
|
mkdir -p $baseIncludeInternalPath
|
|
|
|
# Copy Include
|
|
echo "Copying Include files..." >&3
|
|
CopyFilesFromList $includePath includeFiles[@] $baseIncludePath
|
|
echo "Copying Include files for Version 2..." >&3
|
|
CopyFilesFromList $includePath20 includeFiles20[@] $baseIncludePath
|
|
echo "Copying Include internal files for Version 2..." >&3
|
|
CopyFilesFromList $includeInternalPath20 includeInternalFiles20[@] $baseIncludeInternalPath
|
|
|
|
# Copy Examples
|
|
echo "Copying Examples..." >&3
|
|
cp -pr Examples $baseDropPath
|
|
|
|
# Copy Tutorials
|
|
echo "Copying Tutorials..." >&3
|
|
cp -pr Tutorials $baseDropPath
|
|
|
|
# Copy PretrainedModels
|
|
echo "Copying PretrainedModels..." >&3
|
|
cp -pr PretrainedModels $baseDropPath
|
|
|
|
# Copy Manual
|
|
echo "Copying Manual..." >&3
|
|
cp -pr Manual $baseDropPath
|
|
|
|
# Copy Scripts (Scripts folder from the root of the Repo)
|
|
echo "Copying Scripts..." >&3
|
|
cp -pr Scripts $baseDropPath
|
|
# Remove some unneeded files
|
|
rm -f $baseDropPath/Scripts/pytest.ini
|
|
rm -rf $baseDropPath/Scripts/install/windows
|
|
|
|
# Copy Extras
|
|
echo "Copying Extras..." >&3
|
|
cp -pr $extrasPath/* $baseDropPath
|
|
|
|
# Copy Dependencies
|
|
echo "Copying Dependencies..." >&3
|
|
|
|
# Make dependencies directory
|
|
mkdir -p $baseDependenciesPath
|
|
|
|
# Copy MKL
|
|
echo "Copying MKL" >&3
|
|
CopyFilesFromList $mklPath mklFiles[@] $baseDependenciesPath
|
|
|
|
# Copy Open CV
|
|
echo "Copying Open CV..." >&3
|
|
CopyFilesFromList $opencvPath opencvFiles[@] $baseDependenciesPath
|
|
|
|
# Copy libzip
|
|
echo "Copying libzip..." >&3
|
|
CopyFilesFromList $libzipPath libzipFiles[@] $baseDependenciesPath
|
|
|
|
# Copy OpenBLAS (for Kaldi)
|
|
echo "Copying OpenBLAS (for Kaldi)..." >&3
|
|
CopyFilesFromList $openblasPath openblasFiles[@] $baseDependenciesPath
|
|
|
|
# Copy Kaldi
|
|
echo "Copying Kaldi..." >&3
|
|
CopyFilesFromList $kaldiPath kaldiFiles[@] $baseDependenciesPath
|
|
|
|
# Copy OpenFst (from Kaldi)
|
|
echo "Copying OpenFst (Kaldi)..." >&3
|
|
CopyFilesFromList $openfstPath openfstFiles[@] $baseDependenciesPath
|
|
|
|
# GPU Drops only
|
|
if [[ $targetConfiguration != "cpu" ]]; then
|
|
|
|
# Copy CUDA
|
|
echo "Copying CUDA..." >&3
|
|
CopyFilesFromList $cudaPath cudaFiles[@] $baseDependenciesPath
|
|
|
|
# Copy cuDNN
|
|
echo "Copying cuDNN..." >&3
|
|
CopyFilesFromList $cudnnPath cudnnFiles[@] $baseDependenciesPath/libcudnn.so.7
|
|
|
|
# Copy NCCL
|
|
echo "Copying NCCL..." >&3
|
|
CopyFilesFromList $ncclPath ncclFiles[@] $baseDependenciesPath/libnccl.so.2
|
|
|
|
fi
|
|
|
|
echo "Creating version.txt file" >&3
|
|
printf 'CNTK-%s\nRelease\n%s\n%s\n' \
|
|
"$releaseTag" "$publicTargetConfiguration" "$commit" \
|
|
> $baseDropPath/version.txt
|
|
|
|
echo "Making Archive and cleaning up..." >&3
|
|
# Make GZipped TAR
|
|
cd $basePath
|
|
tar --owner=root ${verbose:+-v} -czf "$outputFile" cntk
|
|
|
|
# Log some file hashes
|
|
sha256sum "$outputFile" $publicTargetConfiguration/*.whl
|
|
|
|
# Remove TAR sources
|
|
rm -r cntk
|