Linux: CNTK v2 Python build support through Makefile
This commit is contained in:
Mark Hillebrand
Родитель
f78cb8b102
Коммит
ddb1434e50
118
Makefile
118
Makefile
|
|
@ -35,6 +35,12 @@
|
|||
# defaults to /usr/local/
|
||||
# BOOST_PATH= path to Boost installation, so $(BOOST_PATH)/include/boost/test/unit_test.hpp
|
||||
# defaults to /usr/local/boost-1.60.0
|
||||
# PYTHON_SUPPORT=true iff CNTK v2 Python module should be build
|
||||
# SWIG_PATH= path to SWIG (>= 3.0.10)
|
||||
# PYTHON_VERSIONS= list of Python versions to build for
|
||||
# A Python version is identified by "34" or "35".
|
||||
# PYTHON34_PATH= path to Python 3.4 interpreter
|
||||
# PYTHON35_PATH= path to Python 3.5 interpreter
|
||||
# These can be overridden on the command line, e.g. make BUILDTYPE=debug
|
||||
|
||||
# TODO: Build static libraries for common dependencies that are shared by multiple
|
||||
|
|
@ -75,7 +81,7 @@ COMMON_FLAGS:= -D_POSIX_SOURCE -D_XOPEN_SOURCE=600 -D__USE_XOPEN2K -std=c++11
|
|||
CPPFLAGS:=
|
||||
CXXFLAGS:= -msse4.1 -mssse3 -std=c++0x -fopenmp -fpermissive -fPIC -Werror -fcheck-new
|
||||
LIBPATH:=
|
||||
LIBS:=
|
||||
LIBS_LIST:=
|
||||
LDFLAGS:=
|
||||
|
||||
CXXVER_GE480:= $(shell expr `$(CXX) -dumpversion | sed -e 's/\.\([0-9][0-9]\)/\1/g' -e 's/\.\([0-9]\)/0\1/g' -e 's/^[0-9]\{3,4\}$$/&00/'` \>= 40800)
|
||||
|
|
@ -85,6 +91,8 @@ endif
|
|||
|
||||
SEPARATOR = "=-----------------------------------------------------------="
|
||||
ALL:=
|
||||
ALL_LIBS:=
|
||||
LIBS_FULLPATH:=
|
||||
SRC:=
|
||||
|
||||
# Make sure all is the first (i.e. default) target, but we can't actually define it
|
||||
|
|
@ -120,13 +128,13 @@ ifdef CUDA_PATH
|
|||
# Set up CUDA includes and libraries
|
||||
INCLUDEPATH += $(CUDA_PATH)/include
|
||||
LIBPATH += $(CUDA_PATH)/lib64
|
||||
LIBS += -lcublas -lcudart -lcuda -lcurand -lcusparse -lnvidia-ml
|
||||
LIBS_LIST += cublas cudart cuda curand cusparse nvidia-ml
|
||||
|
||||
# Set up cuDNN if needed
|
||||
ifdef CUDNN_PATH
|
||||
INCLUDEPATH += $(CUDNN_PATH)/cuda/include
|
||||
LIBPATH += $(CUDNN_PATH)/cuda/lib64
|
||||
LIBS += -lcudnn
|
||||
LIBS_LIST += cudnn
|
||||
COMMON_FLAGS +=-DUSE_CUDNN
|
||||
endif
|
||||
else
|
||||
|
|
@ -137,13 +145,13 @@ endif
|
|||
|
||||
ifeq ("$(MATHLIB)","mkl")
|
||||
INCLUDEPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/include
|
||||
LIBS += -lm
|
||||
LIBS_LIST += m
|
||||
ifeq ("$(MKL_THREADING)","sequential")
|
||||
LIBPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/x64/sequential
|
||||
LIBS += -lmkl_cntk_s
|
||||
LIBS_LIST += mkl_cntk_s
|
||||
else
|
||||
LIBPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/x64/parallel
|
||||
LIBS += -lmkl_cntk_p -liomp5 -lpthread
|
||||
LIBS_LIST += mkl_cntk_p iomp5 pthread
|
||||
endif
|
||||
COMMON_FLAGS += -DUSE_MKL
|
||||
endif
|
||||
|
|
@ -151,7 +159,7 @@ endif
|
|||
ifeq ("$(MATHLIB)","openblas")
|
||||
INCLUDEPATH += $(OPENBLAS_PATH)/include
|
||||
LIBPATH += $(OPENBLAS_PATH)/lib
|
||||
LIBS += -lopenblas -lm -lpthread
|
||||
LIBS_LIST += openblas m pthread
|
||||
CPPFLAGS += -DUSE_OPENBLAS
|
||||
endif
|
||||
|
||||
|
|
@ -162,10 +170,11 @@ ifdef KALDI_PATH
|
|||
ATLASINC = $(KALDI_PATH)/tools/ATLAS/include
|
||||
|
||||
INCLUDEPATH += $(KALDI_PATH)/src $(ATLASINC) $(FSTROOT)/include
|
||||
CPPFLAGS+= -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400
|
||||
CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400
|
||||
|
||||
KALDI_LIBPATH += $(KALDI_PATH)/src/lib
|
||||
KALDI_LIBS += -lkaldi-util -lkaldi-matrix -lkaldi-base -lkaldi-hmm -lkaldi-cudamatrix -lkaldi-nnet -lkaldi-lat
|
||||
KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat
|
||||
KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST))
|
||||
endif
|
||||
|
||||
ifdef SUPPORT_AVX2
|
||||
|
|
@ -219,11 +228,14 @@ ifdef CNTK_CUDA_DEVICE_DEBUGINFO
|
|||
CUFLAGS += -G
|
||||
endif
|
||||
|
||||
#######
|
||||
# Create the library link options for the linker.
|
||||
# LIBS_LIST must not be changed beyond this point.
|
||||
LIBS:= $(addprefix -l,$(LIBS_LIST))
|
||||
|
||||
OBJDIR:= $(BUILD_TOP)/.build
|
||||
BINDIR:= $(BUILD_TOP)/bin
|
||||
LIBDIR:= $(BUILD_TOP)/lib
|
||||
PYTHONDIR:= $(BUILD_TOP)/python
|
||||
|
||||
ORIGINLIBDIR:='$$ORIGIN/../lib'
|
||||
ORIGINDIR:='$$ORIGIN'
|
||||
|
|
@ -324,7 +336,7 @@ MATH_SRC+=$(READER_SRC)
|
|||
MATH_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(MATH_SRC)))
|
||||
|
||||
CNTKMATH_LIB:= $(LIBDIR)/lib$(CNTKMATH).so
|
||||
ALL += $(CNTKMATH_LIB)
|
||||
ALL_LIBS += $(CNTKMATH_LIB)
|
||||
SRC+=$(MATH_SRC)
|
||||
|
||||
$(CNTKMATH_LIB): $(MATH_OBJ)
|
||||
|
|
@ -397,7 +409,7 @@ CNTKLIBRARY:=cntklibrary-$(CNTKLIBRARY_VERSION)
|
|||
CNTKLIBRARY_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(CNTKLIBRARY_SRC)))
|
||||
|
||||
CNTKLIBRARY_LIB:=$(LIBDIR)/lib$(CNTKLIBRARY).so
|
||||
ALL+=$(CNTKLIBRARY_LIB)
|
||||
ALL_LIBS+=$(CNTKLIBRARY_LIB)
|
||||
SRC+=$(CNTKLIBRARY_SRC)
|
||||
|
||||
$(CNTKLIBRARY_LIB): $(CNTKLIBRARY_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -469,7 +481,7 @@ EVAL_SRC+=$(SEQUENCE_TRAINING_LIB_SRC)
|
|||
EVAL_OBJ:=$(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(EVAL_SRC)))
|
||||
|
||||
EVAL_LIB:=$(LIBDIR)/lib$(EVAL).so
|
||||
ALL+=$(EVAL_LIB)
|
||||
ALL_LIBS+=$(EVAL_LIB)
|
||||
SRC+=$(EVAL_SRC)
|
||||
|
||||
$(EVAL_LIB): $(EVAL_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -510,7 +522,7 @@ BINARYREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(BINARYREADER_SRC))
|
|||
|
||||
BINARY_READER:= $(LIBDIR)/BinaryReader.so
|
||||
|
||||
#ALL += $(BINARY_READER)
|
||||
#ALL_LIBS += $(BINARY_READER)
|
||||
#SRC+=$(BINARYREADER_SRC)
|
||||
|
||||
$(BINARY_READER): $(BINARYREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -530,7 +542,7 @@ HTKMLFREADER_SRC =\
|
|||
HTKMLFREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(HTKMLFREADER_SRC))
|
||||
|
||||
HTKMLFREADER:=$(LIBDIR)/HTKMLFReader.so
|
||||
ALL+=$(HTKMLFREADER)
|
||||
ALL_LIBS+=$(HTKMLFREADER)
|
||||
SRC+=$(HTKMLFREADER_SRC)
|
||||
|
||||
$(LIBDIR)/HTKMLFReader.so: $(HTKMLFREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -548,7 +560,7 @@ COMPOSITEDATAREADER_SRC =\
|
|||
COMPOSITEDATAREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(COMPOSITEDATAREADER_SRC))
|
||||
|
||||
COMPOSITEDATAREADER:=$(LIBDIR)/CompositeDataReader.so
|
||||
ALL+=$(COMPOSITEDATAREADER)
|
||||
ALL_LIBS+=$(COMPOSITEDATAREADER)
|
||||
SRC+=$(COMPOSITEDATAREADER_SRC)
|
||||
|
||||
$(LIBDIR)/CompositeDataReader.so: $(COMPOSITEDATAREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -571,7 +583,7 @@ HTKDESERIALIZERS_SRC =\
|
|||
HTKDESERIALIZERS_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(HTKDESERIALIZERS_SRC))
|
||||
|
||||
HTKDESERIALIZERS:=$(LIBDIR)/HTKDeserializers.so
|
||||
ALL+=$(HTKDESERIALIZERS)
|
||||
ALL_LIBS+=$(HTKDESERIALIZERS)
|
||||
SRC+=$(HTKDESERIALIZERS_SRC)
|
||||
|
||||
$(LIBDIR)/HTKDeserializers.so: $(HTKDESERIALIZERS_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -591,7 +603,7 @@ LMSEQUENCEREADER_SRC =\
|
|||
LMSEQUENCEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LMSEQUENCEREADER_SRC))
|
||||
|
||||
LMSEQUENCEREADER:= $(LIBDIR)/LMSequenceReader.so
|
||||
ALL+=$(LMSEQUENCEREADER)
|
||||
ALL_LIBS+=$(LMSEQUENCEREADER)
|
||||
SRC+=$(LMSEQUENCEREADER_SRC)
|
||||
|
||||
$(LMSEQUENCEREADER): $(LMSEQUENCEREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -612,7 +624,7 @@ LUSEQUENCEREADER_SRC =\
|
|||
LUSEQUENCEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LUSEQUENCEREADER_SRC))
|
||||
|
||||
LUSEQUENCEREADER:=$(LIBDIR)/LUSequenceReader.so
|
||||
ALL+=$(LUSEQUENCEREADER)
|
||||
ALL_LIBS+=$(LUSEQUENCEREADER)
|
||||
SRC+=$(LUSEQUENCEREADER_SRC)
|
||||
|
||||
$(LUSEQUENCEREADER): $(LUSEQUENCEREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -631,7 +643,7 @@ UCIFASTREADER_SRC =\
|
|||
UCIFASTREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UCIFASTREADER_SRC))
|
||||
|
||||
UCIFASTREADER:=$(LIBDIR)/UCIFastReader.so
|
||||
ALL += $(UCIFASTREADER)
|
||||
ALL_LIBS += $(UCIFASTREADER)
|
||||
SRC+=$(UCIFASTREADER_SRC)
|
||||
|
||||
$(UCIFASTREADER): $(UCIFASTREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -649,7 +661,7 @@ LIBSVMBINARYREADER_SRC =\
|
|||
LIBSVMBINARYREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LIBSVMBINARYREADER_SRC))
|
||||
|
||||
LIBSVMBINARYREADER:=$(LIBDIR)/LibSVMBinaryReader.so
|
||||
ALL += $(LIBSVMBINARYREADER)
|
||||
ALL_LIBS += $(LIBSVMBINARYREADER)
|
||||
SRC+=$(LIBSVMBINARYREADER_SRC)
|
||||
|
||||
$(LIBSVMBINARYREADER): $(LIBSVMBINARYREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -667,7 +679,7 @@ SPARSEPCREADER_SRC =\
|
|||
SPARSEPCREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(SPARSEPCREADER_SRC))
|
||||
|
||||
SPARSEPCREADER:=$(LIBDIR)/SparsePCReader.so
|
||||
ALL += $(SPARSEPCREADER)
|
||||
ALL_LIBS += $(SPARSEPCREADER)
|
||||
SRC+=$(SPARSEPCREADER_SRC)
|
||||
|
||||
$(SPARSEPCREADER): $(SPARSEPCREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -688,7 +700,7 @@ CNTKTEXTFORMATREADER_SRC =\
|
|||
CNTKTEXTFORMATREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(CNTKTEXTFORMATREADER_SRC))
|
||||
|
||||
CNTKTEXTFORMATREADER:=$(LIBDIR)/CNTKTextFormatReader.so
|
||||
ALL += $(CNTKTEXTFORMATREADER)
|
||||
ALL_LIBS += $(CNTKTEXTFORMATREADER)
|
||||
SRC+=$(CNTKTEXTFORMATREADER_SRC)
|
||||
|
||||
$(CNTKTEXTFORMATREADER): $(CNTKTEXTFORMATREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -713,7 +725,7 @@ KALDI2READER_SRC = \
|
|||
KALDI2READER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(KALDI2READER_SRC))
|
||||
|
||||
KALDI2READER:=$(LIBDIR)/Kaldi2Reader.so
|
||||
ALL+=$(KALDI2READER)
|
||||
ALL_LIBS+=$(KALDI2READER)
|
||||
SRC+=$(KALDI2READER_SRC)
|
||||
|
||||
$(KALDI2READER): $(KALDI2READER_OBJ) | $(CNTKMATH_LIB)
|
||||
|
|
@ -731,17 +743,18 @@ ifdef BOOST_PATH
|
|||
|
||||
INCLUDEPATH += $(BOOST_PATH)/include
|
||||
|
||||
IMAGE_READER_LIBS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs
|
||||
IMAGEREADER_LIBS_LIST := opencv_core opencv_imgproc opencv_imgcodecs
|
||||
|
||||
ifdef LIBZIP_PATH
|
||||
CPPFLAGS += -DUSE_ZIP
|
||||
#both directories are needed for building libzip
|
||||
INCLUDEPATH += $(LIBZIP_PATH)/include
|
||||
INCLUDEPATH += $(LIBZIP_PATH)/lib/libzip/include
|
||||
IMAGE_READER_LIBS += -lzip
|
||||
# Both directories are needed for building libzip
|
||||
INCLUDEPATH += $(LIBZIP_PATH)/include $(LIBZIP_PATH)/lib/libzip/include
|
||||
LIBPATH += $(LIBZIP_PATH)/lib
|
||||
IMAGEREADER_LIBS_LIST += zip
|
||||
endif
|
||||
|
||||
IMAGEREADER_LIBS:= $(addprefix -l,$(IMAGEREADER_LIBS_LIST))
|
||||
|
||||
IMAGEREADER_SRC =\
|
||||
$(SOURCEDIR)/Readers/ImageReader/Exports.cpp \
|
||||
$(SOURCEDIR)/Readers/ImageReader/ImageConfigHelper.cpp \
|
||||
|
|
@ -753,7 +766,7 @@ IMAGEREADER_SRC =\
|
|||
IMAGEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(IMAGEREADER_SRC))
|
||||
|
||||
IMAGEREADER:=$(LIBDIR)/ImageReader.so
|
||||
ALL += $(IMAGEREADER)
|
||||
ALL_LIBS += $(IMAGEREADER)
|
||||
SRC+=$(IMAGEREADER_SRC)
|
||||
|
||||
INCLUDEPATH += $(OPENCV_PATH)/include
|
||||
|
|
@ -761,7 +774,7 @@ LIBPATH += $(OPENCV_PATH)/lib $(OPENCV_PATH)/release/lib
|
|||
|
||||
$(IMAGEREADER): $(IMAGEREADER_OBJ) | $(CNTKMATH_LIB)
|
||||
@echo $(SEPARATOR)
|
||||
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ -l$(CNTKMATH) $(IMAGE_READER_LIBS)
|
||||
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH) $(LIBPATH)) -o $@ $^ -l$(CNTKMATH) $(IMAGEREADER_LIBS)
|
||||
endif
|
||||
endif
|
||||
|
||||
|
|
@ -973,11 +986,52 @@ unittests: $(UNITTEST_EVAL) $(UNITTEST_READER) $(UNITTEST_NETWORK) $(UNITTEST_MA
|
|||
|
||||
endif
|
||||
|
||||
ifeq ("$(PYTHON_SUPPORT)","true")
|
||||
|
||||
# Libraries needed for the run-time (i.e., excluding test binaries)
|
||||
# TODO MPI doesn't appear explicitly here, hidden by mpic++ usage (but currently, it should be user installed)
|
||||
RUNTIME_LIBS_LIST := $(LIBS_LIST) $(IMAGEREADER_LIBS_LIST) $(KALDI_LIBS_LIST)
|
||||
RUNTIME_LIBS_EXCLUDE_LIST := m pthread nvidia-ml
|
||||
EXTRA_LIBS_BASENAMES:=$(addsuffix .so,$(addprefix lib,$(filter-out $(RUNTIME_LIBS_EXCLUDE_LIST),$(RUNTIME_LIBS_LIST))))
|
||||
|
||||
# TODO dependencies
|
||||
# TODO intermediate build results should go below $OBJDIR
|
||||
.PHONY: python
|
||||
python: $(ALL_LIBS)
|
||||
@bash -c '\
|
||||
set -x; \
|
||||
declare -A py_paths; \
|
||||
py_paths[34]=$(PYTHON34_PATH); \
|
||||
py_paths[35]=$(PYTHON35_PATH); \
|
||||
export LD_LIBRARY_PATH=$$(echo $(LIBPATH) $(KALDI_LIBPATH) | tr " " :); \
|
||||
! (ldd $(LIBDIR)/* | grep "not found") && \
|
||||
export CNTK_EXTRA_LIBRARIES=$$(ldd $(LIBDIR)/* | grep "^\s.*=> " | cut -d ">" -f 2- --only-delimited | cut -d "(" -f 1 --only-delimited | sort -u | grep -Ff <(echo $(EXTRA_LIBS_BASENAMES) | xargs -n1)) && \
|
||||
test -x $(SWIG_PATH) && \
|
||||
export CNTK_LIB_PATH=$$(readlink -f $(LIBDIR)) && \
|
||||
PYTHONDIR=$$(readlink -f $(PYTHONDIR)) && \
|
||||
test $$? -eq 0 && \
|
||||
cd bindings/python && \
|
||||
export PATH=$(SWIG_PATH):$$PATH; \
|
||||
for ver in $(PYTHON_VERSIONS); \
|
||||
do \
|
||||
test -x $${py_paths[$$ver]}; \
|
||||
$${py_paths[$$ver]} setup.py \
|
||||
build \
|
||||
bdist_wheel \
|
||||
--dist-dir $$PYTHONDIR || exit $$?; \
|
||||
done'
|
||||
|
||||
ALL += python
|
||||
|
||||
endif
|
||||
|
||||
########################################
|
||||
# General compile and dependency rules
|
||||
########################################
|
||||
|
||||
VPATH := $(sort $(dir $(SRC)))
|
||||
ALL += $(ALL_LIBS)
|
||||
|
||||
VPATH := $(sort $(dir $(SRC)))
|
||||
|
||||
# Define object files
|
||||
OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(SRC)))
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ FLAVORS="debug:release"
|
|||
TARGETS="cpu:gpu"
|
||||
MATH_LIBRARY="mkl"
|
||||
TESTTARGETS="cpu:gpu"
|
||||
EXTRA_CONFIGURE_OPTIONS=
|
||||
|
||||
# parsing command line arguments:
|
||||
while [[ $# > 0 ]]
|
||||
|
|
@ -41,6 +42,7 @@ case $key in
|
|||
echo " -cba|--clean-build-after - clean up the enlistment binaries after build"
|
||||
echo " -rnd|--random-output-suffix - add random suffix to output directory"
|
||||
echo " -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
|
||||
echo " -x|--extra-configure-options <options> - extra options to pass to configure"
|
||||
echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script"
|
||||
exit 1
|
||||
;;
|
||||
|
|
@ -96,6 +98,10 @@ case $key in
|
|||
-cc|--code-coverage)
|
||||
CODE_COVERAGE=yes
|
||||
;;
|
||||
-x|--extra-configure-options)
|
||||
EXTRA_CONFIGURE_OPTIONS="$2"
|
||||
shift # past argument
|
||||
;;
|
||||
-o|--output-directory)
|
||||
OUTPUT_DIR="$2"
|
||||
shift # past argument
|
||||
|
|
@ -243,7 +249,7 @@ if [[ $BUILD == 1 ]]; then
|
|||
OneBitSGDOPT=yes
|
||||
fi
|
||||
fi
|
||||
./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT
|
||||
./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT $EXTRA_CONFIGURE_OPTIONS
|
||||
if [[ $CLEAN_BEFORE == 1 ]]; then
|
||||
make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# Licensed under the MIT license. See LICENSE.md file in the project root
|
||||
# for full license information.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
|
|
@ -7,16 +7,19 @@ from warnings import warn
|
|||
from setuptools import setup, Extension, find_packages
|
||||
import numpy
|
||||
|
||||
if sys.version_info.major < 3:
|
||||
print("Detected Python v2, which is not yet supported")
|
||||
IS_WINDOWS = platform.system() == 'Windows'
|
||||
|
||||
if IS_WINDOWS and sys.version_info.major < 3:
|
||||
print("Detected Python v2 on Windows, which is not yet supported")
|
||||
sys.exit(1)
|
||||
|
||||
if shutil.which("swig") is None:
|
||||
|
||||
# TODO should handle swig path specified via build_ext --swig-path
|
||||
# TODO FIXME
|
||||
if os.system('swig -version 1>/dev/null 2>/dev/null') != 0:
|
||||
print("Please install swig (>= 3.0.10) and include it in your path.\n")
|
||||
sys.exit(1)
|
||||
|
||||
IS_WINDOWS = platform.system() == 'Windows'
|
||||
|
||||
if IS_WINDOWS:
|
||||
if shutil.which("cl") is None:
|
||||
print("Compiler was not found in path. Please run this from a Visual Studio 2013 x64 Native Tools Command Prompt,\n"
|
||||
|
|
@ -40,6 +43,7 @@ PROJ_LIB_PATH = os.path.join(os.path.dirname(__file__), "cntk", "libs")
|
|||
if 'CNTK_LIB_PATH' in os.environ:
|
||||
CNTK_LIB_PATH = os.environ['CNTK_LIB_PATH']
|
||||
else:
|
||||
# Assumes GPU SKU is being built
|
||||
if IS_WINDOWS:
|
||||
CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release")
|
||||
else:
|
||||
|
|
@ -93,6 +97,11 @@ for fn in rt_libs:
|
|||
tgt_file = proj_lib_path(fn)
|
||||
shutil.copy(src_file, tgt_file)
|
||||
|
||||
if 'CNTK_EXTRA_LIBRARIES' in os.environ:
|
||||
for lib in os.environ['CNTK_EXTRA_LIBRARIES'].split():
|
||||
shutil.copy(lib, PROJ_LIB_PATH)
|
||||
rt_libs.append(strip_path(lib))
|
||||
|
||||
# For package_data we need to have names relative to the cntk module.
|
||||
rt_libs = [os.path.join('libs', fn) for fn in rt_libs]
|
||||
|
||||
|
|
|
|||
|
|
@ -10,11 +10,12 @@
|
|||
configure=$0
|
||||
build_top=$PWD
|
||||
|
||||
have_cuda=no
|
||||
cuda_path=
|
||||
cuda_check=include/cuda.h
|
||||
enable_cuda=
|
||||
|
||||
enable_python=
|
||||
|
||||
# CNTK Custom MKL Version
|
||||
cntk_custom_mkl_version=2
|
||||
|
||||
|
|
@ -63,6 +64,15 @@ have_libzip=no
|
|||
libzip_path=
|
||||
libzip_check=include/zip.h
|
||||
|
||||
have_swig=no
|
||||
swig_path=
|
||||
swig_check=bin/swig
|
||||
swig_required_version=3.0.10
|
||||
|
||||
py_check=bin/python
|
||||
py_versions=
|
||||
declare -A py_paths
|
||||
|
||||
mathlib=
|
||||
|
||||
default_use_1bitsgd=no
|
||||
|
|
@ -89,6 +99,7 @@ default_cubs="cub-1.4.1"
|
|||
default_cudnns="cudnn-5.1 cudnn-5.0"
|
||||
default_opencvs="opencv-3.1.0 opencv-3.0.0"
|
||||
default_libzips="libzip-1.1.2"
|
||||
default_swig="swig-3.0.10"
|
||||
|
||||
function default_paths ()
|
||||
{
|
||||
|
|
@ -192,6 +203,15 @@ function is_hardlinked ()
|
|||
echo $r
|
||||
}
|
||||
|
||||
function check_version ()
|
||||
{
|
||||
local expected_version=$1
|
||||
local actual_version=$2
|
||||
[ "$actual_version" = "" ] && return 1
|
||||
[ "$( printf "%s\n" $actual_version $expected_version | sort -V | head -1)" = "$expected_version" ]
|
||||
return $?
|
||||
}
|
||||
|
||||
function default_use_cuda ()
|
||||
{
|
||||
if test x$(find_cuda) = x || test x$(find_gdk_include) = x || test x$(find_gdk_nvml_lib) = x || test x$(find_cudnn) = x || test x$(find_cub) = x
|
||||
|
|
@ -203,6 +223,65 @@ function default_use_cuda ()
|
|||
}
|
||||
enable_cuda=$(default_use_cuda)
|
||||
|
||||
# TODO allow to pick up from path?
|
||||
function check_swig ()
|
||||
{
|
||||
local swig_dir="$1"
|
||||
local swig_bin="$swig_dir/$swig_check"
|
||||
|
||||
# Check that binary is executable and we can retrieve the version
|
||||
test -x "$swig_bin" || return $?
|
||||
"$swig_bin" -version 2>&1 1>/dev/null || return $?
|
||||
|
||||
local actual_version=$("$swig_bin" -version | grep '^SWIG Version ' | head -1 | grep -o '[0-9.]*')
|
||||
if check_version "$swig_required_version" "$actual_version"
|
||||
then
|
||||
echo "$swig_dir"
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function find_swig ()
|
||||
{
|
||||
check_swig "$(find_dir "$default_swig" "$swig_check")"
|
||||
}
|
||||
|
||||
function check_python ()
|
||||
{
|
||||
# Required version: exact match against major and minor version
|
||||
local required_version="$1"
|
||||
local py_dir="$2"
|
||||
local py_bin="$py_dir/$py_check"
|
||||
[ -x "$py_bin" ] || return 1
|
||||
local py_version=$("$py_bin" -c "import sys; sys.stdout.write('{0}{1}'.format(sys.version_info.major,sys.version_info.minor))")
|
||||
[ "$?" = "0" ] && [ -x "$py_bin" ] && [ "$py_version" = "$required_version" ] && {
|
||||
echo $py_dir
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
# TODO allow to pick up from path?
|
||||
function find_python ()
|
||||
{
|
||||
local required_version="$1"
|
||||
local py_dir=$(find_dir "" "$py_check")
|
||||
check_python "$required_version" "$py_dir"
|
||||
}
|
||||
|
||||
function default_use_python ()
|
||||
{
|
||||
if test x$(find_swig) = x || test x$(find_python 34)$(find_python 35) = x
|
||||
then
|
||||
echo no
|
||||
else
|
||||
echo yes
|
||||
fi
|
||||
}
|
||||
enable_python=$(default_use_python)
|
||||
|
||||
function show_default ()
|
||||
{
|
||||
if test x$1 = x
|
||||
|
|
@ -222,6 +301,7 @@ function show_help ()
|
|||
echo " --add directory add directory to library search path"
|
||||
echo " --1bitsgd[=(yes|no)] use 1Bit SGD $(show_default ${default_use_1bitsgd})"
|
||||
echo " --cuda[=(yes|no)] use cuda GPU $(show_default $(default_use_cuda))"
|
||||
echo " --python[=(yes|no)] with Python bindings $(show_default $(default_use_python))"
|
||||
echo " --with-cuda[=directory] $(show_default $(find_cuda))"
|
||||
echo " --with-cub[=directory] $(show_default $(find_cub))"
|
||||
echo " --with-gdk-include[=directory] $(show_default $(find_gdk_include))"
|
||||
|
|
@ -236,6 +316,11 @@ function show_help ()
|
|||
echo " --with-libzip[=directory] $(show_default $(find_libzip))"
|
||||
echo " --with-code-coverage[=(yes|no)] $(show_default ${default_use_code_coverage})"
|
||||
echo " --with-boost[=directory] $(show_default $(find_boost))"
|
||||
echo " --with-py-versions=(space-separated list of 34, 35)"
|
||||
echo " --with-py34-path[=directory] $(show_default $(find_python 34))"
|
||||
echo " --with-py35-path[=directory] $(show_default $(find_python 35))"
|
||||
echo " --with-swig[=directory] $(show_default $(find_swig))"
|
||||
|
||||
echo "Libraries search path:"
|
||||
for head in $(default_paths)
|
||||
do
|
||||
|
|
@ -304,8 +389,86 @@ do
|
|||
fi
|
||||
;;
|
||||
|
||||
--python*)
|
||||
if test x$optarg = xyes || test x$optarg = xno
|
||||
then
|
||||
enable_python=$optarg
|
||||
else
|
||||
echo "Invalid value for --python $optarg"
|
||||
show_help
|
||||
exit
|
||||
fi
|
||||
;;
|
||||
|
||||
--with-py-versions*)
|
||||
if test "x$optarg" = "x"
|
||||
then
|
||||
echo "Mandatory parameter for --with-py-versions missing. Use '--python no' to disable Python support completely."
|
||||
show_help
|
||||
exit 1
|
||||
else
|
||||
for ver in $optarg
|
||||
do
|
||||
case $ver in
|
||||
34 | 35)
|
||||
;;
|
||||
*)
|
||||
echo "Invalid value for --with-py-versions $optarg: invalid version $ver"
|
||||
show_help
|
||||
exit
|
||||
esac
|
||||
done
|
||||
# TODO filter duplicates?
|
||||
py_versions="$optarg"
|
||||
fi
|
||||
;;
|
||||
|
||||
--with-py34-path*|--with-py35-path*)
|
||||
enable_python=yes
|
||||
py_version=${key:9:2}
|
||||
if test x$optarg = x
|
||||
then
|
||||
py_paths[$py_version]=$(find_python $py_version)
|
||||
if test x${py_paths[$py_version]} = x
|
||||
then
|
||||
echo "Cannot find Python $py_version directory."
|
||||
echo "Please specify a value for $key"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if check_python $py_version "$optarg"
|
||||
then
|
||||
py_paths[$py_version]=$optarg
|
||||
else
|
||||
echo "Invalid Python $py_version directory $optarg"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
--with-swig*)
|
||||
have_swig=yes
|
||||
if test x$optarg = x
|
||||
then
|
||||
swig_path=$(find_swig)
|
||||
if test x$swig_path = x
|
||||
then
|
||||
echo "Cannot find SWIG directory."
|
||||
echo "Please specify a value for --with-swig"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if check_swig "$optarg"
|
||||
then
|
||||
swig_path=$optarg
|
||||
else
|
||||
echo "Invalid SWIG directory $optarg"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
--with-cuda*)
|
||||
have_cuda=yes
|
||||
enable_cuda=yes
|
||||
if test x$optarg = x
|
||||
then
|
||||
|
|
@ -326,6 +489,7 @@ do
|
|||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
--with-cub*)
|
||||
have_cub=yes
|
||||
if test x$optarg = x
|
||||
|
|
@ -573,15 +737,15 @@ fi
|
|||
# If no math library was specified, search for mkl
|
||||
if test x$have_mkl = xno && test x$have_openblas = xno
|
||||
then
|
||||
mkl_path=$(find_mkl)
|
||||
if test x$mkl_path = x
|
||||
then
|
||||
echo "Cannot find a CPU math library."
|
||||
echo "Please specify --with-mkl, --with-mkl-sequential, --with-openblas with a path."
|
||||
exit 1
|
||||
else
|
||||
mathlib=mkl
|
||||
fi
|
||||
mkl_path=$(find_mkl)
|
||||
if test x$mkl_path = x
|
||||
then
|
||||
echo "Cannot find a CPU math library."
|
||||
echo "Please specify --with-mkl, --with-mkl-sequential, --with-openblas with a path."
|
||||
exit 1
|
||||
else
|
||||
mathlib=mkl
|
||||
fi
|
||||
fi
|
||||
|
||||
# If no cuda library specified, search for one
|
||||
|
|
@ -645,6 +809,47 @@ then
|
|||
fi
|
||||
fi
|
||||
|
||||
if test $enable_python = yes && test x$swig_path = x
|
||||
then
|
||||
swig_path=$(find_swig)
|
||||
if test x$swig_path = x
|
||||
then
|
||||
echo 'Cannot locate SWIG (>= 3.0.10), which is required Python support.'
|
||||
echo Check bindings/python/swig_install.sh for build instructions.
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if test $enable_python = yes
|
||||
then
|
||||
# Fill unfilled defaul paths
|
||||
[ -z "${py_paths[34]}" ] && find_python 34 && py_paths[34]=$(find_python 34)
|
||||
[ -z "${py_paths[35]}" ] && find_python 35 && py_paths[35]=$(find_python 35)
|
||||
|
||||
# Unless there's a specified order or restriction, take all configured Python versions (old to new)
|
||||
[ -z "$py_versions" ] && py_versions=${!py_paths[@]}
|
||||
py_versions="$py_versions "
|
||||
|
||||
# Make sure each requested version is configured, remove other parts
|
||||
for ver in 34 35
|
||||
do
|
||||
case $py_versions in
|
||||
*$ver\ *)
|
||||
# Build for version is requested, check if configured
|
||||
if [ -z "${py_paths[$ver]}" ]
|
||||
then
|
||||
echo Build for Python version $ver was requested, but is not configured.
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# Build for version is not requested, unconfigure
|
||||
unset -v py_paths[$ver]
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if test x$opencv_path = x
|
||||
then
|
||||
opencv_path=$(find_opencv)
|
||||
|
|
@ -712,6 +917,17 @@ if test $enable_cuda = yes ; then
|
|||
echo CUB_PATH=$cub_path >> $config
|
||||
echo CUDNN_PATH=$cudnn_path >> $config
|
||||
fi
|
||||
if test $enable_python = yes ; then
|
||||
echo PYTHON_SUPPORT=true >> $config
|
||||
echo SWIG_PATH=$swig_path/bin >> $config
|
||||
# N.B. PYTHON_VERSIONS at least needs to be parseable by bash (cf. Tools/generate_build_info)
|
||||
echo PYTHON_VERSIONS= >> $config
|
||||
for ver in $py_versions
|
||||
do
|
||||
echo PYTHON_VERSIONS+=$ver >> $config
|
||||
echo PYTHON${ver}_PATH=${py_paths[$ver]}/$py_check >> $config
|
||||
done
|
||||
fi
|
||||
if test x$kaldi_path != x ; then
|
||||
echo KALDI_PATH=$kaldi_path >> $config
|
||||
fi
|
||||
|
|
@ -746,5 +962,7 @@ then
|
|||
printf '\t$(MAKE) -C $(dir) BUILD_TOP=$(BUILD_TOP) $@\n' >> $makefile
|
||||
fi
|
||||
echo run
|
||||
echo '>make -j all'
|
||||
echo '>make -j $(nproc) all'
|
||||
echo to build
|
||||
|
||||
# vim:set expandtab tabstop=4 shiftwidth=4:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче