diff --git a/CNTK.sln b/CNTK.sln index 84f3cba78..0946f365c 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1261,6 +1261,23 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalClientTest", "Tests\E EndProject Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "CNTKv2", "bindings\python\examples\CNTKv2.pyproj", "{1A078FC2-21C0-4F42-9A5B-0E84E944BC74}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonBindings", "bindings\python\PythonBindings.vcxproj", "{CD721536-CFD3-413E-A3D7-FB0FAF989635}" + ProjectSection(ProjectDependencies) = postProject + {9BD0A711-0BBD-45B6-B81C-053F03C26CFB} = {9BD0A711-0BBD-45B6-B81C-053F03C26CFB} + {33D2FD22-DEF2-4507-A58A-368F641AEBE5} = {33D2FD22-DEF2-4507-A58A-368F641AEBE5} + {D667AF32-028A-4A5D-BE19-F46776F0F6B2} = {D667AF32-028A-4A5D-BE19-F46776F0F6B2} + {9A2F2441-5972-4EA8-9215-4119FCE0FB68} = {9A2F2441-5972-4EA8-9215-4119FCE0FB68} + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} = {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} + {91973E60-A7BE-4C86-8FDB-59C88A0B3715} = {91973E60-A7BE-4C86-8FDB-59C88A0B3715} + {014DA766-B37B-4581-BC26-963EA5507931} = {014DA766-B37B-4581-BC26-963EA5507931} + {CE429AA2-3778-4619-8FD1-49BA3B81197B} = {CE429AA2-3778-4619-8FD1-49BA3B81197B} + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6} = {62836DC1-DF77-4B98-BF2D-45C943B7DDC6} + {E5606ECE-48CA-4464-BB12-09D81D02B9EF} = {E5606ECE-48CA-4464-BB12-09D81D02B9EF} + {1D5787D4-52E4-45DB-951B-82F220EE0C6A} = {1D5787D4-52E4-45DB-951B-82F220EE0C6A} + {7B7A51ED-AA8E-4660-A805-D50235A02120} = {7B7A51ED-AA8E-4660-A805-D50235A02120} + {E6646FFE-3588-4276-8A15-8D65C22711C1} = {E6646FFE-3588-4276-8A15-8D65C22711C1} + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug_CpuOnly|Any CPU = Debug_CpuOnly|Any CPU @@ -1987,6 +2004,26 @@ Global {1A078FC2-21C0-4F42-9A5B-0E84E944BC74}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A078FC2-21C0-4F42-9A5B-0E84E944BC74}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {1A078FC2-21C0-4F42-9A5B-0E84E944BC74}.Release|x64.ActiveCfg = Release|Any CPU + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug_CpuOnly|Any CPU.ActiveCfg = Debug_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug_CpuOnly|Mixed Platforms.ActiveCfg = Debug_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug_CpuOnly|Mixed Platforms.Build.0 = Debug_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug|Any CPU.ActiveCfg = Debug|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug|Mixed Platforms.ActiveCfg = Debug|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug|Mixed Platforms.Build.0 = Debug|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug|x64.ActiveCfg = Debug|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Debug|x64.Build.0 = Debug|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release_CpuOnly|Any CPU.ActiveCfg = Release_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release_CpuOnly|Mixed Platforms.ActiveCfg = Release_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release_CpuOnly|Mixed Platforms.Build.0 = Release_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release|Any CPU.ActiveCfg = Release|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release|Mixed Platforms.ActiveCfg = Release|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release|Mixed Platforms.Build.0 = Release|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release|x64.ActiveCfg = Release|x64 + {CD721536-CFD3-413E-A3D7-FB0FAF989635}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2161,5 +2198,6 @@ Global {CCC07E8E-F33A-4AF7-9F60-93E2AA61C75E} = {05E45AF7-C069-4057-BC16-0A532D068CE4} {1C6E6C53-1AA7-4B69-913E-B97BB5A872CF} = {05E45AF7-C069-4057-BC16-0A532D068CE4} {1A078FC2-21C0-4F42-9A5B-0E84E944BC74} = {47755F2E-D674-4175-9E38-8EA053455072} + {CD721536-CFD3-413E-A3D7-FB0FAF989635} = {DD043083-71A4-409A-AA91-F9C548DCF7EC} EndGlobalSection EndGlobal diff --git a/Makefile b/Makefile index e58035566..32bb708d4 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,12 @@ # defaults to /usr/local/ # BOOST_PATH= path to Boost installation, so $(BOOST_PATH)/include/boost/test/unit_test.hpp # defaults to /usr/local/boost-1.60.0 +# PYTHON_SUPPORT=true iff CNTK v2 Python module should be build +# SWIG_PATH= path to SWIG (>= 3.0.10) +# PYTHON_VERSIONS= list of Python versions to build for +# A Python version is identified by "34" or "35". +# PYTHON34_PATH= path to Python 3.4 interpreter +# PYTHON35_PATH= path to Python 3.5 interpreter # These can be overridden on the command line, e.g. make BUILDTYPE=debug # TODO: Build static libraries for common dependencies that are shared by multiple @@ -80,7 +86,7 @@ COMMON_FLAGS:= -D_POSIX_SOURCE -D_XOPEN_SOURCE=600 -D__USE_XOPEN2K -std=c++11 CPPFLAGS:= CXXFLAGS:= $(SSE_FLAGS) -std=c++0x -fopenmp -fpermissive -fPIC -Werror -fcheck-new LIBPATH:= -LIBS:= +LIBS_LIST:= LDFLAGS:= CXXVER_GE480:= $(shell expr `$(CXX) -dumpversion | sed -e 's/\.\([0-9][0-9]\)/\1/g' -e 's/\.\([0-9]\)/0\1/g' -e 's/^[0-9]\{3,4\}$$/&00/'` \>= 40800) @@ -90,6 +96,8 @@ endif SEPARATOR = "=-----------------------------------------------------------=" ALL:= +ALL_LIBS:= +LIBS_FULLPATH:= SRC:= # Make sure all is the first (i.e. default) target, but we can't actually define it @@ -125,13 +133,13 @@ ifdef CUDA_PATH # Set up CUDA includes and libraries INCLUDEPATH += $(CUDA_PATH)/include LIBPATH += $(CUDA_PATH)/lib64 - LIBS += -lcublas -lcudart -lcuda -lcurand -lcusparse -lnvidia-ml + LIBS_LIST += cublas cudart cuda curand cusparse nvidia-ml # Set up cuDNN if needed ifdef CUDNN_PATH INCLUDEPATH += $(CUDNN_PATH)/cuda/include LIBPATH += $(CUDNN_PATH)/cuda/lib64 - LIBS += -lcudnn + LIBS_LIST += cudnn COMMON_FLAGS +=-DUSE_CUDNN endif else @@ -142,13 +150,13 @@ endif ifeq ("$(MATHLIB)","mkl") INCLUDEPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/include - LIBS += -lm + LIBS_LIST += m ifeq ("$(MKL_THREADING)","sequential") LIBPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/x64/sequential - LIBS += -lmkl_cntk_s + LIBS_LIST += mkl_cntk_s else LIBPATH += $(MKL_PATH)/$(CNTK_CUSTOM_MKL_VERSION)/x64/parallel - LIBS += -lmkl_cntk_p -liomp5 -lpthread + LIBS_LIST += mkl_cntk_p iomp5 pthread endif COMMON_FLAGS += -DUSE_MKL endif @@ -156,7 +164,7 @@ endif ifeq ("$(MATHLIB)","openblas") INCLUDEPATH += $(OPENBLAS_PATH)/include LIBPATH += $(OPENBLAS_PATH)/lib - LIBS += -lopenblas -lm -lpthread + LIBS_LIST += openblas m pthread CPPFLAGS += -DUSE_OPENBLAS endif @@ -167,10 +175,11 @@ ifdef KALDI_PATH ATLASINC = $(KALDI_PATH)/tools/ATLAS/include INCLUDEPATH += $(KALDI_PATH)/src $(ATLASINC) $(FSTROOT)/include - CPPFLAGS+= -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400 + CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400 KALDI_LIBPATH += $(KALDI_PATH)/src/lib - KALDI_LIBS += -lkaldi-util -lkaldi-matrix -lkaldi-base -lkaldi-hmm -lkaldi-cudamatrix -lkaldi-nnet -lkaldi-lat + KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat + KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST)) endif ifdef SUPPORT_AVX2 @@ -224,11 +233,14 @@ ifdef CNTK_CUDA_DEVICE_DEBUGINFO CUFLAGS += -G endif -####### +# Create the library link options for the linker. +# LIBS_LIST must not be changed beyond this point. +LIBS:= $(addprefix -l,$(LIBS_LIST)) OBJDIR:= $(BUILD_TOP)/.build BINDIR:= $(BUILD_TOP)/bin LIBDIR:= $(BUILD_TOP)/lib +PYTHONDIR:= $(BUILD_TOP)/python ORIGINLIBDIR:='$$ORIGIN/../lib' ORIGINDIR:='$$ORIGIN' @@ -329,7 +341,7 @@ MATH_SRC+=$(READER_SRC) MATH_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(MATH_SRC))) CNTKMATH_LIB:= $(LIBDIR)/lib$(CNTKMATH).so -ALL += $(CNTKMATH_LIB) +ALL_LIBS += $(CNTKMATH_LIB) SRC+=$(MATH_SRC) $(CNTKMATH_LIB): $(MATH_OBJ) @@ -402,7 +414,7 @@ CNTKLIBRARY:=cntklibrary-$(CNTKLIBRARY_VERSION) CNTKLIBRARY_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(CNTKLIBRARY_SRC))) CNTKLIBRARY_LIB:=$(LIBDIR)/lib$(CNTKLIBRARY).so -ALL+=$(CNTKLIBRARY_LIB) +ALL_LIBS+=$(CNTKLIBRARY_LIB) SRC+=$(CNTKLIBRARY_SRC) $(CNTKLIBRARY_LIB): $(CNTKLIBRARY_OBJ) | $(CNTKMATH_LIB) @@ -474,7 +486,7 @@ EVAL_SRC+=$(SEQUENCE_TRAINING_LIB_SRC) EVAL_OBJ:=$(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(EVAL_SRC))) EVAL_LIB:=$(LIBDIR)/lib$(EVAL).so -ALL+=$(EVAL_LIB) +ALL_LIBS+=$(EVAL_LIB) SRC+=$(EVAL_SRC) $(EVAL_LIB): $(EVAL_OBJ) | $(CNTKMATH_LIB) @@ -515,7 +527,7 @@ BINARYREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(BINARYREADER_SRC)) BINARY_READER:= $(LIBDIR)/BinaryReader.so -#ALL += $(BINARY_READER) +#ALL_LIBS += $(BINARY_READER) #SRC+=$(BINARYREADER_SRC) $(BINARY_READER): $(BINARYREADER_OBJ) | $(CNTKMATH_LIB) @@ -535,7 +547,7 @@ HTKMLFREADER_SRC =\ HTKMLFREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(HTKMLFREADER_SRC)) HTKMLFREADER:=$(LIBDIR)/HTKMLFReader.so -ALL+=$(HTKMLFREADER) +ALL_LIBS+=$(HTKMLFREADER) SRC+=$(HTKMLFREADER_SRC) $(LIBDIR)/HTKMLFReader.so: $(HTKMLFREADER_OBJ) | $(CNTKMATH_LIB) @@ -553,7 +565,7 @@ COMPOSITEDATAREADER_SRC =\ COMPOSITEDATAREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(COMPOSITEDATAREADER_SRC)) COMPOSITEDATAREADER:=$(LIBDIR)/CompositeDataReader.so -ALL+=$(COMPOSITEDATAREADER) +ALL_LIBS+=$(COMPOSITEDATAREADER) SRC+=$(COMPOSITEDATAREADER_SRC) $(LIBDIR)/CompositeDataReader.so: $(COMPOSITEDATAREADER_OBJ) | $(CNTKMATH_LIB) @@ -576,7 +588,7 @@ HTKDESERIALIZERS_SRC =\ HTKDESERIALIZERS_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(HTKDESERIALIZERS_SRC)) HTKDESERIALIZERS:=$(LIBDIR)/HTKDeserializers.so -ALL+=$(HTKDESERIALIZERS) +ALL_LIBS+=$(HTKDESERIALIZERS) SRC+=$(HTKDESERIALIZERS_SRC) $(LIBDIR)/HTKDeserializers.so: $(HTKDESERIALIZERS_OBJ) | $(CNTKMATH_LIB) @@ -596,7 +608,7 @@ LMSEQUENCEREADER_SRC =\ LMSEQUENCEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LMSEQUENCEREADER_SRC)) LMSEQUENCEREADER:= $(LIBDIR)/LMSequenceReader.so -ALL+=$(LMSEQUENCEREADER) +ALL_LIBS+=$(LMSEQUENCEREADER) SRC+=$(LMSEQUENCEREADER_SRC) $(LMSEQUENCEREADER): $(LMSEQUENCEREADER_OBJ) | $(CNTKMATH_LIB) @@ -617,7 +629,7 @@ LUSEQUENCEREADER_SRC =\ LUSEQUENCEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LUSEQUENCEREADER_SRC)) LUSEQUENCEREADER:=$(LIBDIR)/LUSequenceReader.so -ALL+=$(LUSEQUENCEREADER) +ALL_LIBS+=$(LUSEQUENCEREADER) SRC+=$(LUSEQUENCEREADER_SRC) $(LUSEQUENCEREADER): $(LUSEQUENCEREADER_OBJ) | $(CNTKMATH_LIB) @@ -636,7 +648,7 @@ UCIFASTREADER_SRC =\ UCIFASTREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UCIFASTREADER_SRC)) UCIFASTREADER:=$(LIBDIR)/UCIFastReader.so -ALL += $(UCIFASTREADER) +ALL_LIBS += $(UCIFASTREADER) SRC+=$(UCIFASTREADER_SRC) $(UCIFASTREADER): $(UCIFASTREADER_OBJ) | $(CNTKMATH_LIB) @@ -654,7 +666,7 @@ LIBSVMBINARYREADER_SRC =\ LIBSVMBINARYREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(LIBSVMBINARYREADER_SRC)) LIBSVMBINARYREADER:=$(LIBDIR)/LibSVMBinaryReader.so -ALL += $(LIBSVMBINARYREADER) +ALL_LIBS += $(LIBSVMBINARYREADER) SRC+=$(LIBSVMBINARYREADER_SRC) $(LIBSVMBINARYREADER): $(LIBSVMBINARYREADER_OBJ) | $(CNTKMATH_LIB) @@ -672,7 +684,7 @@ SPARSEPCREADER_SRC =\ SPARSEPCREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(SPARSEPCREADER_SRC)) SPARSEPCREADER:=$(LIBDIR)/SparsePCReader.so -ALL += $(SPARSEPCREADER) +ALL_LIBS += $(SPARSEPCREADER) SRC+=$(SPARSEPCREADER_SRC) $(SPARSEPCREADER): $(SPARSEPCREADER_OBJ) | $(CNTKMATH_LIB) @@ -693,7 +705,7 @@ CNTKTEXTFORMATREADER_SRC =\ CNTKTEXTFORMATREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(CNTKTEXTFORMATREADER_SRC)) CNTKTEXTFORMATREADER:=$(LIBDIR)/CNTKTextFormatReader.so -ALL += $(CNTKTEXTFORMATREADER) +ALL_LIBS += $(CNTKTEXTFORMATREADER) SRC+=$(CNTKTEXTFORMATREADER_SRC) $(CNTKTEXTFORMATREADER): $(CNTKTEXTFORMATREADER_OBJ) | $(CNTKMATH_LIB) @@ -718,7 +730,7 @@ KALDI2READER_SRC = \ KALDI2READER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(KALDI2READER_SRC)) KALDI2READER:=$(LIBDIR)/Kaldi2Reader.so -ALL+=$(KALDI2READER) +ALL_LIBS+=$(KALDI2READER) SRC+=$(KALDI2READER_SRC) $(KALDI2READER): $(KALDI2READER_OBJ) | $(CNTKMATH_LIB) @@ -736,17 +748,18 @@ ifdef BOOST_PATH INCLUDEPATH += $(BOOST_PATH)/include -IMAGE_READER_LIBS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs +IMAGEREADER_LIBS_LIST := opencv_core opencv_imgproc opencv_imgcodecs ifdef LIBZIP_PATH CPPFLAGS += -DUSE_ZIP - #both directories are needed for building libzip - INCLUDEPATH += $(LIBZIP_PATH)/include - INCLUDEPATH += $(LIBZIP_PATH)/lib/libzip/include - IMAGE_READER_LIBS += -lzip + # Both directories are needed for building libzip + INCLUDEPATH += $(LIBZIP_PATH)/include $(LIBZIP_PATH)/lib/libzip/include LIBPATH += $(LIBZIP_PATH)/lib + IMAGEREADER_LIBS_LIST += zip endif +IMAGEREADER_LIBS:= $(addprefix -l,$(IMAGEREADER_LIBS_LIST)) + IMAGEREADER_SRC =\ $(SOURCEDIR)/Readers/ImageReader/Exports.cpp \ $(SOURCEDIR)/Readers/ImageReader/ImageConfigHelper.cpp \ @@ -758,7 +771,7 @@ IMAGEREADER_SRC =\ IMAGEREADER_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(IMAGEREADER_SRC)) IMAGEREADER:=$(LIBDIR)/ImageReader.so -ALL += $(IMAGEREADER) +ALL_LIBS += $(IMAGEREADER) SRC+=$(IMAGEREADER_SRC) INCLUDEPATH += $(OPENCV_PATH)/include @@ -766,7 +779,7 @@ LIBPATH += $(OPENCV_PATH)/lib $(OPENCV_PATH)/release/lib $(IMAGEREADER): $(IMAGEREADER_OBJ) | $(CNTKMATH_LIB) @echo $(SEPARATOR) - $(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ -l$(CNTKMATH) $(IMAGE_READER_LIBS) + $(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ -l$(CNTKMATH) $(IMAGEREADER_LIBS) endif endif @@ -978,11 +991,53 @@ unittests: $(UNITTEST_EVAL) $(UNITTEST_READER) $(UNITTEST_NETWORK) $(UNITTEST_MA endif +# For now only build Release. +ifeq ("$(PYTHON_SUPPORT) $(BUILDTYPE)","true release") + +# Libraries needed for the run-time (i.e., excluding test binaries) +# TODO MPI doesn't appear explicitly here, hidden by mpic++ usage (but currently, it should be user installed) +RUNTIME_LIBS_LIST := $(LIBS_LIST) $(IMAGEREADER_LIBS_LIST) $(KALDI_LIBS_LIST) +RUNTIME_LIBS_EXCLUDE_LIST := m pthread nvidia-ml +EXTRA_LIBS_BASENAMES:=$(addsuffix .so,$(addprefix lib,$(filter-out $(RUNTIME_LIBS_EXCLUDE_LIST),$(RUNTIME_LIBS_LIST)))) + +# TODO dependencies +# TODO intermediate build results should go below $OBJDIR +.PHONY: python +python: $(ALL_LIBS) + @bash -c '\ + set -x; \ + declare -A py_paths; \ + py_paths[34]=$(PYTHON34_PATH); \ + py_paths[35]=$(PYTHON35_PATH); \ + export LD_LIBRARY_PATH=$$(echo $(LIBPATH) $(KALDI_LIBPATH) | tr " " :); \ + ! (ldd $(LIBDIR)/* | grep "not found") && \ + export CNTK_EXTRA_LIBRARIES=$$(ldd $(LIBDIR)/* | grep "^\s.*=> " | cut -d ">" -f 2- --only-delimited | cut -d "(" -f 1 --only-delimited | sort -u | grep -Ff <(echo $(EXTRA_LIBS_BASENAMES) | xargs -n1)) && \ + test -x $(SWIG_PATH) && \ + export CNTK_LIB_PATH=$$(readlink -f $(LIBDIR)) && \ + PYTHONDIR=$$(readlink -f $(PYTHONDIR)) && \ + test $$? -eq 0 && \ + cd bindings/python && \ + export PATH=$(SWIG_PATH):$$PATH; \ + for ver in $(PYTHON_VERSIONS); \ + do \ + test -x $${py_paths[$$ver]}; \ + $${py_paths[$$ver]} setup.py \ + build_ext \ + bdist_wheel \ + --dist-dir $$PYTHONDIR || exit $$?; \ + done' + +ALL += python + +endif + ######################################## # General compile and dependency rules ######################################## -VPATH := $(sort $(dir $(SRC))) +ALL += $(ALL_LIBS) + +VPATH := $(sort $(dir $(SRC))) # Define object files OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(SRC))) diff --git a/Tools/build-and-test b/Tools/build-and-test index 67af0a8e0..2091e5c17 100755 --- a/Tools/build-and-test +++ b/Tools/build-and-test @@ -19,6 +19,7 @@ FLAVORS="debug:release" TARGETS="cpu:gpu" MATH_LIBRARY="mkl" TESTTARGETS="cpu:gpu" +EXTRA_CONFIGURE_OPTIONS= # parsing command line arguments: while [[ $# > 0 ]] @@ -41,6 +42,7 @@ case $key in echo " -cba|--clean-build-after - clean up the enlistment binaries after build" echo " -rnd|--random-output-suffix - add random suffix to output directory" echo " -o|--output-directory - specify output directory to use (by default those will be in .run-)" + echo " -x|--extra-configure-options - extra options to pass to configure" echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script" exit 1 ;; @@ -96,6 +98,10 @@ case $key in -cc|--code-coverage) CODE_COVERAGE=yes ;; + -x|--extra-configure-options) + EXTRA_CONFIGURE_OPTIONS="$2" + shift # past argument + ;; -o|--output-directory) OUTPUT_DIR="$2" shift # past argument @@ -243,7 +249,7 @@ if [[ $BUILD == 1 ]]; then OneBitSGDOPT=yes fi fi - ./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT + ./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT $EXTRA_CONFIGURE_OPTIONS if [[ $CLEAN_BEFORE == 1 ]]; then make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $? fi diff --git a/bindings/python/PythonBindings.vcxproj b/bindings/python/PythonBindings.vcxproj new file mode 100644 index 000000000..e0806eaf2 --- /dev/null +++ b/bindings/python/PythonBindings.vcxproj @@ -0,0 +1,50 @@ + + + + + Debug + x64 + + + Release + x64 + + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + + + + {CD721536-CFD3-413E-A3D7-FB0FAF989635} + MakeFileProj + + + + + Makefile + v120 + $(DebugBuild) + + + + + + + + + + .\vsbuild.bat "$(OutDir)" "$(DebugBuild)" "$(GpuBuild)" "$(SWIG_PATH)" "$(CNTK_PY_VERSIONS)" "$(CNTK_PY27_PATH)" "$(CNTK_PY34_PATH)" "$(CNTK_PY35_PATH)" + dist/pythonwheel + rmdir /s /q build + + + + + + + + diff --git a/bindings/python/setup.py b/bindings/python/setup.py index d18007dc7..9cad6c90d 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -7,16 +7,18 @@ from warnings import warn from setuptools import setup, Extension, find_packages import numpy -if sys.version_info.major < 3: - print("Detected Python v2, which is not yet supported") +IS_WINDOWS = platform.system() == 'Windows' + +if IS_WINDOWS and sys.version_info.major < 3: + print("Detected Python v2 on Windows, which is not yet supported") sys.exit(1) -if shutil.which("swig") is None: + +# TODO should handle swig path specified via build_ext --swig-path +if os.system('swig -version 1>%s 2>%s' % (os.devnull, os.devnull)) != 0: print("Please install swig (>= 3.0.10) and include it in your path.\n") sys.exit(1) -IS_WINDOWS = platform.system() == 'Windows' - if IS_WINDOWS: if shutil.which("cl") is None: print("Compiler was not found in path. Please run this from a Visual Studio 2013 x64 Native Tools Command Prompt,\n" @@ -40,6 +42,7 @@ PROJ_LIB_PATH = os.path.join(os.path.dirname(__file__), "cntk", "libs") if 'CNTK_LIB_PATH' in os.environ: CNTK_LIB_PATH = os.environ['CNTK_LIB_PATH'] else: + # Assumes GPU SKU is being built if IS_WINDOWS: CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release") else: @@ -93,6 +96,11 @@ for fn in rt_libs: tgt_file = proj_lib_path(fn) shutil.copy(src_file, tgt_file) +if 'CNTK_EXTRA_LIBRARIES' in os.environ: + for lib in os.environ['CNTK_EXTRA_LIBRARIES'].split(): + shutil.copy(lib, PROJ_LIB_PATH) + rt_libs.append(strip_path(lib)) + # For package_data we need to have names relative to the cntk module. rt_libs = [os.path.join('libs', fn) for fn in rt_libs] diff --git a/bindings/python/vsbuild.bat b/bindings/python/vsbuild.bat new file mode 100644 index 000000000..a13df2467 --- /dev/null +++ b/bindings/python/vsbuild.bat @@ -0,0 +1,69 @@ +@echo off +setlocal enabledelayedexpansion + +REM Copyright (c) Microsoft. All rights reserved. +REM +REM Licensed under the MIT license. See LICENSE.md file in the project root +REM for full license information. +REM ============================================================================== + +REM Grab the parameters +REM +REM Note: don't rely on environment variables, since properties may have been +REM overridden at msbuild invocation. + +set p_OutDir=%~1 +set p_DebugBuild=%~2 +set p_GpuBuild=%~3 +set p_SWIG_PATH=%~4 +set p_CNTK_PY_VERSIONS=%~5 +set p_CNTK_PY27_PATH=%~6 +set p_CNTK_PY34_PATH=%~7 +set p_CNTK_PY35_PATH=%~8 + +REM Construct p_CNTK_PY_VERSIONS if not explicitly defined +REM (Note: to disable Python build completely, no CNTK_PYx_PATH variable must be defined) +if not defined p_CNTK_PY_VERSIONS ( + REM Note: leading space doesn't hurt + if defined p_CNTK_PY27_PATH set p_CNTK_PY_VERSIONS=!p_CNTK_PY_VERSIONS! 27 + if defined p_CNTK_PY34_PATH set p_CNTK_PY_VERSIONS=!p_CNTK_PY_VERSIONS! 34 + if defined p_CNTK_PY35_PATH set p_CNTK_PY_VERSIONS=!p_CNTK_PY_VERSIONS! 35 +) + +REM Validate p_CNTK_PY_VERSIONS contents. +REM TODO Python 3.4 only for now +for %%p in (%p_CNTK_PY_VERSIONS%) do ( + if not "%%~p" == "34" echo Build for unsupported Python version '%%~p' requested, stopping&exit /b 1 + set nothingToBuild= +) + +REM Validate p_CNTK_PY_VERSIONS contents. +REM (Note: Don't merge with above loop; more robust parsing) +set nothingToBuild=1 +for %%p in (%p_CNTK_PY_VERSIONS%) do ( + call set extraPath=!p_CNTK_PY%%~p_PATH! + if not defined extraPath echo Build for Python version '%%~p' requested, but CNTK_PY%%~p_PATH not defined, stopping&exit /b 1 + set nothingToBuild= +) +if defined nothingToBuild echo Python support not configured to build.&exit /b 0 + +if "%p_DebugBuild%" == "true" echo Currently no Python build for Debug configurations, exiting.&exit /b 0 + +call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +set CNTK_LIB_PATH=%p_OutDir% +set DIST_DIR=%p_OutDir%\Python +set PATH=%p_SWIG_PATH%;%PATH% +set MSSdk=1 +set DISTUTILS_USE_SDK=1 + +REM Build everything in supplied order +set oldPath=%PATH% +for %%p in (%p_CNTK_PY_VERSIONS%) do ( + call set extraPath=!p_CNTK_PY%%~p_PATH! + echo Building for Python version '%%~p', extra path is !extraPath! + set PATH=!extraPath!;!oldPath! + python.exe .\setup.py ^ + build_ext --inplace --force --compiler msvc --plat-name=win-amd64 ^ + bdist_wheel --dist-dir "%DIST_DIR%" + if errorlevel 1 exit /b 1 +) diff --git a/configure b/configure index fb5a1491c..83b3440bb 100755 --- a/configure +++ b/configure @@ -10,11 +10,12 @@ configure=$0 build_top=$PWD -have_cuda=no cuda_path= cuda_check=include/cuda.h enable_cuda= +enable_python= + # CNTK Custom MKL Version cntk_custom_mkl_version=2 @@ -63,6 +64,15 @@ have_libzip=no libzip_path= libzip_check=include/zip.h +have_swig=no +swig_path= +swig_check=bin/swig +swig_required_version=3.0.10 + +py_check=bin/python +py_versions= +declare -A py_paths + mathlib= default_use_1bitsgd=no @@ -89,6 +99,7 @@ default_cubs="cub-1.4.1" default_cudnns="cudnn-5.1 cudnn-5.0" default_opencvs="opencv-3.1.0 opencv-3.0.0" default_libzips="libzip-1.1.2" +default_swig="swig-3.0.10" function default_paths () { @@ -192,6 +203,15 @@ function is_hardlinked () echo $r } +function check_version () +{ + local expected_version=$1 + local actual_version=$2 + [ -z "$actual_version" ] && return 1 + [ "$( printf "%s\n" $actual_version $expected_version | sort -V | head -1)" = "$expected_version" ] + return $? +} + function default_use_cuda () { if test x$(find_cuda) = x || test x$(find_gdk_include) = x || test x$(find_gdk_nvml_lib) = x || test x$(find_cudnn) = x || test x$(find_cub) = x @@ -203,6 +223,66 @@ function default_use_cuda () } enable_cuda=$(default_use_cuda) +# TODO allow to pick up from path? +function check_swig () +{ + local swig_dir="$1" + local swig_bin="$swig_dir/$swig_check" + + # Check that binary is executable and we can retrieve the version + test -x "$swig_bin" || return $? + "$swig_bin" -version 2>&1 1>/dev/null || return $? + + local actual_version=$("$swig_bin" -version | grep '^SWIG Version ' | head -1 | grep -o '[0-9.]*') + if check_version "$swig_required_version" "$actual_version" + then + echo "$swig_dir" + return 0 + else + return 1 + fi +} + +function find_swig () +{ + check_swig "$(find_dir "$default_swig" "$swig_check")" +} + +function check_python () +{ + # Required version: exact match against major and minor version + local required_version="$1" + local py_dir="$2" + local py_bin="$py_dir/$py_check" + [ -x "$py_bin" ] || return 1 + local py_version=$("$py_bin" -c "import sys; sys.stdout.write('{0}{1}'.format(sys.version_info.major,sys.version_info.minor))") + [ "$?" = "0" ] && [ -x "$py_bin" ] && [ "$py_version" = "$required_version" ] && { + echo $py_dir + return 0 + } + return 1 +} + +# TODO allow to pick up from path? +function find_python () +{ + local required_version="$1" + local py_dir=$(find_dir "" "$py_check") + check_python "$required_version" "$py_dir" +} + +function default_use_python () +{ +# Keeping it default "no" for a little while: +# if test x$(find_swig) = x || test x$(find_python 34)$(find_python 35) = x +# then + echo no +# else +# echo yes +# fi +} +enable_python=$(default_use_python) + function show_default () { if test x$1 = x @@ -222,6 +302,7 @@ function show_help () echo " --add directory add directory to library search path" echo " --1bitsgd[=(yes|no)] use 1Bit SGD $(show_default ${default_use_1bitsgd})" echo " --cuda[=(yes|no)] use cuda GPU $(show_default $(default_use_cuda))" + echo " --python[=(yes|no)] with Python bindings $(show_default $(default_use_python))" echo " --with-cuda[=directory] $(show_default $(find_cuda))" echo " --with-cub[=directory] $(show_default $(find_cub))" echo " --with-gdk-include[=directory] $(show_default $(find_gdk_include))" @@ -236,6 +317,11 @@ function show_help () echo " --with-libzip[=directory] $(show_default $(find_libzip))" echo " --with-code-coverage[=(yes|no)] $(show_default ${default_use_code_coverage})" echo " --with-boost[=directory] $(show_default $(find_boost))" + echo " --with-py-versions=(space-separated list of 34, 35)" + echo " --with-py34-path[=directory] $(show_default $(find_python 34))" + echo " --with-py35-path[=directory] $(show_default $(find_python 35))" + echo " --with-swig[=directory] $(show_default $(find_swig))" + echo "Libraries search path:" for head in $(default_paths) do @@ -304,8 +390,87 @@ do fi ;; + --python*) + if test x$optarg = xyes || test x$optarg = xno + then + enable_python=$optarg + else + echo "Invalid value for --python $optarg" + show_help + exit + fi + ;; + + --with-py-versions*) + enable_python=yes + if test "x$optarg" = "x" + then + echo "Mandatory parameter for --with-py-versions missing. Use '--python no' to disable Python support completely." + show_help + exit 1 + else + for ver in $optarg + do + case $ver in + 34 | 35) + ;; + *) + echo "Invalid value for --with-py-versions $optarg: invalid version $ver" + show_help + exit + esac + done + # TODO filter duplicates? + py_versions="$optarg" + fi + ;; + + --with-py34-path*|--with-py35-path*) + enable_python=yes + py_version=${key:9:2} + if test x$optarg = x + then + py_paths[$py_version]=$(find_python $py_version) + if test x${py_paths[$py_version]} = x + then + echo "Cannot find Python $py_version directory." + echo "Please specify a value for $key" + exit 1 + fi + else + if check_python $py_version "$optarg" + then + py_paths[$py_version]=$optarg + else + echo "Invalid Python $py_version directory $optarg" + exit 1 + fi + fi + ;; + + --with-swig*) + have_swig=yes + if test x$optarg = x + then + swig_path=$(find_swig) + if test x$swig_path = x + then + echo "Cannot find SWIG directory." + echo "Please specify a value for --with-swig" + exit 1 + fi + else + if check_swig "$optarg" + then + swig_path=$optarg + else + echo "Invalid SWIG directory $optarg" + exit 1 + fi + fi + ;; + --with-cuda*) - have_cuda=yes enable_cuda=yes if test x$optarg = x then @@ -326,6 +491,7 @@ do fi fi ;; + --with-cub*) have_cub=yes if test x$optarg = x @@ -573,15 +739,15 @@ fi # If no math library was specified, search for mkl if test x$have_mkl = xno && test x$have_openblas = xno then - mkl_path=$(find_mkl) - if test x$mkl_path = x - then - echo "Cannot find a CPU math library." - echo "Please specify --with-mkl, --with-mkl-sequential, --with-openblas with a path." - exit 1 - else - mathlib=mkl - fi + mkl_path=$(find_mkl) + if test x$mkl_path = x + then + echo "Cannot find a CPU math library." + echo "Please specify --with-mkl, --with-mkl-sequential, --with-openblas with a path." + exit 1 + else + mathlib=mkl + fi fi # If no cuda library specified, search for one @@ -645,6 +811,47 @@ then fi fi +if test $enable_python = yes && test x$swig_path = x +then + swig_path=$(find_swig) + if test x$swig_path = x + then + echo 'Cannot locate SWIG (>= 3.0.10), which is required Python support.' + echo Check bindings/python/swig_install.sh for build instructions. + exit 1 + fi +fi + +if test $enable_python = yes +then + # Fill unfilled default paths + [ -z "${py_paths[34]}" ] && find_python 34 && py_paths[34]=$(find_python 34) + [ -z "${py_paths[35]}" ] && find_python 35 && py_paths[35]=$(find_python 35) + + # Unless there's a specified order or restriction, take all configured Python versions (old to new) + [ -z "$py_versions" ] && py_versions=${!py_paths[@]} + py_versions="$py_versions " + + # Make sure each requested version is configured, remove other parts + for ver in 34 35 + do + case $py_versions in + *$ver\ *) + # Build for version is requested, check if configured + if [ -z "${py_paths[$ver]}" ] + then + echo Build for Python version $ver was requested, but is not configured. + exit 1 + fi + ;; + *) + # Build for version is not requested, unconfigure + unset -v py_paths[$ver] + ;; + esac + done +fi + if test x$opencv_path = x then opencv_path=$(find_opencv) @@ -712,6 +919,17 @@ if test $enable_cuda = yes ; then echo CUB_PATH=$cub_path >> $config echo CUDNN_PATH=$cudnn_path >> $config fi +if test $enable_python = yes ; then + echo PYTHON_SUPPORT=true >> $config + echo SWIG_PATH=$swig_path/bin >> $config + # N.B. PYTHON_VERSIONS at least needs to be parseable by bash (cf. Tools/generate_build_info) + echo PYTHON_VERSIONS= >> $config + for ver in $py_versions + do + echo PYTHON_VERSIONS+=$ver >> $config + echo PYTHON${ver}_PATH=${py_paths[$ver]}/$py_check >> $config + done +fi if test x$kaldi_path != x ; then echo KALDI_PATH=$kaldi_path >> $config fi @@ -746,5 +964,7 @@ then printf '\t$(MAKE) -C $(dir) BUILD_TOP=$(BUILD_TOP) $@\n' >> $makefile fi echo run -echo '>make -j all' +echo '>make -j $(nproc) all' echo to build + +# vim:set expandtab tabstop=4 shiftwidth=4: