Move hard-coded CNTK version to a common place

This will help to build CNTK for nighlty builds with required CNTK
version. If environment variable 'BUILD_CNTK_VERSION' is set, then CNTK
will be build for that version as a public release('+' won't be appended
to CNTK version). Otherwise hard-coded CNTK version will be used as a
private build (2.4+).
This commit is contained in:
Manik Jindal 2018-02-20 10:54:07 -08:00
Родитель 4017a1664a
Коммит 461b82d69f
16 изменённых файлов: 117 добавлений и 38 удалений

1
.gitignore поставляемый
Просмотреть файл

@ -224,6 +224,7 @@ bindings/python/cntk/cntk_py.py
bindings/python/cntk/libs/
bindings/python/cntk/cntk_py_wrap.cpp
bindings/python/cntk/cntk_py_wrap.h
bindings/python/cntk/VERSION
bindings/python/dist/
bindings/python/doc/cntk.*.rst
bindings/python/doc/cntk.rst

Просмотреть файл

@ -30,7 +30,34 @@
<HasJava>false</HasJava>
<HasJava Condition="Exists('$(JAVA_HOME)\bin\javac.exe')">true</HasJava>
<CntkComponentVersion>2.4</CntkComponentVersion>
<!-- Set CNTK version related properties -->
<!-- CntkVersion:
CNTK version which should be used where CNTK version is required. Ex: print version or tag CNTK binaries. Default value is the last released version of CNTK. -->
<!-- NOTE: Modify both CntkVersion and PublicBuild during MAJOR RELEASE -->
<CntkVersion>2.4</CntkVersion>
<!-- PublicBuild:
True if build binaries are meant to shared publicly with CNTK community. -->
<!-- NOTE: Modify both CntkVersion and PublicBuild during MAJOR RELEASE -->
<PublicBuild>false</PublicBuild>
<!-- CntkVersionProvidedExternally:
Hard-coded CntkVersion can be overridden if property BUILD_CNTK_VERSION is present. -->
<CntkVersionProvidedExternally>false</CntkVersionProvidedExternally>
<CntkVersionProvidedExternally Condition=" '$(BUILD_CNTK_VERSION)' != '' ">true</CntkVersionProvidedExternally>
<CntkVersion Condition="$(CntkVersionProvidedExternally)">$(BUILD_CNTK_VERSION)</CntkVersion>
<PublicBuild Condition="$(CntkVersionProvidedExternally)">true</PublicBuild>
<!-- CntkVersionBanner:
Cntk Version banner is printed wherever CntkVersion should be printed. ex: python -c 'import cntk;cntk.__version__'. -->
<CntkVersionBanner>$(CntkVersion)</CntkVersionBanner>
<CntkVersionBanner Condition="!$(PublicBuild)">$(CntkVersionBanner)+</CntkVersionBanner>
<!-- CntkComponentVersion:
Cntk binaries (generated by build) are appended with CntkComponentVersion. Ex: Cntk.Core-$(CntkComponentVersion).dll -->
<CntkComponentVersion>$(CntkVersion)</CntkComponentVersion>
<CntkComponentVersion Condition="$(DebugBuild)">$(CntkComponentVersion)d</CntkComponentVersion>
</PropertyGroup>

Просмотреть файл

@ -149,7 +149,7 @@
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>CNTK_COMPONENT_VERSION="$(CntkComponentVersion)"</PreprocessorDefinitions>
<PreprocessorDefinitions>CNTK_VERSION="$(CntkVersion)";CNTK_VERSION_BANNER="$(CntkVersionBanner)";CNTK_COMPONENT_VERSION="$(CntkComponentVersion)"</PreprocessorDefinitions>
<!-- UWP does not use MPI -->
<PreprocessorDefinitions Condition="!$(IsUWP)">%(PreprocessorDefinitions);HAS_MPI=1</PreprocessorDefinitions>
<PreprocessorDefinitions Condition="'$(CudaVersion)' == '9.0'">%(PreprocessorDefinitions);CUDA_NO_HALF;__CUDA_NO_HALF_OPERATORS__</PreprocessorDefinitions>

Просмотреть файл

@ -9,6 +9,8 @@
# that provides
# BUILDTYPE= One of release or debug
# defaults to release
# BUILD_VERSION= CNTK version number to be used while building
# BUILD_PUBLIC= One of yes or no
# MKL_PATH= path to MKLML installation
# only needed if MATHLIB=mkl
# GDK_INCLUDE_PATH= path to CUDA GDK include path, so $(GDK_INCLUDE_PATH)/nvml.h exists
@ -271,11 +273,23 @@ ORIGINDIR:='$$ORIGIN'
# Components VERSION info
########################################
CNTK_COMPONENT_VERSION := 2.4
# CNTK version which should be used where CNTK version is required. Ex: print version or tag CNTK binaries.
CNTK_VERSION := $(BUILD_VERSION)
# Cntk Version banner is printed wherever CNTK_VERSION should be printed. ex: python -c 'import cntk;cntk.__version__'.
CNTK_VERSION_BANNER := $(CNTK_VERSION)
ifeq ("$(BUILD_PUBLIC)","no")
CNTK_VERSION_BANNER := $(CNTK_VERSION_BANNER)+
endif
# Cntk binaries (generated by build) are appended with CNTK_COMPONENT_VERSION. Ex: libCntk.Core-$(CNTK_COMPONENT_VERSION).dll
CNTK_COMPONENT_VERSION := $(CNTK_VERSION)
ifeq ("$(BUILDTYPE)","debug")
CNTK_COMPONENT_VERSION := $(CNTK_COMPONENT_VERSION)d
endif
CPPFLAGS += -DCNTK_VERSION="$(CNTK_VERSION)"
CPPFLAGS += -DCNTK_VERSION_BANNER="$(CNTK_VERSION_BANNER)"
CPPFLAGS += -DCNTK_COMPONENT_VERSION="$(CNTK_COMPONENT_VERSION)"
CNTKMATH:=Cntk.Math-$(CNTK_COMPONENT_VERSION)
@ -1430,6 +1444,8 @@ python: $(PYTHON_LIBS)
py_paths[36]=$(PYTHON36_PATH); \
export LD_LIBRARY_PATH=$$LD_LIBRARY_PATH:$$(echo $(GDK_NVML_LIB_PATH) $(LIBPATH) | tr " " :); \
ldd $$(find $(LIBDIR) -maxdepth 1 -type f -print) | grep "not found" && false; \
export CNTK_VERSION=$(CNTK_VERSION); \
export CNTK_VERSION_BANNER=$(CNTK_VERSION_BANNER); \
export CNTK_COMPONENT_VERSION=$(CNTK_COMPONENT_VERSION); \
export CNTK_LIBRARIES="$(PYTHON_LIBS)"; \
export CNTK_EXTRA_LIBRARIES=$$(ldd $(LIBDIR)/* | grep "^\s.*=> " | cut -d ">" -f 2- --only-delimited | cut -d "(" -f 1 --only-delimited | sort -u | grep -Ff <(echo $(PYTHON_EXTRA_LIBS_BASENAMES) | xargs -n1)); \

Просмотреть файл

@ -597,7 +597,11 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp
static void PrintBanner(int argc, wchar_t* argv[], const string& timestamp)
{
fprintf(stderr, "CNTK 2.4+ (");
#ifndef CNTK_VERSION_BANNER
#error CNTK_VERSION_BANNER must be set
#endif
#define MACRO_TO_STRING(s) #s
fprintf(stderr, "CNTK %s (", MACRO_TO_STRING(CNTK_VERSION_BANNER));
#ifdef _GIT_EXIST
fprintf(stderr, "%s %.6s, ", _BUILDBRANCH_, _BUILDSHA1_);
#endif

Просмотреть файл

@ -9,8 +9,9 @@
#include "CNTKLibrary.h"
#define CNTK_ONNX_MODEL_VERSION 1
#define MACRO_TO_STRING(s) #s
const std::string CNTK_ONNX_PRODUCER_NAME = "CNTK";
const std::string CNTK_ONNX_PRODUCER_VERSION = "2.4";
const std::string CNTK_ONNX_PRODUCER_VERSION = MACRO_TO_STRING(CNTK_VERSION);
namespace ONNXIR
{

Просмотреть файл

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug_UWP</Configuration>
@ -11,7 +11,7 @@
<AssemblyName>ImageRecognitionAppCS</AssemblyName>
<DefaultLanguage>en-US</DefaultLanguage>
<TargetPlatformIdentifier>UAP</TargetPlatformIdentifier>
<TargetPlatformVersion>10.0.14393.0</TargetPlatformVersion>
<TargetPlatformVersion>10.0.16299.0</TargetPlatformVersion>
<TargetPlatformMinVersion>10.0.10586.0</TargetPlatformMinVersion>
<MinimumVisualStudioVersion>14</MinimumVisualStudioVersion>
<FileAlignment>512</FileAlignment>
@ -46,7 +46,6 @@
<ItemGroup>
<!-- A reference to the entire .Net Framework and Windows SDK are automatically included -->
<Content Include="Assets\imagenet1000_clsid.txt" />
<None Include="project.json" />
</ItemGroup>
<ItemGroup>
<Compile Include="App.xaml.cs">
@ -65,8 +64,10 @@
<Content Include="ImageRecognitionAppCS_TemporaryKey.pfx" />
</ItemGroup>
<PropertyGroup>
<CNTKBinSuffix />
<CNTKBinSuffix Condition="$(Configuration.StartsWith('Debug'))">d</CNTKBinSuffix>
<CntkVersion>2.4</CntkVersion>
<CntkVersion Condition=" '$(BUILD_CNTK_VERSION)' != '' ">$(BUILD_CNTK_VERSION)</CntkVersion>
<CntkComponentVersion>$(CntkVersion)</CntkComponentVersion>
<CntkComponentVersion Condition="$(Configuration.StartsWith('Debug'))">$(CntkComponentVersion)d</CntkComponentVersion>
</PropertyGroup>
<ItemGroup>
<Content Include="Properties\Default.rd.xml" />
@ -77,10 +78,10 @@
<Content Include="Assets\Square44x44Logo.targetsize-24_altform-unplated.png" />
<Content Include="Assets\StoreLogo.png" />
<Content Include="Assets\Wide310x150Logo.scale-200.png" />
<Content Include="$(CNTKRoot)\$(Platform)\$(Configuration)\Cntk.Core_app-2.4$(CNTKBinSuffix).dll">
<Content Include="$(CNTKRoot)\$(Platform)\$(Configuration)\Cntk.Core_app-$(CntkComponentVersion).dll">
<DeploymentContent>true</DeploymentContent>
</Content>
<Content Include="$(CNTKRoot)\$(Platform)\$(Configuration)\Cntk.Math_app-2.4$(CNTKBinSuffix).dll">
<Content Include="$(CNTKRoot)\$(Platform)\$(Configuration)\Cntk.Math_app-$(CntkComponentVersion).dll">
<DeploymentContent>true</DeploymentContent>
</Content>
<Content Include="$(CNTKRoot)\$(Platform)\$(Configuration)\libopenblas.dll">
@ -103,6 +104,11 @@
<HintPath>$(CNTKRoot)$(Platform)\$(Configuration)\ImageRecognizerLib.winmd</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NETCore.UniversalWindowsPlatform">
<Version>5.2.2</Version>
</PackageReference>
</ItemGroup>
<PropertyGroup Condition=" '$(VisualStudioVersion)' == '' or '$(VisualStudioVersion)' &lt; '14.0' ">
<VisualStudioVersion>14.0</VisualStudioVersion>
</PropertyGroup>

Просмотреть файл

@ -1,16 +0,0 @@
{
"dependencies": {
"Microsoft.NETCore.UniversalWindowsPlatform": "5.2.2"
},
"frameworks": {
"uap10.0": {}
},
"runtimes": {
"win10-arm": {},
"win10-arm-aot": {},
"win10-x86": {},
"win10-x86-aot": {},
"win10-x64": {},
"win10-x64-aot": {}
}
}

Просмотреть файл

@ -14,6 +14,7 @@ RUN=1
CLEAN_AFTER=0
CLEAN_BEFORE=0
RANDOM_OUTPUT=0
BUILD_VERSION=
CODE_COVERAGE=no
FLAVORS="debug:release"
TARGETS="cpu:gpu"
@ -40,6 +41,7 @@ case $key in
echo " -cc|--code-coverage - build with support for code coverage (gcov)"
echo " -cb|--clean-build - clean up the enlistment binaries before build"
echo " -cba|--clean-build-after - clean up the enlistment binaries after build"
echo " -bv|--build-version <build_version> - build using given version assuming public availablity (default hard-coded in configure file)"
echo " -rnd|--random-output-suffix - add random suffix to output directory"
echo " -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
echo " -x|--extra-configure-options <options> - extra options to pass to configure"
@ -68,6 +70,10 @@ case $key in
CLEAN_AFTER=1
BUILD=1
;;
-bv|--build-version)
BUILD_VERSION="${2,,}"
shift # past argument
;;
-f|--flavors)
FLAVORS="${2,,}"
shift # past argument
@ -249,7 +255,7 @@ if [[ $BUILD == 1 ]]; then
OneBitSGDOPT=yes
fi
fi
./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT $EXTRA_CONFIGURE_OPTIONS
./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT --with-build-version=$BUILD_VERSION $EXTRA_CONFIGURE_OPTIONS
if [[ $CLEAN_BEFORE == 1 ]]; then
make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
fi

Просмотреть файл

@ -40,7 +40,7 @@
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<NMakeBuildCommandLine>.\vsbuild.bat "$(OutDir)" "$(DebugBuild)" "$(GpuBuild)" "$(CntkComponentVersion)" "$(SWIG_PATH)" "$(CNTK_PY_VERSIONS)" "$(CNTK_PY27_PATH)" "$(CNTK_PY34_PATH)" "$(CNTK_PY35_PATH)" "$(CNTK_PY36_PATH)"</NMakeBuildCommandLine>
<NMakeBuildCommandLine>.\vsbuild.bat "$(OutDir)" "$(DebugBuild)" "$(GpuBuild)" "$(CntkVersion)" "$(CntkVersionBanner)" "$(CntkComponentVersion)" "$(SWIG_PATH)" "$(CNTK_PY_VERSIONS)" "$(CNTK_PY27_PATH)" "$(CNTK_PY34_PATH)" "$(CNTK_PY35_PATH)" "$(CNTK_PY36_PATH)"</NMakeBuildCommandLine>
<NMakeOutput>dist/pythonwheel</NMakeOutput>
<NMakeCleanCommandLine>rmdir /s /q build</NMakeCleanCommandLine>
</PropertyGroup>

Просмотреть файл

@ -18,7 +18,9 @@ call "%VS2017INSTALLDIR%\VC\Auxiliary\build\vcvarsall.bat" amd64 -vcvars_ver=14.
set MSSdk=1
set DISTUTILS_USE_SDK=1
set CNTK_COMPONENT_VERSION=2.4
set CNTK_VERSION=2.4
set CNTK_VERSION_BANNER=%CNTK_VERSION%+
set CNTK_COMPONENT_VERSION=%CNTK_VERSION%
python .\setup.py build_ext --inplace --force --compiler msvc
if errorlevel 1 exit /b 1

Просмотреть файл

@ -3,11 +3,15 @@
# for full license information.
# ==============================================================================
__version__ = '2.4+'
import os
os.environ["PATH"] += os.pathsep + os.path.join(os.path.dirname(__file__), 'libs')
# Read version information
version_file = open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'r')
__version__ = version_file.read()
version_file.close()
del version_file
import numpy as np
from . import cntk_py

Просмотреть файл

@ -9,8 +9,8 @@ If you have installed CNTK on your machine, after going through the :cntkwiki:`i
you can start using CNTK from Python right away (don't forget to ``activate`` your Python environment if you did not install CNTK into your root environment):
>>> import cntk
>>> cntk.__version__
'2.4+'
You can check CNTK version using ``cntk.__version__``.
>>> cntk.minus([1, 2, 3], [4, 5, 6]).eval()
array([-3., -3., -3.], dtype=float32)

Просмотреть файл

@ -82,6 +82,11 @@ else:
rt_libs = [strip_path(fn) for fn in glob(os.path.join(CNTK_LIB_PATH,
'*' + libname_rt_ext))]
# copy CNTK_VERSION_BANNER to VERSION file
version_file = open(os.path.join(os.path.dirname(__file__), "cntk", "VERSION"), 'w')
version_file.write(os.environ['CNTK_VERSION_BANNER'])
version_file.close()
# copy over the libraries to the cntk base directory so that the rpath is
# correctly set
if os.path.exists(PROJ_LIB_PATH):
@ -122,7 +127,7 @@ else:
]
extra_link_args = []
# Expecting the dependent libs (libcntklibrary-2.4.so, etc.) inside
# Expecting the dependent libs (libcntklibrary-[CNTK_COMPONENT_VERSION].so, etc.) inside
# site-packages/cntk/libs.
runtime_library_dirs = ['$ORIGIN/cntk/libs']
os.environ["CXX"] = "mpic++"
@ -158,7 +163,7 @@ cntk_module = Extension(
# Do not include examples
packages = [x for x in find_packages() if x.startswith('cntk') and not x.startswith('cntk.swig')]
package_data = { 'cntk': ['pytest.ini', 'io/tests/tf_data.txt', 'contrib/deeprl/tests/data/initial_policy_network.dnn'] }
package_data = { 'cntk': ['pytest.ini', 'io/tests/tf_data.txt', 'contrib/deeprl/tests/data/initial_policy_network.dnn', 'VERSION'] }
package_data['cntk'] += rt_libs
kwargs = dict(package_data = package_data)
@ -171,7 +176,7 @@ if IS_PY2:
cntk_install_requires.append('enum34>=1.1.6')
setup(name="cntk",
version="2.4",
version=os.environ['CNTK_VERSION'],
url="http://cntk.ai",
ext_modules=[cntk_module],
packages=packages,

Просмотреть файл

@ -15,6 +15,10 @@ REM overridden at msbuild invocation.
set p_OutDir=%~1
set p_DebugBuild=%~2
set p_GpuBuild=%~3
set p_CNTK_VERSION=%~4
shift
set p_CNTK_VERSION_BANNER=%~4
shift
set p_CNTK_COMPONENT_VERSION=%~4
set p_SWIG_PATH=%~5
set p_CNTK_PY_VERSIONS=%~6
@ -56,6 +60,8 @@ set CNTK_LIB_PATH=%p_OutDir%
set DIST_DIR=%p_OutDir%\Python
set PATH=%p_SWIG_PATH%;%PATH%
set CNTK_VERSION=%p_CNTK_VERSION%
set CNTK_VERSION_BANNER=%p_CNTK_VERSION_BANNER%
set CNTK_COMPONENT_VERSION=%p_CNTK_COMPONENT_VERSION%
set MSSdk=1
set DISTUTILS_USE_SDK=1

17
configure поставляемый
Просмотреть файл

@ -18,6 +18,13 @@ enable_python=
enable_java=
# cntk build version
# Modify both of the following during major release
default_build_version="2.4"
default_build_public="no"
build_version=$default_build_version
build_public=$default_build_public
# NCCL communication library
have_nccl=no
nccl_path=
@ -406,6 +413,7 @@ function show_help ()
echo " --with-swig[=directory] $(show_default $(find_swig))"
echo " --with-mpi[=directory] $(show_default $(find_mpi))"
echo " --with-halide[=directory] $(show_default $(find_halide))"
echo " --with-build-version=[version] $default_build_version"
echo "Libraries search path:"
for head in $(default_paths)
@ -948,6 +956,13 @@ do
fi
fi
;;
--with-build-version*)
if test "x$optarg" != "x"
then
build_version=$optarg
build_public="yes"
fi
;;
*)
echo Invalid option $key
show_help
@ -1180,6 +1195,8 @@ config=$build_top/Config.make
echo Generating $config
echo "#Configuration file for cntk" > $config
echo BUILDTYPE=$buildtype >> $config
echo BUILD_VERSION=$build_version >> $config
echo BUILD_PUBLIC=$build_public >> $config
echo MATHLIB=$mathlib >> $config
case $mathlib in
mkl)