diff --git a/MachineLearning/CNTK/CNTK.cpp b/MachineLearning/CNTK/CNTK.cpp index 399cc17d0..7435bb988 100644 --- a/MachineLearning/CNTK/CNTK.cpp +++ b/MachineLearning/CNTK/CNTK.cpp @@ -17,8 +17,8 @@ #include #if defined(_WIN32) #include "io.h" -#include "buildinfo.h" #endif +#include "buildinfo.h" #include "hostname.h" #ifdef LEAKDETECT #include "vld.h" // for memory leak detection @@ -50,12 +50,12 @@ #include "ProgressTracing.h" #include "fileutil.h" #include "ScriptableObjects.h" -#include "BrainScriptEvaluator.h" -#include "BrainScriptParser.h" +#include "BrainScriptEvaluator.h" +#include "BrainScriptParser.h" -#ifndef let -#define let const auto -#endif +#ifndef let +#define let const auto +#endif // TODO: Get rid of this global Microsoft::MSR::CNTK::MPIWrapper *g_mpi = nullptr; @@ -775,80 +775,80 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config) } } -template -class BrainScriptNetworkBuilder : public IComputationNetBuilder -{ - typedef shared_ptr ComputationNetworkPtr; - ComputationNetworkPtr m_net; - ScriptableObjects::ConfigLambdaPtr m_createNetworkFn; - DEVICEID_TYPE m_deviceId; -public: - // the constructor remembers the config lambda - // TODO: Really this should just take the lambda itself, or rather, this class should just be replaced by a lambda. But we need the IConfigRecord for templates to be compile-compatible with old CNTK config. - BrainScriptNetworkBuilder(const ScriptableObjects::IConfigRecord & config) - { - m_deviceId = config[L"deviceId"]; // TODO: only needed for LoadNetworkFromFile() which should go away anyway +template +class BrainScriptNetworkBuilder : public IComputationNetBuilder +{ + typedef shared_ptr ComputationNetworkPtr; + ComputationNetworkPtr m_net; + ScriptableObjects::ConfigLambdaPtr m_createNetworkFn; + DEVICEID_TYPE m_deviceId; +public: + // the constructor remembers the config lambda + // TODO: Really this should just take the lambda itself, or rather, this class should just be replaced by a lambda. But we need the IConfigRecord for templates to be compile-compatible with old CNTK config. + BrainScriptNetworkBuilder(const ScriptableObjects::IConfigRecord & config) + { + m_deviceId = config[L"deviceId"]; // TODO: only needed for LoadNetworkFromFile() which should go away anyway m_createNetworkFn = config[L"createNetwork"].AsPtr(); - } - // not supported for old CNTK - BrainScriptNetworkBuilder(const ConfigParameters & config) { NOT_IMPLEMENTED; } - - // build a ComputationNetwork from description language - virtual /*IComputationNetBuilder::*/ComputationNetworkPtr BuildNetworkFromDescription(ComputationNetwork* = nullptr) override - { - vector args; // this lambda has no arguments - ScriptableObjects::ConfigLambda::NamedParams namedArgs; - let netValue = m_createNetworkFn->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription"); - m_net = netValue.AsPtr(); - if (m_net->GetDeviceId() < 0) - fprintf(stderr, "BrainScriptNetworkBuilder using CPU\n"); - else - fprintf(stderr, "BrainScriptNetworkBuilder using GPU %d\n", (int)m_net->GetDeviceId()); - return m_net; - } - - // load an existing file--this is the same code as for NDLNetworkBuilder.h (OK to copy it here because this is temporary code anyway) - // TODO: This does not belong into NetworkBuilder, since the code is the same for all. Just create the network and load the darn thing. - virtual /*IComputationNetBuilder::*/ComputationNetwork* LoadNetworkFromFile(const wstring& modelFileName, bool forceLoad = true, - bool bAllowNoCriterionNode = false, ComputationNetwork* anotherNetwork = nullptr) override - { - if (!m_net || m_net->GetTotalNumberOfNodes() == 0 || forceLoad) //not built or force load --TODO: why all these options? - { - auto net = make_shared(m_deviceId); - net->LoadFromFile(modelFileName, FileOptions::fileOptionsBinary, bAllowNoCriterionNode, anotherNetwork); - m_net = net; - } - m_net->ResetEvalTimeStamp(); - return m_net.get(); - } -}; - -// TODO: decide where these should go. Also, do we need three variables? -extern wstring standardFunctions; -extern wstring commonMacros; -extern wstring computationNodes; + } + // not supported for old CNTK + BrainScriptNetworkBuilder(const ConfigParameters & config) { NOT_IMPLEMENTED; } -// helper that returns 'float' or 'double' depending on ElemType -template static const wchar_t * ElemTypeName(); -template<> /*static*/ const wchar_t * ElemTypeName() { return L"float"; } -template<> /*static*/ const wchar_t * ElemTypeName() { return L"double"; } - -function GetCreateNetworkFn(const ScriptableObjects::IConfigRecord & config) -{ + // build a ComputationNetwork from description language + virtual /*IComputationNetBuilder::*/ComputationNetworkPtr BuildNetworkFromDescription(ComputationNetwork* = nullptr) override + { + vector args; // this lambda has no arguments + ScriptableObjects::ConfigLambda::NamedParams namedArgs; + let netValue = m_createNetworkFn->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription"); + m_net = netValue.AsPtr(); + if (m_net->GetDeviceId() < 0) + fprintf(stderr, "BrainScriptNetworkBuilder using CPU\n"); + else + fprintf(stderr, "BrainScriptNetworkBuilder using GPU %d\n", (int)m_net->GetDeviceId()); + return m_net; + } + + // load an existing file--this is the same code as for NDLNetworkBuilder.h (OK to copy it here because this is temporary code anyway) + // TODO: This does not belong into NetworkBuilder, since the code is the same for all. Just create the network and load the darn thing. + virtual /*IComputationNetBuilder::*/ComputationNetwork* LoadNetworkFromFile(const wstring& modelFileName, bool forceLoad = true, + bool bAllowNoCriterionNode = false, ComputationNetwork* anotherNetwork = nullptr) override + { + if (!m_net || m_net->GetTotalNumberOfNodes() == 0 || forceLoad) //not built or force load --TODO: why all these options? + { + auto net = make_shared(m_deviceId); + net->LoadFromFile(modelFileName, FileOptions::fileOptionsBinary, bAllowNoCriterionNode, anotherNetwork); + m_net = net; + } + m_net->ResetEvalTimeStamp(); + return m_net.get(); + } +}; + +// TODO: decide where these should go. Also, do we need three variables? +extern wstring standardFunctions; +extern wstring commonMacros; +extern wstring computationNodes; + +// helper that returns 'float' or 'double' depending on ElemType +template static const wchar_t * ElemTypeName(); +template<> /*static*/ const wchar_t * ElemTypeName() { return L"float"; } +template<> /*static*/ const wchar_t * ElemTypeName() { return L"double"; } + +function GetCreateNetworkFn(const ScriptableObjects::IConfigRecord & config) +{ // createNetwork() is a BrainScript lambda that creates the model // We create a C++ wrapper around it, which we then pass to Train(). auto createNetworkConfigLambda = config[L"createNetwork"].AsPtr(); return [createNetworkConfigLambda](DEVICEID_TYPE /*deviceId*/) { // execute the lambda - vector args; // this lambda has no arguments - ScriptableObjects::ConfigLambda::NamedParams namedArgs; - let netValue = createNetworkConfigLambda->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription"); - // typecast the result to the desired type - return netValue.AsPtr(); + vector args; // this lambda has no arguments + ScriptableObjects::ConfigLambda::NamedParams namedArgs; + let netValue = createNetworkConfigLambda->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription"); + // typecast the result to the desired type + return netValue.AsPtr(); }; -} -function GetCreateNetworkFn(const ConfigParameters &) { NOT_IMPLEMENTED; } // old CNTK config does not support lambdas +} +function GetCreateNetworkFn(const ConfigParameters &) { NOT_IMPLEMENTED; } // old CNTK config does not support lambdas // function to create an object of a certain type, using both old CNTK config and BrainScript template @@ -901,26 +901,26 @@ void DoTrain(const ConfigRecordType & config) // legacy test mode for BrainScript. Will go away once we fully integrate with BS. else if (config.Exists(L"ExperimentalNetworkBuilder")) { - // We interface with outer old CNTK config by taking the inner part, which we get as a string, as BrainScript. - // We prepend a few standard definitions, and also definition of deviceId and precision, which all objects will pull out again when they are being constructed. - // BUGBUG: We are not getting TextLocations right in this way! Do we need to inject location markers into the source? Moot once we fully switch to BS - wstring sourceCode = config(L"ExperimentalNetworkBuilder"); - let expr = BS::ParseConfigDictFromString(standardFunctions + computationNodes + commonMacros - + msra::strfun::wstrprintf(L"deviceId = %d ; precision = '%ls' ; network = new ComputationNetwork ", (int)deviceId, ElemTypeName()) // TODO: check if typeid needs postprocessing - + sourceCode, vector()); // source code has the form [ ... ] + // We interface with outer old CNTK config by taking the inner part, which we get as a string, as BrainScript. + // We prepend a few standard definitions, and also definition of deviceId and precision, which all objects will pull out again when they are being constructed. + // BUGBUG: We are not getting TextLocations right in this way! Do we need to inject location markers into the source? Moot once we fully switch to BS + wstring sourceCode = config(L"ExperimentalNetworkBuilder"); + let expr = BS::ParseConfigDictFromString(standardFunctions + computationNodes + commonMacros + + msra::strfun::wstrprintf(L"deviceId = %d ; precision = '%ls' ; network = new ComputationNetwork ", (int)deviceId, ElemTypeName()) // TODO: check if typeid needs postprocessing + + sourceCode, vector()); // source code has the form [ ... ] createNetworkFn = [expr](DEVICEID_TYPE /*deviceId*/) { - // evaluate the parse tree--specifically the top-level field 'network'--which will create the network - let object = EvaluateField(expr, L"network"); // this comes back as a BS::Object - let network = dynamic_pointer_cast(object); // cast it - // This should not really fail since we constructed the source code above such that this is the right type. - // However, it is possible (though currently not meaningful) to locally declare a different 'precision' value. - // In that case, the network might come back with a different element type. We need a runtime check for that. - if (!network) - RuntimeError("BuildNetworkFromDescription: network has the wrong element type (float vs. double)"); - // success - network->ResetEvalTimeStamp(); - return network; + // evaluate the parse tree--specifically the top-level field 'network'--which will create the network + let object = EvaluateField(expr, L"network"); // this comes back as a BS::Object + let network = dynamic_pointer_cast(object); // cast it + // This should not really fail since we constructed the source code above such that this is the right type. + // However, it is possible (though currently not meaningful) to locally declare a different 'precision' value. + // In that case, the network might come back with a different element type. We need a runtime check for that. + if (!network) + RuntimeError("BuildNetworkFromDescription: network has the wrong element type (float vs. double)"); + // success + network->ResetEvalTimeStamp(); + return network; }; } else @@ -948,32 +948,32 @@ void DoTrain(const ConfigRecordType & config) optimizer->Train(createNetworkFn, deviceId, dataReader.get(), cvDataReader.get(), makeMode); } -namespace Microsoft { namespace MSR { namespace ScriptableObjects { - - using namespace Microsoft::MSR::CNTK; - - // ----------------------------------------------------------------------- - // register ComputationNode with the ScriptableObject system - // ----------------------------------------------------------------------- - +namespace Microsoft { namespace MSR { namespace ScriptableObjects { + + using namespace Microsoft::MSR::CNTK; + + // ----------------------------------------------------------------------- + // register ComputationNode with the ScriptableObject system + // ----------------------------------------------------------------------- + class TrainAction { }; - template<> shared_ptr MakeRuntimeObject(const IConfigRecordPtr configp) - { - const IConfigRecord & config = *configp; + template<> shared_ptr MakeRuntimeObject(const IConfigRecordPtr configp) + { + const IConfigRecord & config = *configp; wstring precision = config[L"precision"]; // dispatch on ElemType if (precision == L"float") - DoTrain(config); + DoTrain(config); else if (precision == L"double") - DoTrain(config); + DoTrain(config); else RuntimeError("invalid value '%ls' for 'precision', must be 'float' or 'double'", precision.c_str()); - - return make_shared(); // return a dummy object - } - - // register ComputationNode with the ScriptableObject system - ScriptableObjects::ConfigurableRuntimeTypeRegister::Add registerTrainAction(L"TrainAction"); -}}} + + return make_shared(); // return a dummy object + } + + // register ComputationNode with the ScriptableObject system + ScriptableObjects::ConfigurableRuntimeTypeRegister::Add registerTrainAction(L"TrainAction"); +}}} template void DoAdapt(const ConfigParameters& config) @@ -1373,7 +1373,7 @@ void DoConvertFromDbn(const ConfigParameters& config) wstring dbnModelPath = config(L"dbnModelPath"); auto netBuilder = make_shared>(config); - ComputationNetworkPtr net = netBuilder->BuildNetworkFromDbnFile(dbnModelPath); + ComputationNetworkPtr net = netBuilder->BuildNetworkFromDbnFile(dbnModelPath); net->SaveToFile(modelPath); } @@ -1643,24 +1643,36 @@ std::string TimeDateStamp() return buf; } -#ifdef _WIN32 void PrintBuiltInfo() { fprintf(stderr, "-------------------------------------------------------------------\n"); fprintf(stderr, "Build info: \n\n"); fprintf(stderr, "\t\tBuilt time: %s %s\n", __DATE__, __TIME__); fprintf(stderr, "\t\tLast modified date: %s\n", __TIMESTAMP__); - fprintf(stderr, "\t\tBuilt by %s on %s\n", _BUILDER_, _BUILDMACHINE_); - fprintf(stderr, "\t\tBuild Path: %s\n", _BUILDPATH_); +#ifdef _BUILDTYPE_ + fprintf(stderr, "\t\tBuild type: %s\n", _BUILDTYPE_); +#endif +#ifdef _MATHLIB_ + fprintf(stderr, "\t\tMath lib: %s\n", _MATHLIB_); +#endif +#ifdef _CUDA_PATH_ fprintf(stderr, "\t\tCUDA_PATH: %s\n", _CUDA_PATH_); +#endif +#ifdef _CUB_PATH_ + fprintf(stderr, "\t\tCUDA_PATH: %s\n", _CUB_PATH_); +#endif #ifdef _GIT_EXIST fprintf(stderr, "\t\tBuild Branch: %s\n", _BUILDBRANCH_); fprintf(stderr, "\t\tBuild SHA1: %s\n", _BUILDSHA1_); #endif - fprintf(stderr, "-------------------------------------------------------------------\n"); - -} +#ifdef _BUILDER_ + fprintf(stderr, "\t\tBuilt by %s on %s\n", _BUILDER_, _BUILDMACHINE_); +#endif +#ifdef _BUILDPATH_ + fprintf(stderr, "\t\tBuild Path: %s\n", _BUILDPATH_); #endif + fprintf(stderr, "-------------------------------------------------------------------\n"); +} void PrintUsageInfo() { @@ -1733,7 +1745,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra // change working directory if (workingDir != L"") - _wchdir(workingDir.c_str()); + _wchdir(workingDir.c_str()); // compile the BrainScript wstring bs = L"[\n"; @@ -1748,7 +1760,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra let expr = BS::ParseConfigExpression(bs, move(includePaths)); // parse let valp = BS::Evaluate(expr); // evaluate parse into a dictionary - let & config = valp.AsRef(); // this is the dictionary + let & config = valp.AsRef(); // this is the dictionary // legacy parameters that have changed spelling if (config.Find(L"DoneFile")) // variables follow camel case (start with lower-case letters) @@ -1779,9 +1791,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra } // echo config info to log -#ifdef _WIN32 PrintBuiltInfo(); -#endif // execute the actions //std::string type = config(L"precision", "float"); @@ -1867,9 +1877,7 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which i RedirectStdErr(logpath); } -#ifdef _WIN32 PrintBuiltInfo(); -#endif std::string timestamp = TimeDateStamp(); //dump config info diff --git a/Makefile b/Makefile index 48d15c842..1d8790b91 100644 --- a/Makefile +++ b/Makefile @@ -180,6 +180,18 @@ ORIGINDIR:='$$ORIGIN' CNTKMATH:=cntkmath + +######################################## +# Build info +######################################## + +BUILDINFO:= MachineLearning/CNTK/buildinfo.h + +$(BUILDINFO): Scripts/genrate_build_info + @echo creating $@ for $(ARCH) with build type $(BUILDTYPE) + @Scripts/genrate_build_info + + ######################################## # Math library ######################################## @@ -451,7 +463,7 @@ CNTK_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(C CNTK:=$(BINDIR)/cntk ALL+=$(CNTK) -$(CNTK): $(CNTK_OBJ) | $(CNTKMATH_LIB) +$(CNTK): $(BUILDINFO) $(CNTK_OBJ) | $(CNTKMATH_LIB) @echo $(SEPARATOR) @mkdir -p $(dir $@) @echo building output for $(ARCH) with build type $(BUILDTYPE) @@ -485,7 +497,10 @@ $(OBJDIR)/%.o : %.cpp Makefile @mkdir -p $(dir $@) $(CXX) -c $< -o $@ $(CPPFLAGS) $(CXXFLAGS) $(INCLUDEPATH:%=-I%) -MD -MP -MF ${@:.o=.d} -.PHONY: clean buildall all +.PHONY: force clean buildall all + +force: $(BUILDINFO) + clean: @echo $(SEPARATOR) diff --git a/Scripts/genrate_build_info b/Scripts/genrate_build_info new file mode 100755 index 000000000..d115e1a72 --- /dev/null +++ b/Scripts/genrate_build_info @@ -0,0 +1,104 @@ +#!/bin/bash + + +# description: +# this script is used to generated buildinfo.h in MachineLearning/CNTK which will contain the following infomation to be displayed at runtime: +# CUDA_PATH (if exists) +# CUB_PATH (if exists) +# GIT_COMMIT +# GTT_BRANCH +# BUILDTYPE (release/debug) +# MATHLIB (MKL/ACML) + +usage () +{ + echo "usage: $0 " + echo "-------------------------------------------------------------------" + echo "This script is used to generate buildinfo.h in MachineLearning/CNTK" + echo "This script needs to be called from the top level directory of CNTK project" + echo "This script assumes git can be used" + echo "This script assumes Config.make has been made" + echo "-------------------------------------------------------------------" + if [ ! -z "$1" ] ; then + echo "ERROR message: $1" + fi + exit 1 +} + +Has_Git() +{ + if hash git 2>/dev/null; then + return 0 + else + return 1 + fi +} + +makebuildinfo() +{ + target=$1 + BUILDTYPE=$2 + MATHLIB=$3 + GIT_COMMIT=$4 + GIT_BRANCH=$5 + CUDA_PATH=$6 + CUB_PATH=$7 + + printf "#ifndef _BUILDINFO_H\n" > $target + printf "#define _BUILDINFO_H\n" >> $target + printf "#define _GIT_EXIST\n" >> $target + printf "#define _MATHLIB_ \"%s\"\n" $MATHLIB >> $target + printf "#define _BUILDSHA1_ \"%s\"\n" $GIT_COMMIT >> $target + printf "#define _BUILDBRANCH_ \"%s\"\n" $GIT_BRANCH >> $target + if [ ! -z "$CUDA_PATH" ]; then + printf "#define _CUDA_PATH_ \"%s\"\n" $CUDA_PATH >> $target + fi + if [ ! -z "$CUB_PATH" ]; then + printf "#define _CUB_PATH_ \"%s\"\n" $CUB_PATH >> $target + fi + printf "#define _BUILDTYPE_ \"%s\"\n" $BUILDTYPE >> $target + printf "#endif\n" >> $target +} + +#//////////////////////////////////////////////////////# +# main function # +#//////////////////////////////////////////////////////# +if [ $# -ne 0 ]; then + usage +fi + + +# 1. check whether we have git and what is the sha-1 value +if Has_Git; then has_git=1; else has_git=0; usage "git not exist"; fi +GIT_COMMIT=`git rev-parse HEAD` +GIT_BRANCH=`git rev-parse --abbrev-ref HEAD` + +# 2. looking into Config.make +if [ ! -e Config.make ] ; then + usage "Config.make not exists" +fi +source "Config.make" + +# 3. whether we have CUDA_PATH +if [ -z "${CUDA_PATH+x}" ]; then + CUDAPATH="" +else + CUDAPATH=$CUDA_PATH +fi + +# 4. whether we have CUB_PATH +if [ -z "${CUB_PATH+x}" ]; then + CUBPATH="" +else + CUBPATH=$CUB_PATH +fi + +# 5. make buildinfo.h +target=MachineLearning/CNTK/buildinfo.h +if [ ! -d MachineLearning ] ; then + usage +fi +if [ -e MachineLearning/CNTK/buildinfo.h ] ; then + rm MachineLearning/CNTK/buildinfo.h +fi +makebuildinfo $target $BUILDTYPE $MATHLIB $GIT_COMMIT $GIT_BRANCH $CUDAPATH $CUBPATH