Enable PrintBuildInfo for both Windows and Linux

This commit is contained in:
Yongqiang Wang 2015-11-24 16:38:37 -08:00
Родитель 2043cd189a
Коммит 33cf6a013f
3 изменённых файлов: 251 добавлений и 124 удалений

Просмотреть файл

@ -17,8 +17,8 @@
#include <algorithm>
#if defined(_WIN32)
#include "io.h"
#include "buildinfo.h"
#endif
#include "buildinfo.h"
#include "hostname.h"
#ifdef LEAKDETECT
#include "vld.h" // for memory leak detection
@ -50,12 +50,12 @@
#include "ProgressTracing.h"
#include "fileutil.h"
#include "ScriptableObjects.h"
#include "BrainScriptEvaluator.h"
#include "BrainScriptParser.h"
#include "BrainScriptEvaluator.h"
#include "BrainScriptParser.h"
#ifndef let
#define let const auto
#endif
#ifndef let
#define let const auto
#endif
// TODO: Get rid of this global
Microsoft::MSR::CNTK::MPIWrapper *g_mpi = nullptr;
@ -775,80 +775,80 @@ void DoWriteWordAndClassInfo(const ConfigParameters& config)
}
}
template<class ElemType>
class BrainScriptNetworkBuilder : public IComputationNetBuilder<ElemType>
{
typedef shared_ptr<ComputationNetwork> ComputationNetworkPtr;
ComputationNetworkPtr m_net;
ScriptableObjects::ConfigLambdaPtr m_createNetworkFn;
DEVICEID_TYPE m_deviceId;
public:
// the constructor remembers the config lambda
// TODO: Really this should just take the lambda itself, or rather, this class should just be replaced by a lambda. But we need the IConfigRecord for templates to be compile-compatible with old CNTK config.
BrainScriptNetworkBuilder(const ScriptableObjects::IConfigRecord & config)
{
m_deviceId = config[L"deviceId"]; // TODO: only needed for LoadNetworkFromFile() which should go away anyway
template<class ElemType>
class BrainScriptNetworkBuilder : public IComputationNetBuilder<ElemType>
{
typedef shared_ptr<ComputationNetwork> ComputationNetworkPtr;
ComputationNetworkPtr m_net;
ScriptableObjects::ConfigLambdaPtr m_createNetworkFn;
DEVICEID_TYPE m_deviceId;
public:
// the constructor remembers the config lambda
// TODO: Really this should just take the lambda itself, or rather, this class should just be replaced by a lambda. But we need the IConfigRecord for templates to be compile-compatible with old CNTK config.
BrainScriptNetworkBuilder(const ScriptableObjects::IConfigRecord & config)
{
m_deviceId = config[L"deviceId"]; // TODO: only needed for LoadNetworkFromFile() which should go away anyway
m_createNetworkFn = config[L"createNetwork"].AsPtr<ScriptableObjects::ConfigLambda>();
}
// not supported for old CNTK
BrainScriptNetworkBuilder(const ConfigParameters & config) { NOT_IMPLEMENTED; }
// build a ComputationNetwork from description language
virtual /*IComputationNetBuilder::*/ComputationNetworkPtr BuildNetworkFromDescription(ComputationNetwork* = nullptr) override
{
vector<ScriptableObjects::ConfigValuePtr> args; // this lambda has no arguments
ScriptableObjects::ConfigLambda::NamedParams namedArgs;
let netValue = m_createNetworkFn->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription");
m_net = netValue.AsPtr<ComputationNetwork>();
if (m_net->GetDeviceId() < 0)
fprintf(stderr, "BrainScriptNetworkBuilder using CPU\n");
else
fprintf(stderr, "BrainScriptNetworkBuilder using GPU %d\n", (int)m_net->GetDeviceId());
return m_net;
}
// load an existing file--this is the same code as for NDLNetworkBuilder.h (OK to copy it here because this is temporary code anyway)
// TODO: This does not belong into NetworkBuilder, since the code is the same for all. Just create the network and load the darn thing.
virtual /*IComputationNetBuilder::*/ComputationNetwork* LoadNetworkFromFile(const wstring& modelFileName, bool forceLoad = true,
bool bAllowNoCriterionNode = false, ComputationNetwork* anotherNetwork = nullptr) override
{
if (!m_net || m_net->GetTotalNumberOfNodes() == 0 || forceLoad) //not built or force load --TODO: why all these options?
{
auto net = make_shared<ComputationNetwork>(m_deviceId);
net->LoadFromFile<ElemType>(modelFileName, FileOptions::fileOptionsBinary, bAllowNoCriterionNode, anotherNetwork);
m_net = net;
}
m_net->ResetEvalTimeStamp();
return m_net.get();
}
};
// TODO: decide where these should go. Also, do we need three variables?
extern wstring standardFunctions;
extern wstring commonMacros;
extern wstring computationNodes;
}
// not supported for old CNTK
BrainScriptNetworkBuilder(const ConfigParameters & config) { NOT_IMPLEMENTED; }
// helper that returns 'float' or 'double' depending on ElemType
template<class ElemType> static const wchar_t * ElemTypeName();
template<> /*static*/ const wchar_t * ElemTypeName<float>() { return L"float"; }
template<> /*static*/ const wchar_t * ElemTypeName<double>() { return L"double"; }
function<ComputationNetworkPtr(DEVICEID_TYPE)> GetCreateNetworkFn(const ScriptableObjects::IConfigRecord & config)
{
// build a ComputationNetwork from description language
virtual /*IComputationNetBuilder::*/ComputationNetworkPtr BuildNetworkFromDescription(ComputationNetwork* = nullptr) override
{
vector<ScriptableObjects::ConfigValuePtr> args; // this lambda has no arguments
ScriptableObjects::ConfigLambda::NamedParams namedArgs;
let netValue = m_createNetworkFn->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription");
m_net = netValue.AsPtr<ComputationNetwork>();
if (m_net->GetDeviceId() < 0)
fprintf(stderr, "BrainScriptNetworkBuilder using CPU\n");
else
fprintf(stderr, "BrainScriptNetworkBuilder using GPU %d\n", (int)m_net->GetDeviceId());
return m_net;
}
// load an existing file--this is the same code as for NDLNetworkBuilder.h (OK to copy it here because this is temporary code anyway)
// TODO: This does not belong into NetworkBuilder, since the code is the same for all. Just create the network and load the darn thing.
virtual /*IComputationNetBuilder::*/ComputationNetwork* LoadNetworkFromFile(const wstring& modelFileName, bool forceLoad = true,
bool bAllowNoCriterionNode = false, ComputationNetwork* anotherNetwork = nullptr) override
{
if (!m_net || m_net->GetTotalNumberOfNodes() == 0 || forceLoad) //not built or force load --TODO: why all these options?
{
auto net = make_shared<ComputationNetwork>(m_deviceId);
net->LoadFromFile<ElemType>(modelFileName, FileOptions::fileOptionsBinary, bAllowNoCriterionNode, anotherNetwork);
m_net = net;
}
m_net->ResetEvalTimeStamp();
return m_net.get();
}
};
// TODO: decide where these should go. Also, do we need three variables?
extern wstring standardFunctions;
extern wstring commonMacros;
extern wstring computationNodes;
// helper that returns 'float' or 'double' depending on ElemType
template<class ElemType> static const wchar_t * ElemTypeName();
template<> /*static*/ const wchar_t * ElemTypeName<float>() { return L"float"; }
template<> /*static*/ const wchar_t * ElemTypeName<double>() { return L"double"; }
function<ComputationNetworkPtr(DEVICEID_TYPE)> GetCreateNetworkFn(const ScriptableObjects::IConfigRecord & config)
{
// createNetwork() is a BrainScript lambda that creates the model
// We create a C++ wrapper around it, which we then pass to Train().
auto createNetworkConfigLambda = config[L"createNetwork"].AsPtr<ScriptableObjects::ConfigLambda>();
return [createNetworkConfigLambda](DEVICEID_TYPE /*deviceId*/)
{
// execute the lambda
vector<ScriptableObjects::ConfigValuePtr> args; // this lambda has no arguments
ScriptableObjects::ConfigLambda::NamedParams namedArgs;
let netValue = createNetworkConfigLambda->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription");
// typecast the result to the desired type
return netValue.AsPtr<ComputationNetwork>();
vector<ScriptableObjects::ConfigValuePtr> args; // this lambda has no arguments
ScriptableObjects::ConfigLambda::NamedParams namedArgs;
let netValue = createNetworkConfigLambda->Apply(move(args), move(namedArgs), L"BuildNetworkFromDescription");
// typecast the result to the desired type
return netValue.AsPtr<ComputationNetwork>();
};
}
function<ComputationNetworkPtr(DEVICEID_TYPE)> GetCreateNetworkFn(const ConfigParameters &) { NOT_IMPLEMENTED; } // old CNTK config does not support lambdas
}
function<ComputationNetworkPtr(DEVICEID_TYPE)> GetCreateNetworkFn(const ConfigParameters &) { NOT_IMPLEMENTED; } // old CNTK config does not support lambdas
// function to create an object of a certain type, using both old CNTK config and BrainScript
template<class C>
@ -901,26 +901,26 @@ void DoTrain(const ConfigRecordType & config)
// legacy test mode for BrainScript. Will go away once we fully integrate with BS.
else if (config.Exists(L"ExperimentalNetworkBuilder"))
{
// We interface with outer old CNTK config by taking the inner part, which we get as a string, as BrainScript.
// We prepend a few standard definitions, and also definition of deviceId and precision, which all objects will pull out again when they are being constructed.
// BUGBUG: We are not getting TextLocations right in this way! Do we need to inject location markers into the source? Moot once we fully switch to BS
wstring sourceCode = config(L"ExperimentalNetworkBuilder");
let expr = BS::ParseConfigDictFromString(standardFunctions + computationNodes + commonMacros
+ msra::strfun::wstrprintf(L"deviceId = %d ; precision = '%ls' ; network = new ComputationNetwork ", (int)deviceId, ElemTypeName<ElemType>()) // TODO: check if typeid needs postprocessing
+ sourceCode, vector<wstring>()); // source code has the form [ ... ]
// We interface with outer old CNTK config by taking the inner part, which we get as a string, as BrainScript.
// We prepend a few standard definitions, and also definition of deviceId and precision, which all objects will pull out again when they are being constructed.
// BUGBUG: We are not getting TextLocations right in this way! Do we need to inject location markers into the source? Moot once we fully switch to BS
wstring sourceCode = config(L"ExperimentalNetworkBuilder");
let expr = BS::ParseConfigDictFromString(standardFunctions + computationNodes + commonMacros
+ msra::strfun::wstrprintf(L"deviceId = %d ; precision = '%ls' ; network = new ComputationNetwork ", (int)deviceId, ElemTypeName<ElemType>()) // TODO: check if typeid needs postprocessing
+ sourceCode, vector<wstring>()); // source code has the form [ ... ]
createNetworkFn = [expr](DEVICEID_TYPE /*deviceId*/)
{
// evaluate the parse tree--specifically the top-level field 'network'--which will create the network
let object = EvaluateField(expr, L"network"); // this comes back as a BS::Object
let network = dynamic_pointer_cast<ComputationNetwork>(object); // cast it
// This should not really fail since we constructed the source code above such that this is the right type.
// However, it is possible (though currently not meaningful) to locally declare a different 'precision' value.
// In that case, the network might come back with a different element type. We need a runtime check for that.
if (!network)
RuntimeError("BuildNetworkFromDescription: network has the wrong element type (float vs. double)");
// success
network->ResetEvalTimeStamp();
return network;
// evaluate the parse tree--specifically the top-level field 'network'--which will create the network
let object = EvaluateField(expr, L"network"); // this comes back as a BS::Object
let network = dynamic_pointer_cast<ComputationNetwork>(object); // cast it
// This should not really fail since we constructed the source code above such that this is the right type.
// However, it is possible (though currently not meaningful) to locally declare a different 'precision' value.
// In that case, the network might come back with a different element type. We need a runtime check for that.
if (!network)
RuntimeError("BuildNetworkFromDescription: network has the wrong element type (float vs. double)");
// success
network->ResetEvalTimeStamp();
return network;
};
}
else
@ -948,32 +948,32 @@ void DoTrain(const ConfigRecordType & config)
optimizer->Train(createNetworkFn, deviceId, dataReader.get(), cvDataReader.get(), makeMode);
}
namespace Microsoft { namespace MSR { namespace ScriptableObjects {
using namespace Microsoft::MSR::CNTK;
// -----------------------------------------------------------------------
// register ComputationNode with the ScriptableObject system
// -----------------------------------------------------------------------
namespace Microsoft { namespace MSR { namespace ScriptableObjects {
using namespace Microsoft::MSR::CNTK;
// -----------------------------------------------------------------------
// register ComputationNode with the ScriptableObject system
// -----------------------------------------------------------------------
class TrainAction { };
template<> shared_ptr<Object> MakeRuntimeObject<TrainAction>(const IConfigRecordPtr configp)
{
const IConfigRecord & config = *configp;
template<> shared_ptr<Object> MakeRuntimeObject<TrainAction>(const IConfigRecordPtr configp)
{
const IConfigRecord & config = *configp;
wstring precision = config[L"precision"]; // dispatch on ElemType
if (precision == L"float")
DoTrain<IConfigRecord, float>(config);
DoTrain<IConfigRecord, float>(config);
else if (precision == L"double")
DoTrain<IConfigRecord, double>(config);
DoTrain<IConfigRecord, double>(config);
else
RuntimeError("invalid value '%ls' for 'precision', must be 'float' or 'double'", precision.c_str());
return make_shared<Object>(); // return a dummy object
}
// register ComputationNode with the ScriptableObject system
ScriptableObjects::ConfigurableRuntimeTypeRegister::Add<TrainAction> registerTrainAction(L"TrainAction");
}}}
return make_shared<Object>(); // return a dummy object
}
// register ComputationNode with the ScriptableObject system
ScriptableObjects::ConfigurableRuntimeTypeRegister::Add<TrainAction> registerTrainAction(L"TrainAction");
}}}
template <typename ElemType>
void DoAdapt(const ConfigParameters& config)
@ -1373,7 +1373,7 @@ void DoConvertFromDbn(const ConfigParameters& config)
wstring dbnModelPath = config(L"dbnModelPath");
auto netBuilder = make_shared<SimpleNetworkBuilder<ElemType>>(config);
ComputationNetworkPtr net = netBuilder->BuildNetworkFromDbnFile(dbnModelPath);
ComputationNetworkPtr net = netBuilder->BuildNetworkFromDbnFile(dbnModelPath);
net->SaveToFile(modelPath);
}
@ -1643,24 +1643,36 @@ std::string TimeDateStamp()
return buf;
}
#ifdef _WIN32
void PrintBuiltInfo()
{
fprintf(stderr, "-------------------------------------------------------------------\n");
fprintf(stderr, "Build info: \n\n");
fprintf(stderr, "\t\tBuilt time: %s %s\n", __DATE__, __TIME__);
fprintf(stderr, "\t\tLast modified date: %s\n", __TIMESTAMP__);
fprintf(stderr, "\t\tBuilt by %s on %s\n", _BUILDER_, _BUILDMACHINE_);
fprintf(stderr, "\t\tBuild Path: %s\n", _BUILDPATH_);
#ifdef _BUILDTYPE_
fprintf(stderr, "\t\tBuild type: %s\n", _BUILDTYPE_);
#endif
#ifdef _MATHLIB_
fprintf(stderr, "\t\tMath lib: %s\n", _MATHLIB_);
#endif
#ifdef _CUDA_PATH_
fprintf(stderr, "\t\tCUDA_PATH: %s\n", _CUDA_PATH_);
#endif
#ifdef _CUB_PATH_
fprintf(stderr, "\t\tCUDA_PATH: %s\n", _CUB_PATH_);
#endif
#ifdef _GIT_EXIST
fprintf(stderr, "\t\tBuild Branch: %s\n", _BUILDBRANCH_);
fprintf(stderr, "\t\tBuild SHA1: %s\n", _BUILDSHA1_);
#endif
fprintf(stderr, "-------------------------------------------------------------------\n");
}
#ifdef _BUILDER_
fprintf(stderr, "\t\tBuilt by %s on %s\n", _BUILDER_, _BUILDMACHINE_);
#endif
#ifdef _BUILDPATH_
fprintf(stderr, "\t\tBuild Path: %s\n", _BUILDPATH_);
#endif
fprintf(stderr, "-------------------------------------------------------------------\n");
}
void PrintUsageInfo()
{
@ -1733,7 +1745,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra
// change working directory
if (workingDir != L"")
_wchdir(workingDir.c_str());
_wchdir(workingDir.c_str());
// compile the BrainScript
wstring bs = L"[\n";
@ -1748,7 +1760,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra
let expr = BS::ParseConfigExpression(bs, move(includePaths)); // parse
let valp = BS::Evaluate(expr); // evaluate parse into a dictionary
let & config = valp.AsRef<ScriptableObjects::IConfigRecord>(); // this is the dictionary
let & config = valp.AsRef<ScriptableObjects::IConfigRecord>(); // this is the dictionary
// legacy parameters that have changed spelling
if (config.Find(L"DoneFile")) // variables follow camel case (start with lower-case letters)
@ -1779,9 +1791,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra
}
// echo config info to log
#ifdef _WIN32
PrintBuiltInfo();
#endif
// execute the actions
//std::string type = config(L"precision", "float");
@ -1867,9 +1877,7 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // called from wmain which i
RedirectStdErr(logpath);
}
#ifdef _WIN32
PrintBuiltInfo();
#endif
std::string timestamp = TimeDateStamp();
//dump config info

Просмотреть файл

@ -180,6 +180,18 @@ ORIGINDIR:='$$ORIGIN'
CNTKMATH:=cntkmath
########################################
# Build info
########################################
BUILDINFO:= MachineLearning/CNTK/buildinfo.h
$(BUILDINFO): Scripts/genrate_build_info
@echo creating $@ for $(ARCH) with build type $(BUILDTYPE)
@Scripts/genrate_build_info
########################################
# Math library
########################################
@ -451,7 +463,7 @@ CNTK_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJDIR)/%.o, $(C
CNTK:=$(BINDIR)/cntk
ALL+=$(CNTK)
$(CNTK): $(CNTK_OBJ) | $(CNTKMATH_LIB)
$(CNTK): $(BUILDINFO) $(CNTK_OBJ) | $(CNTKMATH_LIB)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@echo building output for $(ARCH) with build type $(BUILDTYPE)
@ -485,7 +497,10 @@ $(OBJDIR)/%.o : %.cpp Makefile
@mkdir -p $(dir $@)
$(CXX) -c $< -o $@ $(CPPFLAGS) $(CXXFLAGS) $(INCLUDEPATH:%=-I%) -MD -MP -MF ${@:.o=.d}
.PHONY: clean buildall all
.PHONY: force clean buildall all
force: $(BUILDINFO)
clean:
@echo $(SEPARATOR)

104
Scripts/genrate_build_info Executable file
Просмотреть файл

@ -0,0 +1,104 @@
#!/bin/bash
# description:
# this script is used to generated buildinfo.h in MachineLearning/CNTK which will contain the following infomation to be displayed at runtime:
# CUDA_PATH (if exists)
# CUB_PATH (if exists)
# GIT_COMMIT
# GTT_BRANCH
# BUILDTYPE (release/debug)
# MATHLIB (MKL/ACML)
usage ()
{
echo "usage: $0 "
echo "-------------------------------------------------------------------"
echo "This script is used to generate buildinfo.h in MachineLearning/CNTK"
echo "This script needs to be called from the top level directory of CNTK project"
echo "This script assumes git can be used"
echo "This script assumes Config.make has been made"
echo "-------------------------------------------------------------------"
if [ ! -z "$1" ] ; then
echo "ERROR message: $1"
fi
exit 1
}
Has_Git()
{
if hash git 2>/dev/null; then
return 0
else
return 1
fi
}
makebuildinfo()
{
target=$1
BUILDTYPE=$2
MATHLIB=$3
GIT_COMMIT=$4
GIT_BRANCH=$5
CUDA_PATH=$6
CUB_PATH=$7
printf "#ifndef _BUILDINFO_H\n" > $target
printf "#define _BUILDINFO_H\n" >> $target
printf "#define _GIT_EXIST\n" >> $target
printf "#define _MATHLIB_ \"%s\"\n" $MATHLIB >> $target
printf "#define _BUILDSHA1_ \"%s\"\n" $GIT_COMMIT >> $target
printf "#define _BUILDBRANCH_ \"%s\"\n" $GIT_BRANCH >> $target
if [ ! -z "$CUDA_PATH" ]; then
printf "#define _CUDA_PATH_ \"%s\"\n" $CUDA_PATH >> $target
fi
if [ ! -z "$CUB_PATH" ]; then
printf "#define _CUB_PATH_ \"%s\"\n" $CUB_PATH >> $target
fi
printf "#define _BUILDTYPE_ \"%s\"\n" $BUILDTYPE >> $target
printf "#endif\n" >> $target
}
#//////////////////////////////////////////////////////#
# main function #
#//////////////////////////////////////////////////////#
if [ $# -ne 0 ]; then
usage
fi
# 1. check whether we have git and what is the sha-1 value
if Has_Git; then has_git=1; else has_git=0; usage "git not exist"; fi
GIT_COMMIT=`git rev-parse HEAD`
GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
# 2. looking into Config.make
if [ ! -e Config.make ] ; then
usage "Config.make not exists"
fi
source "Config.make"
# 3. whether we have CUDA_PATH
if [ -z "${CUDA_PATH+x}" ]; then
CUDAPATH=""
else
CUDAPATH=$CUDA_PATH
fi
# 4. whether we have CUB_PATH
if [ -z "${CUB_PATH+x}" ]; then
CUBPATH=""
else
CUBPATH=$CUB_PATH
fi
# 5. make buildinfo.h
target=MachineLearning/CNTK/buildinfo.h
if [ ! -d MachineLearning ] ; then
usage
fi
if [ -e MachineLearning/CNTK/buildinfo.h ] ; then
rm MachineLearning/CNTK/buildinfo.h
fi
makebuildinfo $target $BUILDTYPE $MATHLIB $GIT_COMMIT $GIT_BRANCH $CUDAPATH $CUBPATH