Merge remote-tracking branch 'origin/master' into linux-gcc

2015-07-23 10:17:26 -04:00 · 2015-07-23 10:17:26 -04:00 · fb02f1509a
--- a/Common/DataReader.cpp
+++ b/Common/DataReader.cpp
@ -57,11 +57,11 @@ void DataReader<ElemType>::GetDataReader(const ConfigParameters& config)
        string randomizeString = config("randomize");
        if (randomizeString == "None")
        {
-            mDoRandomize = false;
+            this->mDoRandomize = false;
        }
        else if (randomizeString == "Auto")
        {
-            mDoRandomize = true;
+            this->mDoRandomize = true;
        }
    }

--- a/DataReader/Kaldi2Reader/HTKMLFReader.cpp
+++ b/DataReader/Kaldi2Reader/HTKMLFReader.cpp
@ -1771,8 +1771,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
    template<class ElemType>
    void HTKMLFReader<ElemType>::SetSentenceSegBatch(Matrix<ElemType> &sentenceBegin, vector<MinibatchPackingFlag>& minibatchPackingFlag)
    {
-        sentenceBegin.SetValue(m_sentenceBegin);
-        minibatchPackingFlag = m_minibatchPackingFlag;
+        if (!m_framemode)
+        {
+            sentenceBegin.SetValue(m_sentenceBegin);
+            minibatchPackingFlag = m_minibatchPackingFlag;
+        }
    }

    // For Kaldi2Reader, we now make the following assumptions
--- a/DataReader/Kaldi2Reader/rollingwindowsource.h
+++ b/DataReader/Kaldi2Reader/rollingwindowsource.h
@ -341,7 +341,7 @@ namespace msra { namespace dbn {
                    {
                        msra::util::attempt (5, [&]()
                        {
-                            reader.readNoAlloc (ppath, featkind, sampperiod, feat);   // whole file read as columns of feature vectors
+                            reader.readAlloc (ppath, featkind, sampperiod, feat);   // whole file read as columns of feature vectors
                        });
                        if (featdim == 0)   // first time
                            featdim = feat.rows();
--- a/DataReader/Kaldi2Reader/utterancesourcemulti.h
+++ b/DataReader/Kaldi2Reader/utterancesourcemulti.h
@ -111,7 +111,7 @@ class minibatchutterancesourcemulti : public minibatchsource
        }
        // page in data for this chunk
        // We pass in the feature info variables by ref which will be filled lazily upon first read
-        void requiredata (string & featkind, size_t & featdim, unsigned int & sampperiod, const latticesource & latticesource) const
+        void requiredata (string & featkind, size_t & featdim, unsigned int & sampperiod, const latticesource & latticesource, int verbosity=0) const
        {

            if (numutterances() == 0)
@ -142,7 +142,10 @@ class minibatchutterancesourcemulti : public minibatchsource
                        latticesource.getlattices (utteranceset[i].key(), lattices[i], uttframes.cols());
                }
                //fprintf (stderr, "\n");
-                fprintf (stderr, "requiredata: %zu utterances read\n", utteranceset.size());
+				if (verbosity)
+                {
+                    fprintf (stderr, "requiredata: %zu utterances read\n", utteranceset.size());
+                }
            }
            catch (...)
            {
@ -360,12 +363,17 @@ public:
                throw std::runtime_error("minibatchutterancesourcemulti: all feature files must have same number of utterances");

            foreach_index(i, infiles[m]){
-	      utterancedesc utterance(msra::asr::htkfeatreader::parsedpath(infiles[m][i],featuresections[m]), 0);  //mseltzer - is this foolproof for multiio? is classids always non-empty? 
+	            utterancedesc utterance(msra::asr::htkfeatreader::parsedpath(infiles[m][i],featuresections[m]), 0);  //mseltzer - is this foolproof for multiio? is classids always non-empty? 
                const size_t uttframes = utterance.numframes(); // will throw if frame bounds not given --required to be given in this mode
                // we need at least 2 frames for boundary markers to work
                if (uttframes < 2)
-                    throw std::runtime_error("minibatchutterancesource: utterances < 2 frames not supported");
-                if (uttframes > frameref::maxframesperutterance)
+                {
+                    //throw std::runtime_error("minibatchutterancesource: utterances < 2 frames not supported");
+                    fprintf(stderr, "minibatchutterancesource: skipping %d-th file (%d frames) because it less than. frames (%d) for frameref bit field: %S", i, uttframes, 2, key.c_str());
+                    uttduration[i] = 0;
+                    uttisvalid[i] = false;
+                }
+                if (uttframes > frameref::maxframesperutterance || uttframes <2)
                {
                    fprintf(stderr, "minibatchutterancesource: skipping %d-th file (%d frames) because it exceeds max. frames (%d) for frameref bit field: %S", i, uttframes, frameref::maxframesperutterance, key.c_str());
                    uttduration[i] = 0;
@ -672,7 +680,8 @@ private:
            return sweep;

        currentsweep = sweep;
-        fprintf (stderr, "lazyrandomization: re-randomizing for sweep %zu in %s mode\n", currentsweep, framemode ? "frame" : "utterance");
+		if (verbosity>0)
+            fprintf (stderr, "lazyrandomization: re-randomizing for sweep %zu in %s mode\n", currentsweep, framemode ? "frame" : "utterance");

        const size_t sweepts = sweep * _totalframes;     // first global frame index for this sweep

@ -984,6 +993,7 @@ private:
            auto & chunkdata = randomizedchunks[m][k].getchunkdata();
            if (chunkdata.isinram())
            {
+                if (verbosity)
                fprintf (stderr, "releaserandomizedchunk: paging out randomized chunk %zu (frame range [%zu..%zu]), %zu resident in RAM\n",
                     k, randomizedchunks[m][k].globalts, randomizedchunks[m][k].globalte()-1, chunksinram-1);
                chunkdata.releasedata();
@ -1029,10 +1039,11 @@ private:
            {
                auto & chunk = randomizedchunks[m][chunkindex];
                auto & chunkdata = chunk.getchunkdata();
+				if (verbosity)
                fprintf (stderr, "feature set %d: requirerandomizedchunk: paging in randomized chunk %zu (frame range [%zu..%zu]), %zu resident in RAM\n", m, chunkindex, chunk.globalts, chunk.globalte()-1, chunksinram+1);
                msra::util::attempt (5, [&]()   // (reading from network)
                {
-                    chunkdata.requiredata (featkind[m], featdim[m], sampperiod[m], this->lattices);
+                    chunkdata.requiredata (featkind[m], featdim[m], sampperiod[m], this->lattices, verbosity);
                });
            }
            chunksinram++;
@ -1228,7 +1239,8 @@ public:
            const size_t lastchunk = chunkforframepos (globalte-1);
            const size_t windowbegin = randomizedchunks[0][firstchunk].windowbegin;
            const size_t windowend = randomizedchunks[0][lastchunk].windowend;
-            fprintf (stderr, "getbatch: getting randomized frames [%zu..%zu] (%zu frames out of %zu requested) in sweep %zu; chunks [%zu..%zu] -> chunk window [%zu..%zu)\n",
+			if (verbosity)
+                fprintf (stderr, "getbatch: getting randomized frames [%zu..%zu] (%zu frames out of %zu requested) in sweep %zu; chunks [%zu..%zu] -> chunk window [%zu..%zu)\n",
                     globalts, globalte, mbframes, framesrequested, sweep, firstchunk, lastchunk, windowbegin, windowend);
            // release all data outside, and page in all data inside
            for (size_t k = 0; k < windowbegin; k++)
--- a/MachineLearning/CNTK/prebuild.bat
+++ b/MachineLearning/CNTK/prebuild.bat
@ -27,8 +27,11 @@ set a=%~dp0
 set buildpath="%a:\=\\%"
 echo #define _BUILDPATH_    %buildpath%     >> buildinfo.h$$

-set cuda_path="%CUDA_PATH:\=\\%"
-echo #define _CUDA_PATH_    %cuda_path%     >> buildinfo.h$$
+if "%cuda_path%" == "" ( 
+    echo #define _CUDA_PATH_    "NOT_DEFINED"     >> buildinfo.h$$  
+    ) else (
+        echo #define _CUDA_PATH_    "%cuda_path:\=\\%" >> buildinfo.h$$
+    )

 echo #endif >> buildinfo.h$$

--- a/Math/Math/NoGPU.cpp
+++ b/Math/Math/NoGPU.cpp
@ -235,7 +235,8 @@ namespace Microsoft {
            template<class ElemType>
            GPUMatrix<ElemType> GPUSparseMatrix<ElemType>::ColumnSliceToDense(size_t startColumn, size_t numCols) const
            {
-                return ElemType(0);
+                GPUMatrix<ElemType> a;
+                return a;
            }

            template<class ElemType> ElemType GPUSparseMatrix<ElemType>::SumOfAbsElements() const
--- a/Scripts/build-and-test
+++ b/Scripts/build-and-test
@ -5,6 +5,9 @@ BUILD=1
 RUN=1
 CLEAN_AFTER=0
 CLEAN_BEFORE=0
+RANDOM_OUTPUT=0
+FLAVORS="debug:release"
+TARGETS="CPU:GPU"

 # parsing command line arguments:
 while [[ $# > 0 ]]
@ -15,11 +18,14 @@ case $key in
    -h|--help)
    echo "Usage: build-and-test [options]"
    echo "Options:"
-    echo "  -q|--quiet-build - redirect build output to file (by default those will be in <cntk_root>.run-<operating_system>-*)"
+    echo "  -q|--quiet-build - redirect build output to file"
    echo "  -r|--run-only - elides build step, runs the binaries that have already been built"
    echo "  -b|--build-only - just build, do not run"
+    echo "  -f|--flavor <flavor1:flavor2...> - which flavor to build/run (by default $FLAVORS)"
+    echo "  -t|--target <target1:target2...> - which target to run (by default $TARGETS)"
    echo "  -cb|--clean-build - clean up the enlistment binaries before build"
-    echo "  -o|--output-directory <output_dir> - specify output directory to use"
+    echo "  -rnd|--random-output-suffix - add random suffix to output directory"
+    echo "  -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
    echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script"
    exit 1
    ;;
@ -30,6 +36,9 @@ case $key in
    BUILD=0
    RUN=1
    ;;
+    -rnd|--random-output-suffix)
+    RANDOM_OUTPUT=1
+    ;;
    -b|--build-only)
    BUILD=1
    RUN=0
@ -38,6 +47,14 @@ case $key in
    CLEAN_BEFORE=1
    BUILD=1
    ;;
+    -f|--flavors)
+    FLAVORS="$2"
+    shift # past argument
+    ;;
+    -t|--targets)
+    TARGETS="$2"
+    shift # past argument
+    ;;
    -o|--output-directory)
    OUTPUT_DIR="$2"
    shift # past argument
@ -100,7 +117,12 @@ CNTK_ROOT=`dirname $SCRIPT_DIR`

 # Setup the output directory
 if [[ $OUTPUT_DIR == "" ]]; then
-    OUTPUT_DIR="$CNTK_ROOT/.run-$BUILD_OS-$RANDOM"
+    OUTPUT_DIR="$CNTK_ROOT/.run-$BUILD_OS"
+fi
+
+# Add random number at the end of the output directory to prevent overwriting previous results
+if [[ $RANDOM_OUTPUT == 1 ]]; then
+    OUTPUT_DIR="$OUTPUT_DIR-$RANDOM"
 fi

 echo "============ Creating CNTK temp directory in $TMP_ROOT ============"
@ -129,10 +151,14 @@ if [[ $QUIET_BUILD == 1 ]]; then
    echo "============ WARNING: You have selected quiet build. All build output will be placed in ($OUTPUT_DIR) ============"
 fi

+# Initialize lists of flavors and targets
+flavorArray=(${FLAVORS//:/ })
+targetArray=(${TARGETS//:/ })
+
 # Step 2 -- Build the project debug and release, if requested
 if [[ $BUILD == 1 ]]; then
    # Step 2 -- Perform necessary builds
-    for FLAVOR in debug release
+    for FLAVOR in "${flavorArray[@]}"
    do
        # Our make is too noisy right now and it is difficult to spot
        # issues from stdout and stderr. In the quiet mode these are
@ -171,7 +197,7 @@ if [[ $RUN == 1 ]]; then

    cd $PREFIX_DIR

-    for TARGET in CPU GPU
+    for TARGET in "${targetArray[@]}"
    do
        # These sed scripts are simply toggling DeviceNumber argument in the config file
        # If it is set to Auto, it will pick GPU over CPU. At -1 CPU is selected.
@ -181,9 +207,9 @@ if [[ $RUN == 1 ]]; then
            sed -i -e 's/^DeviceNumber.*/DeviceNumber=Auto/g' $CONF_FILE || exit $?
        fi

-        for FLAVOR in debug release
+        for FLAVOR in "${flavorArray[@]}"
        do
-            if [[ FLAVOR == "debug" ]]; then
+            if [[ $FLAVOR == "debug" ]]; then
                FLAVOR_DIR="$DEBUG_DIR"
            else
                FLAVOR_DIR="$RELEASE_DIR"