Merge branch 'master' into sptiwari/onnx1.2_merge

Source/Common/ExceptionWithCallStack.cpp

@@ -156,6 +156,20 @@ static void CollectCallStack(size_t skipLevels, bool makeFunctionNamesStandOut,
 
 #ifdef _WIN32
 #ifndef CNTK_UWP
+    // Callstack generation on Windows is occasionally failing; disabling this functionality for now. Additional info:
+    //
+    // <extracted from email>
+    // NetworkTests.exe is hung creating a call stack for an expected exception thrown in the test code. A bit of googling suggests that the 
+    // problem could be due to an incompatibility between `programs linked with fastlink pdbs and older versions of DbgHelp.dll <https://developercommunity.visualstudio.com/content/problem/38625/debug-help-library-hangs-up.html>`_.
+    // On this machine it looks like we are using DbgHelp.dll version 10.0.14321.1024.
+    //
+    // TODO: WE SHOULD REMOVE THIS HACK ASAP.
+    //
+    char* const flagValue(std::getenv("CNTK_CI_NO_STACKTRACE"));
+
+    if (flagValue != nullptr && _strcmpi(flagValue, "1") == 0)
+        return write("Stack trace generation has been explicitly disabled\n");
+    // End hack
 
     // RtlCaptureStackBackTrace() is a kernel API without default binding, we must manually determine its function pointer.
     typedef USHORT(WINAPI * CaptureStackBackTraceType)(__in ULONG, __in ULONG, __out PVOID*, __out_opt PULONG);

Source/Multiverso

@@ -1 +1 @@
-Subproject commit 584eafa164f3f70900d6cd2304413e70ded31efc
+Subproject commit d9905f820b3e455ca7fdf9cd2c0972083f2bbf55

Source/Readers/ReaderLib/LTNoRandomizer.cpp

@@ -19,8 +19,14 @@ LTNoRandomizer::LTNoRandomizer(DataDeserializerPtr deserializer, bool multithrea
 {
 }
 
+LTNoRandomizer::~LTNoRandomizer()
+{
+    StopPrefetch();
+}
+
 void LTNoRandomizer::Prefetch() const
 {
+
     auto chunkId = m_originalChunkDescriptions[m_currentChunkPosition].m_id;
     m_prefetchedChunk.m_info = m_originalChunkDescriptions[m_currentChunkPosition];
     m_prefetchedChunk.m_data = m_deserializer->GetChunk(chunkId);

Source/Readers/ReaderLib/LTNoRandomizer.h

@@ -8,7 +8,8 @@
 #include <vector>
 #include "LocalTimelineRandomizerBase.h"
 
-namespace CNTK {
+namespace CNTK
+{
 
 // LT - LocalTimeline
 // A randomizer that does not randomize input (identity function over the original timeline).
@@ -22,6 +23,8 @@ public:
         bool multithreadedGetNextSequences = false,
         size_t maxNumberOfInvalidSequences = 0); // per worker
 
+    virtual ~LTNoRandomizer();
+
     std::map<std::wstring, size_t> GetInnerState() override;
     void SetInnerState(const std::map<std::wstring, size_t>& state) override;
 
@@ -46,5 +49,4 @@ private:
 
     mutable PrefetchedChunk m_prefetchedChunk;
 };
-
 }

Source/Readers/ReaderLib/LocalTimelineRandomizerBase.h

@@ -91,12 +91,17 @@ protected:
         return it->second;
     }
 
-    ~LocalTimelineRandomizerBase()
+    void StopPrefetch()
     {
         if (m_prefetch.valid())
             m_prefetch.wait_for(std::chrono::seconds(60));
     }
 
+    ~LocalTimelineRandomizerBase()
+    {
+        StopPrefetch();
+    }
+
     const static SequenceInfo s_endOfSweep; // Marker indicating end of the sweep.
 
     // Original chunk descriptions.

Tests/EndToEndTests/CNTKv2Python/ModuleTests/run-test

@@ -10,7 +10,7 @@ python -c "import pytest; print('PyTest: %s'%pytest.__version__)"
 # TODO why doesn't "py.test --pyargs cntk" work?
 MODULE_DIR="$(python -c "import cntk, os, sys; sys.stdout.write(os.path.dirname(os.path.abspath(cntk.__file__)))")"
 [ $? -eq 0 ] || exit $?
-py.test "$MODULE_DIR" --verbose --deviceid $TEST_DEVICE --doctest-modules
+py.test "$MODULE_DIR" --verbose --deviceid $TEST_DEVICE 
 
 if [ "$?" -eq "0" ]; then
   echo "__COMPLETED__"

Tests/EndToEndTests/UnitTests/V2LibraryCSTests/testcases.yml

@@ -1,10 +1,12 @@
 dataDir: .
 
 tags:
-  # CPU-only for now.
-  - bvt-i (build_sku == 'cpu') and (os == 'windows')
-  - nightly-i (build_sku == 'cpu') and (os == 'windows')
-  - weekly-i (build_sku == 'cpu') and (os == 'windows')
+  # Temporarily disabling tests for weekend build
+  # 
+  # # CPU-only for now.
+  # - bvt-i (build_sku == 'cpu') and (os == 'windows')
+  # - nightly-i (build_sku == 'cpu') and (os == 'windows')
+  # - weekly-i (build_sku == 'cpu') and (os == 'windows')
 
 testCases:
   Each test passes:

Tools/docker/CNTK-CPUOnly-Image/Dockerfile

@@ -41,10 +41,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         # For Java Bindings
         openjdk-8-jdk \
         # For SWIG
-        libpcre3-dev && \
-		# .NET Core SDK
-		apt-transport-https && \
-    rm -rf /var/lib/apt/lists/*
+        libpcre3-dev \
+        # .NET Core SDK
+        apt-transport-https && \
+        # Cleanup
+        rm -rf /var/lib/apt/lists/*
 
 RUN OPENMPI_VERSION=1.10.3 && \
     wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
@@ -107,7 +108,7 @@ RUN PROTOBUF_VERSION=3.1.0 \
 ARG MKLDNN_VERSION=0.14
 ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
 RUN mkdir /usr/local/mklml && \
-    wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION.tgz} | \
+    wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
     tar -xzf - -C /usr/local/mklml && \
     MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
     wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
@@ -160,9 +161,9 @@ RUN cd /root && \
     make -j $(nproc) && \
     make install
 
-COPY ./Patches /tmp/patches
-RUN /tmp/patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
-	rm -rfd /tmp/patches
+COPY Patches /tmp
+RUN /tmp/Patches/patch_swig.sh /usr/local/share/swig/3.0.10 && \
+	rm -rfd /tmp/Patches
 	
 # .NET Core SDK
 RUN cd /tmp && \

Tools/docker/CNTK-GPU-Image/Dockerfile

@@ -44,10 +44,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         # For Java Bindings
         openjdk-8-jdk \
         # For SWIG
-        libpcre3-dev && \
-		# .NET Core SDK
-		apt-transport-https && \
-    rm -rf /var/lib/apt/lists/*
+        libpcre3-dev \
+        # .NET Core SDK
+        apt-transport-https && \
+        # Cleanup
+        rm -rf /var/lib/apt/lists/*
 
 RUN OPENMPI_VERSION=1.10.3 && \
     wget -q -O - https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-${OPENMPI_VERSION}.tar.gz | tar -xzf - && \
@@ -112,7 +113,7 @@ RUN PROTOBUF_VERSION=3.1.0 \
 ARG MKLDNN_VERSION=0.14
 ARG MKLDNN_LONG_VERSION=mklml_lnx_2018.0.3.20180406
 RUN mkdir /usr/local/mklml && \
-    wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION.tgz} | \
+    wget --no-verbose -O - https://github.com/01org/mkl-dnn/releases/download/v${MKLDNN_VERSION}/${MKLDNN_LONG_VERSION}.tgz | \
     tar -xzf - -C /usr/local/mklml && \
     MKLDNN_STRING=mkl-dnn-${MKLDNN_VERSION} && \
     wget --no-verbose -O - https://github.com/01org/mkl-dnn/archive/v${MKLDNN_VERSION}.tar.gz | tar -xzf - && \
@@ -201,7 +202,7 @@ RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
     ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \
     mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \
     ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
-    ln -s /usr/local/cudnn{-$_CUDNN_VERSION,} && \
+    ln -s /usr/local/cudnn-$_CUDNN_VERSION && \
     mkdir -p /usr/src/gdk/nvml/lib && \
     cp -av /usr/local/cuda/lib64/stubs/libnvidia-ml* /usr/src/gdk/nvml/lib && \
     cp -av /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/src/gdk/nvml/lib/libnvidia-ml.so.1 && \

bindings/python/cntk/contrib/crosstalkcaffe/tests/op2cntk_test.py

@@ -43,6 +43,9 @@ def _install_test_layer(op_type, parameters, weights, input_data):
         inputs_variable.append(cntk.input(input_tensor.shape))
     return layer_def, inputs_variable
 
+# TODO: It has been temporarily disabled due to a bug that causes any convolution test
+#    (../../../layers/) to fail after op2cntk_test.py:test_conv_setup() is executed ALONG WITH
+#    ../../deeprl/tests/policy_gradient_test.py:test_update_policy_and_value_function()
 API_SETUP_CONV_DATA = [
     # The test case of conv ops
     (
@@ -81,6 +84,7 @@ def test_conv_setup(op_type, parameters, weights, input_data, expected_out):
     """
     The function to test conv api setup
     """
+    pytest.skip('Temporarily skipping due to an issue with CrossTalkCaffe and Convolution')
     inputs = [np.array(item, dtype=np.float32) for item in input_data]
     outputs = [np.array(item, dtype=np.float32) for item in expected_out]
     layer_def, input_variants = _install_test_layer(op_type, parameters, weights, inputs)
@@ -173,7 +177,7 @@ API_SETUP_BN_DATA = [
 
 @pytest.mark.parametrize("op_type, parameters, weights, input_data, expected_out", API_SETUP_BN_DATA)
 def test_batch_norm_setup(op_type, parameters, weights, input_data, expected_out):
-
+    pytest.skip('Temporarily skipping due to an issue with CrossTalkCaffe and Convolution')
     """
     The function to test batch norm api setup
     """

bindings/python/cntk/core.py

@@ -197,8 +197,8 @@ class NDArrayView(cntk_py.NDArrayView):
             >>> np_sliced = sliced.asarray()
             >>> # Result is an array of shape (2, 3)
             >>> print(np_sliced)
-            [[ 10.  20.  30.]
-             [ 40.  50.  60.]]
+            [[10. 20. 30.]
+             [40. 50. 60.]]
 
         Args:
           start_offset (tuple or list): shape of the same rank as this Value
@@ -484,11 +484,9 @@ class Value(cntk_py.Value):
             >>> z = C.times(i0, np.eye(num_classes))
             >>> value = C.Value.one_hot(sparse_indices, num_classes)
             >>> z.eval({i0: value})
-            [array([[ 0.,  1.,  0.,  0.,  0.,  0.],
-                    [ 0.,  0.,  0.,  0.,  0.,  0.],
-                    [ 0.,  0.,  0.,  0.,  0.,  1.]], dtype=float32),
-             array([[ 0.,  0.,  0.,  0.,  1.,  0.]], dtype=float32)]
-            <BLANKLINE>
+            [array([[0., 1., 0., 0., 0., 0.],
+                   [0., 0., 0., 0., 0., 0.],
+                   [0., 0., 0., 0., 0., 1.]], dtype=float32), array([[0., 0., 0., 0., 1., 0.]], dtype=float32)]
             >>> num_classes = 6
             >>> sample_shape = (2, num_classes)
             >>> sparse_indices = [[1,5,3,2],[4,1]]

bindings/python/cntk/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
 python_files = *.py
 python_classes = *Test
-addopts = --doctest-modules
+#addopts = --doctest-modules
 doctest_optionflags= NORMALIZE_WHITESPACE

bindings/python/cntk/variables.py

@@ -434,8 +434,8 @@ class Constant(VariableMixin, TensorOpsMixin, cntk_py.Constant):
     Example:
          >>> c = C.Constant(1, (2,3))
          >>> c.value
-             array([[ 1.,  1.,  1.],
-                    [ 1.,  1.,  1.]], dtype=float32)
+             array([[1., 1., 1.],
+                    [1., 1., 1.]], dtype=float32)
 
     Args:
        value (`np.ndarray` or `list` or `float` or `int`): Initial value.

configure

@@ -376,12 +376,13 @@ enable_java=$(default_use_java)
 
 function default_use_csharp()
 {
-    if [[ `command -v dotnet` && `command -v swig` && `command -v cmake` ]]
-    then
-        echo yes
-    else
+# TODO: Temporarily disabling csharp for Linux
+#    if [[ `command -v dotnet` && `command -v swig` && `command -v cmake` ]]
+#    then
+#        echo yes
+#    else
         echo no
-    fi
+#    fi
 }
 enable_csharp=$(default_use_csharp)