Add V2 api to get/set trace level

* Add yet another setter for the V1 component (math lib).
  * Add global getter and setter to the V2 API. If the V1 trace level is set
  to Info (the maximum possible value), tracing is also automatically enabled
  in all V1 components (network, GPU memory allocator and math lib).
  * Print an error message if cudaGetDeviceCount fails in BestGpu::Init().

Source/CNTKv2LibraryDll/API/CNTKLibrary.h

@@ -131,6 +131,26 @@ namespace CNTK
         }
     }
 
+    ///
+    /// Enumeration type representing logging verbosity levels.
+    ///
+    enum class TraceLevel : unsigned int
+    {
+        Error = 0,
+        Warning = 1,
+        Info = 2
+    };
+
+    ///
+    /// Specifies global logging verbosity level.
+    ///
+    CNTK_API void SetTraceLevel(TraceLevel value);
+
+    ///
+    /// Returns current logging verbosity level.
+    ///
+    CNTK_API TraceLevel GetTraceLevel();
+
     /// A collection of additional information needed for the distributed trainer to aggregate the gradients
     struct MinibatchInfo
     {
@@ -4786,15 +4806,6 @@ namespace CNTK
     /// 
     struct MinibatchSourceConfig
     {
-        // TODO: This is general enough and be hoisted out once there are specific use-cases outside of
-        // configuring a MinibatchSource.
-        enum TraceLevel : unsigned int
-        {
-            Error = 0,
-            Warning = 1,
-            Info = 2
-        };
-
         ///
         /// Creates a new minibatch source configuration, with enabled randomization and
         /// the randomization window set to DefaultRandomizationWindowInChunks when 'randomize' is
@@ -4835,7 +4846,7 @@ namespace CNTK
         ///
         /// Output verbosity level.
         ///
-        TraceLevel traceLevel { TraceLevel::Warning };
+        TraceLevel traceLevel{ GetTraceLevel() };
 
         ///
         /// Truncation length in samples, non-zero value enables the truncation (only applicable for BPTT,

Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h

@@ -263,6 +263,8 @@ namespace CNTK
 
         CNTK_API void SetGPUMemoryAllocationTraceLevel(int traceLevel);
 
+        CNTK_API void SetMathLibTraceLevel(int traceLevel);
+
         CNTK_API void ForceDeterministicAlgorithms();
         CNTK_API bool ShouldForceDeterministicAlgorithms();
 

Source/CNTKv2LibraryDll/Common.cpp

@@ -426,6 +426,11 @@ namespace CNTK
             Microsoft::MSR::CNTK::TracingGPUMemoryAllocator::SetTraceLevel(traceLevel);
         }
 
+        void SetMathLibTraceLevel(int traceLevel)
+        {
+            Microsoft::MSR::CNTK::SetMathLibTraceLevel(traceLevel);
+        }
+
         void ForceDeterministicAlgorithms()
         {
             Microsoft::MSR::CNTK::Globals::ForceDeterministicAlgorithms();
@@ -458,6 +463,36 @@ namespace CNTK
         }
     }
 
+    std::atomic<TraceLevel> s_traceLevel(TraceLevel::Warning);
+    void SetTraceLevel(TraceLevel value)
+    {
+        using namespace Internal;
+
+        auto previousValue = s_traceLevel.exchange(value);
+
+        if (previousValue == value)
+            return;
+
+        if (value == TraceLevel::Info)
+        {
+            // V1 does not have an intermediate trace level,
+            // the logging is either disabled (trace level = 0)
+            // or enabled (trace level != 0);
+            SetComputationNetworkTraceLevel(int(value));
+            SetMathLibTraceLevel(int(value));
+        }
+        else if (previousValue == TraceLevel::Info)
+        {
+            SetComputationNetworkTraceLevel(0);
+            SetMathLibTraceLevel(0);
+        }
+    }
+
+    TraceLevel GetTraceLevel()
+    {
+        return s_traceLevel.load();
+    }
+
     /*static*/ const NDShape NDShape::Unknown(1, SentinelDimValueForUnknownShape);
 
     /*static*/ std::mutex DeviceDescriptor::s_mutex;

Source/CNTKv2LibraryDll/CompositeFunction.cpp

@@ -311,7 +311,8 @@ namespace CNTK
             }
             else 
             {
-                if (Internal::GetComputationNetworkTraceLevel() > 0) {
+                if (GetTraceLevel() >= TraceLevel::Warning) 
+                {
                     // TODO: all logging functionality should be refactored to live in a logging utility class. 
                     fprintf(stderr, "WARNING: no state information found for the stateful function (%ls) "
                         "when deserializing from a dictionary (version=%zu). "

Source/CNTKv2LibraryDll/PrimitiveFunction.cpp

@@ -960,7 +960,7 @@ namespace CNTK
 
         if (version < 4 && op == PrimitiveOpType::BatchNormalization)
         {
-            if (Internal::GetComputationNetworkTraceLevel() > 0)
+            if (GetTraceLevel() >= TraceLevel::Warning)
             {
                 // TODO: all logging functionality should be refactored to live in a logging utility class. 
                 fprintf(stderr, "WARNING: the dictionary (version=%zu) does not contain a required "

Source/Common/BestGpu.cpp

@@ -352,7 +352,12 @@ void BestGpu::Init()
     // get the count of objects
     cudaError_t err = cudaGetDeviceCount(&m_deviceCount);
     if (err != cudaSuccess)
+    {
+        if (GetMathLibTraceLevel() > 0)
+            fprintf(stderr, "BestGpu::Init() cudaGetDeviceCount failed with the error code %d.\n", (int)err);
+
         m_deviceCount = 0; // if this fails, we have no GPUs
+    }
 
     ProcessorData pdEmpty = {0};
     for (int i = 0; i < m_deviceCount; i++)

bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj

@@ -60,6 +60,7 @@
     <Compile Include="SwigProxyClasses\Axis.cs" />
     <Compile Include="SwigProxyClasses\AxisVector.cs" />
     <Compile Include="SwigProxyClasses\BoolVector.cs" />
+    <Compile Include="SwigProxyClasses\TraceLevel.cs" />
     <Compile Include="SwigProxyClasses\Utils.cs" />
     <Compile Include="SwigProxyClasses\UtilsPINVOKE.cs" />
     <Compile Include="SwigProxyClasses\DataType.cs" />

bindings/python/cntk/io/__init__.py

@@ -9,7 +9,7 @@ from .. import cntk_py, Value
 from ..tensor import ArrayMixin
 from cntk.internal import typemap
 from cntk.device import use_default_device
-from enum import Enum, unique
+from cntk.logging import TraceLevel, get_trace_level
 
 import numpy as np
 import uuid
@@ -112,26 +112,11 @@ class MinibatchData(cntk_py.MinibatchData, ArrayMixin):
     def __len__(self):
         return self.num_sequences
 
-@unique
-class TraceLevel(Enum):
-
-    Error = cntk_py.MinibatchSourceConfig.Error
-    Warning = cntk_py.MinibatchSourceConfig.Warning
-    Info = cntk_py.MinibatchSourceConfig.Info
-
-    def __eq__(self, other):
-        if isinstance(other, TraceLevel):
-            return self.value == other.value
-        return self.value == other
-
-    def __ne__(self, other):
-        return not (self == other)
-
 class MinibatchSource(cntk_py.MinibatchSource):
     '''
     MinibatchSource(deserializers, max_samples=cntk.io.INFINITELY_REPEAT, max_sweeps=cntk.io.INFINITELY_REPEAT,
         randomization_window_in_chunks=cntk.io.DEFAULT_RANDOMIZATION_WINDOW, randomization_window_in_samples=0, 
-        trace_level=cntk.io.TraceLevel.Warning, multithreaded_deserializer=False, frame_mode=False,
+        trace_level=cntk.logging.get_trace_level(), multithreaded_deserializer=False, frame_mode=False,
         truncation_length=0, randomize=None, randomization_window=None, sample_based_randomization_window=None,
         epoch_size=None)
 
@@ -156,8 +141,8 @@ class MinibatchSource(cntk_py.MinibatchSource):
           non-zero value enables randomization. 
           `randomization_window_in_chunks` and `randomization_window_in_samples` are mutually exclusive,
           an exception will be raised if both have non-zero values.
-        trace_level (an instance of :class:`cntk.io.TraceLevel`, defaults to `TraceLevel.Warning`): 
-          the output verbosity level.
+        trace_level (an instance of :class:`cntk.logging.TraceLevel`): the output verbosity level, defaults to 
+          the current logging verbosity level given by :func:`~cntk.logging.get_trace_level`.
         multithreaded_deserializer (`bool`, defaults to `False`): specifies if the deserialization should be 
           done on a single or multiple threads.
         frame_mode (`bool`, defaults to `False`): switches the frame mode on and off. If the frame mode 
@@ -167,7 +152,6 @@ class MinibatchSource(cntk_py.MinibatchSource):
         truncation_length (`int`, defaults to `0`): truncation length in samples, non-zero value enables 
           the truncation (only applicable for BPTT, cannot be used in frame mode, an exception will be raised 
           if frame mode is enabled and the truncation length is non-zero).
-
         randomize (`bool`, defaults to `None`): !DEPRECATED! please use randomization_window_in_chunks or
           randomization_window_in_samples instead
         randomization_window (int, defaults to `None`): !DEPRECATED! please use randomization_window_in_chunks or

bindings/python/cntk/logging/__init__.py

@@ -4,6 +4,46 @@
 # for full license information.
 # ==============================================================================
 
-
+from cntk import cntk_py
 from .progress_print import *
 from .graph import *
+from enum import Enum, unique
+
+@unique
+class TraceLevel(Enum):
+    '''
+    Describes different logging verbosity levels.
+    '''
+
+    Error = cntk_py.TraceLevel_Error
+    Warning = cntk_py.TraceLevel_Warning
+    Info = cntk_py.TraceLevel_Info
+
+    def __eq__(self, other):
+        if isinstance(other, TraceLevel):
+            return self.value == other.value
+        return self.value == other
+
+    def __ne__(self, other):
+        return not (self == other)
+
+def set_trace_level(value):
+    '''
+    Specifies global logging verbosity level.
+
+    Args:
+        value (:class:`~cntk.logging.TraceLevel`): required verbosity level.
+    '''
+    if isinstance(value, TraceLevel):
+        cntk_py.set_trace_level(value.value)
+    else:
+        cntk_py.set_trace_level(value)
+
+def get_trace_level():
+    '''
+    Returns current logging verbosity level.
+
+    Returns:
+        :class:`~cntk.logging.TraceLevel`: current verbosity level.
+    '''
+    return cntk_py.get_trace_level()

bindings/python/cntk/logging/graph.py

@@ -5,7 +5,7 @@
 # ==============================================================================
 
 import os
-from cntk.ops import Variable
+from cntk.variables import Variable
 
 
 def depth_first_search(root, visitor, depth=0):

bindings/python/cntk/tests/misc_test.py

@@ -120,4 +120,18 @@ def test_set_excluded_devices():
   set_excluded_devices([cpu()])
   assert not try_set_default_device(cpu(), False)
   set_excluded_devices([])
-  assert try_set_default_device(cpu(), False)
+  assert try_set_default_device(cpu(), False)
+
+def test_setting_trace_level():
+  from cntk.logging import TraceLevel, set_trace_level, get_trace_level
+
+  value = get_trace_level();
+  assert value == TraceLevel.Warning
+  
+  for level in [TraceLevel.Info, TraceLevel.Error, TraceLevel.Warning]:
+    set_trace_level(level)
+    value = get_trace_level();
+    assert value == level
+    set_trace_level(level.value)
+    value = get_trace_level();
+    assert value == level