diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index b29636598..087d2bcc4 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -131,6 +131,26 @@ namespace CNTK } } + /// + /// Enumeration type representing logging verbosity levels. + /// + enum class TraceLevel : unsigned int + { + Error = 0, + Warning = 1, + Info = 2 + }; + + /// + /// Specifies global logging verbosity level. + /// + CNTK_API void SetTraceLevel(TraceLevel value); + + /// + /// Returns current logging verbosity level. + /// + CNTK_API TraceLevel GetTraceLevel(); + /// A collection of additional information needed for the distributed trainer to aggregate the gradients struct MinibatchInfo { @@ -4786,15 +4806,6 @@ namespace CNTK /// struct MinibatchSourceConfig { - // TODO: This is general enough and be hoisted out once there are specific use-cases outside of - // configuring a MinibatchSource. - enum TraceLevel : unsigned int - { - Error = 0, - Warning = 1, - Info = 2 - }; - /// /// Creates a new minibatch source configuration, with enabled randomization and /// the randomization window set to DefaultRandomizationWindowInChunks when 'randomize' is @@ -4835,7 +4846,7 @@ namespace CNTK /// /// Output verbosity level. /// - TraceLevel traceLevel { TraceLevel::Warning }; + TraceLevel traceLevel{ GetTraceLevel() }; /// /// Truncation length in samples, non-zero value enables the truncation (only applicable for BPTT, diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h index 02c5bd92c..477ddb730 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h @@ -263,6 +263,8 @@ namespace CNTK CNTK_API void SetGPUMemoryAllocationTraceLevel(int traceLevel); + CNTK_API void SetMathLibTraceLevel(int traceLevel); + CNTK_API void ForceDeterministicAlgorithms(); CNTK_API bool ShouldForceDeterministicAlgorithms(); diff --git a/Source/CNTKv2LibraryDll/Common.cpp b/Source/CNTKv2LibraryDll/Common.cpp index 36cacc8dc..75e5c1bef 100644 --- a/Source/CNTKv2LibraryDll/Common.cpp +++ b/Source/CNTKv2LibraryDll/Common.cpp @@ -426,6 +426,11 @@ namespace CNTK Microsoft::MSR::CNTK::TracingGPUMemoryAllocator::SetTraceLevel(traceLevel); } + void SetMathLibTraceLevel(int traceLevel) + { + Microsoft::MSR::CNTK::SetMathLibTraceLevel(traceLevel); + } + void ForceDeterministicAlgorithms() { Microsoft::MSR::CNTK::Globals::ForceDeterministicAlgorithms(); @@ -458,6 +463,36 @@ namespace CNTK } } + std::atomic s_traceLevel(TraceLevel::Warning); + void SetTraceLevel(TraceLevel value) + { + using namespace Internal; + + auto previousValue = s_traceLevel.exchange(value); + + if (previousValue == value) + return; + + if (value == TraceLevel::Info) + { + // V1 does not have an intermediate trace level, + // the logging is either disabled (trace level = 0) + // or enabled (trace level != 0); + SetComputationNetworkTraceLevel(int(value)); + SetMathLibTraceLevel(int(value)); + } + else if (previousValue == TraceLevel::Info) + { + SetComputationNetworkTraceLevel(0); + SetMathLibTraceLevel(0); + } + } + + TraceLevel GetTraceLevel() + { + return s_traceLevel.load(); + } + /*static*/ const NDShape NDShape::Unknown(1, SentinelDimValueForUnknownShape); /*static*/ std::mutex DeviceDescriptor::s_mutex; diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index 8d47401a9..e0979073e 100755 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -311,7 +311,8 @@ namespace CNTK } else { - if (Internal::GetComputationNetworkTraceLevel() > 0) { + if (GetTraceLevel() >= TraceLevel::Warning) + { // TODO: all logging functionality should be refactored to live in a logging utility class. fprintf(stderr, "WARNING: no state information found for the stateful function (%ls) " "when deserializing from a dictionary (version=%zu). " diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index b1dad538b..a282fef08 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -960,7 +960,7 @@ namespace CNTK if (version < 4 && op == PrimitiveOpType::BatchNormalization) { - if (Internal::GetComputationNetworkTraceLevel() > 0) + if (GetTraceLevel() >= TraceLevel::Warning) { // TODO: all logging functionality should be refactored to live in a logging utility class. fprintf(stderr, "WARNING: the dictionary (version=%zu) does not contain a required " diff --git a/Source/Common/BestGpu.cpp b/Source/Common/BestGpu.cpp index 8511ceadf..222bc00e4 100644 --- a/Source/Common/BestGpu.cpp +++ b/Source/Common/BestGpu.cpp @@ -352,7 +352,12 @@ void BestGpu::Init() // get the count of objects cudaError_t err = cudaGetDeviceCount(&m_deviceCount); if (err != cudaSuccess) + { + if (GetMathLibTraceLevel() > 0) + fprintf(stderr, "BestGpu::Init() cudaGetDeviceCount failed with the error code %d.\n", (int)err); + m_deviceCount = 0; // if this fails, we have no GPUs + } ProcessorData pdEmpty = {0}; for (int i = 0; i < m_deviceCount; i++) diff --git a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj index 365bbf211..235d1ffde 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj +++ b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj @@ -60,6 +60,7 @@ + diff --git a/bindings/python/cntk/io/__init__.py b/bindings/python/cntk/io/__init__.py index 38f2ba5e8..5056db7ff 100644 --- a/bindings/python/cntk/io/__init__.py +++ b/bindings/python/cntk/io/__init__.py @@ -9,7 +9,7 @@ from .. import cntk_py, Value from ..tensor import ArrayMixin from cntk.internal import typemap from cntk.device import use_default_device -from enum import Enum, unique +from cntk.logging import TraceLevel, get_trace_level import numpy as np import uuid @@ -112,26 +112,11 @@ class MinibatchData(cntk_py.MinibatchData, ArrayMixin): def __len__(self): return self.num_sequences -@unique -class TraceLevel(Enum): - - Error = cntk_py.MinibatchSourceConfig.Error - Warning = cntk_py.MinibatchSourceConfig.Warning - Info = cntk_py.MinibatchSourceConfig.Info - - def __eq__(self, other): - if isinstance(other, TraceLevel): - return self.value == other.value - return self.value == other - - def __ne__(self, other): - return not (self == other) - class MinibatchSource(cntk_py.MinibatchSource): ''' MinibatchSource(deserializers, max_samples=cntk.io.INFINITELY_REPEAT, max_sweeps=cntk.io.INFINITELY_REPEAT, randomization_window_in_chunks=cntk.io.DEFAULT_RANDOMIZATION_WINDOW, randomization_window_in_samples=0, - trace_level=cntk.io.TraceLevel.Warning, multithreaded_deserializer=False, frame_mode=False, + trace_level=cntk.logging.get_trace_level(), multithreaded_deserializer=False, frame_mode=False, truncation_length=0, randomize=None, randomization_window=None, sample_based_randomization_window=None, epoch_size=None) @@ -156,8 +141,8 @@ class MinibatchSource(cntk_py.MinibatchSource): non-zero value enables randomization. `randomization_window_in_chunks` and `randomization_window_in_samples` are mutually exclusive, an exception will be raised if both have non-zero values. - trace_level (an instance of :class:`cntk.io.TraceLevel`, defaults to `TraceLevel.Warning`): - the output verbosity level. + trace_level (an instance of :class:`cntk.logging.TraceLevel`): the output verbosity level, defaults to + the current logging verbosity level given by :func:`~cntk.logging.get_trace_level`. multithreaded_deserializer (`bool`, defaults to `False`): specifies if the deserialization should be done on a single or multiple threads. frame_mode (`bool`, defaults to `False`): switches the frame mode on and off. If the frame mode @@ -167,7 +152,6 @@ class MinibatchSource(cntk_py.MinibatchSource): truncation_length (`int`, defaults to `0`): truncation length in samples, non-zero value enables the truncation (only applicable for BPTT, cannot be used in frame mode, an exception will be raised if frame mode is enabled and the truncation length is non-zero). - randomize (`bool`, defaults to `None`): !DEPRECATED! please use randomization_window_in_chunks or randomization_window_in_samples instead randomization_window (int, defaults to `None`): !DEPRECATED! please use randomization_window_in_chunks or diff --git a/bindings/python/cntk/logging/__init__.py b/bindings/python/cntk/logging/__init__.py index c288e41fe..1c02d8f15 100644 --- a/bindings/python/cntk/logging/__init__.py +++ b/bindings/python/cntk/logging/__init__.py @@ -4,6 +4,46 @@ # for full license information. # ============================================================================== - +from cntk import cntk_py from .progress_print import * from .graph import * +from enum import Enum, unique + +@unique +class TraceLevel(Enum): + ''' + Describes different logging verbosity levels. + ''' + + Error = cntk_py.TraceLevel_Error + Warning = cntk_py.TraceLevel_Warning + Info = cntk_py.TraceLevel_Info + + def __eq__(self, other): + if isinstance(other, TraceLevel): + return self.value == other.value + return self.value == other + + def __ne__(self, other): + return not (self == other) + +def set_trace_level(value): + ''' + Specifies global logging verbosity level. + + Args: + value (:class:`~cntk.logging.TraceLevel`): required verbosity level. + ''' + if isinstance(value, TraceLevel): + cntk_py.set_trace_level(value.value) + else: + cntk_py.set_trace_level(value) + +def get_trace_level(): + ''' + Returns current logging verbosity level. + + Returns: + :class:`~cntk.logging.TraceLevel`: current verbosity level. + ''' + return cntk_py.get_trace_level() \ No newline at end of file diff --git a/bindings/python/cntk/logging/graph.py b/bindings/python/cntk/logging/graph.py index 37a141aa3..84cc36d19 100644 --- a/bindings/python/cntk/logging/graph.py +++ b/bindings/python/cntk/logging/graph.py @@ -5,7 +5,7 @@ # ============================================================================== import os -from cntk.ops import Variable +from cntk.variables import Variable def depth_first_search(root, visitor, depth=0): diff --git a/bindings/python/cntk/tests/misc_test.py b/bindings/python/cntk/tests/misc_test.py index 80a27d817..049dff7db 100644 --- a/bindings/python/cntk/tests/misc_test.py +++ b/bindings/python/cntk/tests/misc_test.py @@ -120,4 +120,18 @@ def test_set_excluded_devices(): set_excluded_devices([cpu()]) assert not try_set_default_device(cpu(), False) set_excluded_devices([]) - assert try_set_default_device(cpu(), False) \ No newline at end of file + assert try_set_default_device(cpu(), False) + +def test_setting_trace_level(): + from cntk.logging import TraceLevel, set_trace_level, get_trace_level + + value = get_trace_level(); + assert value == TraceLevel.Warning + + for level in [TraceLevel.Info, TraceLevel.Error, TraceLevel.Warning]: + set_trace_level(level) + value = get_trace_level(); + assert value == level + set_trace_level(level.value) + value = get_trace_level(); + assert value == level \ No newline at end of file