diff --git a/Source/ComputationNetworkLib/EvaluationNodes.h b/Source/ComputationNetworkLib/EvaluationNodes.h
index 1db940673..7c4b4c4dc 100644
--- a/Source/ComputationNetworkLib/EvaluationNodes.h
+++ b/Source/ComputationNetworkLib/EvaluationNodes.h
@@ -62,6 +62,7 @@ public:
         Value().AssignNumOfDiff(*m_maxIndexes0, *m_maxIndexes1, m_topK > 1);
 //#if NANCHECK
         Value().HasNan("ClassificationError");
+        fprintf(stderr, "Check Nan values");
 //#endif
 #if DUMPOUTPUT
         Value().Print("ClassificationErrorNode");
diff --git a/Source/Math/Matrix.cpp b/Source/Math/Matrix.cpp
index 7a26dcc37..171aa79c6 100755
--- a/Source/Math/Matrix.cpp
+++ b/Source/Math/Matrix.cpp
@@ -5844,15 +5844,15 @@ bool Matrix<ElemType>::HasNan(const char* name) const
 
     if (IsEmpty())
         return false;
-
+    fprintf(stderr, "Check Nan values");
     // if GPU then first detect NaN there, will be faster
     if (GetDeviceId() != CPUDEVICE)
     {
         Matrix<ElemType> sum(GetDeviceId());
         sum.AssignSumOfElements(*this);
         auto x = sum.Get00Element();
-        if (!std::isnan(x))
-            return false;
+        if (std::isnan(x))
+            LogicError("Nan found");
     }
 
     // const auto & us = *this;
@@ -5862,6 +5862,7 @@ bool Matrix<ElemType>::HasNan(const char* name) const
         if (std::isnan(us(i, j)))
         {
             fprintf(stderr, "HasNan: NaN detected at %s (%ld,%ld) in (%d,%d) matrix\n", name, i, j, (int) GetNumRows(), (int) GetNumCols());
+            LogicError("Nan found");
             return true;
         }
     return false;