Remove debug info generation option for CUDA compilation in debug flavor builds and also enable fast-math optimizations . These changes have been done to eliminate differences in GPU results for the E2E tests between debug and release flavors. Setting environment variable CNTK_CUDA_DEVICE_DEBUGINFO=1 will enable debug info generation. The baselines for all E2E tests have also been updated in accordance with this change

Makefile

@@ -151,7 +151,7 @@ ifeq ("$(BUILDTYPE)","debug")
 
   CXXFLAGS += -g
   CPPFLAGS += -D_DEBUG
-  CUFLAGS += -O0 -G -lineinfo  $(GENCODE_FLAGS)
+  CUFLAGS += -O0 -use_fast_math -lineinfo  $(GENCODE_FLAGS)
 endif
 
 ifeq ("$(BUILDTYPE)","release")
@@ -165,6 +165,10 @@ ifeq ("$(BUILDTYPE)","release")
   CUFLAGS += -O3 -use_fast_math -lineinfo $(GENCODE_FLAGS)
 endif
 
+ifdef CNTK_CUDA_DEVICE_DEBUGINFO
+  CUFLAGS += -G
+endif
+
 #######
 
 OBJDIR:= $(BUILD_TOP)/.build

Math/Math/CNTKMathCUDA.vcxproj

@@ -85,6 +85,9 @@
       <GenerateLineInfo>true</GenerateLineInfo>
       <AdditionalCompilerOptions>/WX</AdditionalCompilerOptions>
       <CodeGeneration>$(CudaCodeGen)</CodeGeneration>
+      <FastMath>true</FastMath>
+      <GPUDebugInfo>false</GPUDebugInfo>
+      <GPUDebugInfo Condition="'$(CNTK_CUDA_DEVICE_DEBUGINFO)'=='1'">true</GPUDebugInfo>
     </CudaCompile>
     <PostBuildEvent>
       <Command>xcopy /D /I /Y "$(CudaPath)\bin\cudart64_*.dll" $(OutputPath)</Command>
@@ -114,8 +117,6 @@
       <OptimizeReferences>true</OptimizeReferences>
     </Link>
     <CudaCompile>
-      <FastMath>true</FastMath>
-      <GPUDebugInfo>false</GPUDebugInfo>
       <HostDebugInfo>false</HostDebugInfo>
     </CudaCompile>
   </ItemDefinitionGroup>

Tests/Speech/DNN/DiscriminativePreTraining/testcases.yml

@@ -3,11 +3,7 @@ tags:
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
      - bvt-s  (flavor=='debug') ^ (device=='cpu')
      # running unconditionally on every Nightly job in 'S' leg
-     # TODO: Temporary disabling Release-GPU because of a known bug causing large variance between
-     # Release and Debug configurations for GPU only for this (Speech/DNN/DiscriminativePreTraining) test. 
-     # This will be re-enabled after the bug has been addressed. 
-     # DO NOT COPY this disablement for other tests!!
-     - nightly-s  (flavor!='release') or (device!='gpu')
+     - nightly-s
 
 testCases:
   CNTK Run must be completed: