Fix WCOS/Win32 linking bugs (#3126)

* Fix WCOS/Win32 linking bugs * Remove unused NODEFAULTLIB flags * Avoid plain target_link_libraries signature * Avoid plain target_link_libraries signature * Fix library list escaping * Use library list instead of string * Remove duplicate link to windowsapp.lib * Remove Win32 build workarounds * Specify CMake policies before initializing language * Expose Win32 header definitions during build * Force set API family * Enable Win32 APIs in featurizer * Use MT dynamic CRT * Expose Win32 specific functions * Disable app container globally * Disable default wide functions in featurizers * Add featurizers to test include path * Workaround https://gitlab.kitware.com/cmake/cmake/issues/19428 * Revert pipeline debugging hacks * Skip /FI in CUDA sources * Default to Win32 builds * Enable WCOS when using WinML * Use generator expression to apply CMAKE_MSVC_RUNTIME_LIBRARY to C++ only
2020-03-19 08:52:40 -07:00 · 2020-03-19 08:52:40 -07:00 · 3bdb0b620a
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -4,16 +4,14 @@

 # Minimum CMake required
 cmake_minimum_required(VERSION 3.13)
-
-# Project
-project(onnxruntime C CXX)
 cmake_policy(SET CMP0069 NEW)
-set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
-
 if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.15")
  cmake_policy(SET CMP0092 NEW)
 endif()

+# Project
+project(onnxruntime C CXX)
+
 include(CheckCXXCompilerFlag)
 include(CheckLanguage)

@ -92,7 +90,6 @@ option(onnxruntime_USE_WINML "Build with WinML support" OFF)
 option(onnxruntime_USE_ACL "Build with ACL support" OFF)
 option(onnxruntime_ENABLE_INSTRUMENT "Enable Instrument with Event Tracing for Windows (ETW)" OFF)
 option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF)
-option(onnxruntime_ENABLE_WCOS "Build for Windows Core OS" OFF)
 #The onnxruntime_PREFER_SYSTEM_LIB is mainly designed for package managers like apt/yum/vcpkg.
 #Please note, by default Protobuf_USE_STATIC_LIBS is OFF but it's recommended to turn it ON on Windows. You should set it properly when onnxruntime_PREFER_SYSTEM_LIB is ON otherwise you'll hit linkage errors.
 #If you have already installed protobuf(or the others) in your system at the default system paths(like /usr/include), then it's better to set onnxruntime_PREFER_SYSTEM_LIB ON. Otherwise onnxruntime may see two different protobuf versions and we won't know which one will be used, the worst case could be onnxruntime picked up header files from one of them but the binaries from the other one.
@ -122,6 +119,11 @@ else()
    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /d2FH4-")
    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /d2FH4-")
  endif()
+  if(WINDOWS_STORE)
+    # cmake/external/protobuf/src/google/protobuf/compiler/subprocess.cc and onnxruntime/core/platform/windows/env.cc call a bunch of Win32 APIs.
+    # For now, we'll set the API family to desktop globally to expose Win32 symbols in headers; this must be fixed!
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:/FI${CMAKE_CURRENT_SOURCE_DIR}/set_winapi_family_desktop.h>")
+  endif()
 endif()

 if(onnxruntime_USE_OPENMP)
@ -875,4 +877,3 @@ if (onnxruntime_BUILD_CSHARP)
 #  set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0")
  include(onnxruntime_csharp.cmake)
 endif()
-
--- a/cmake/CMakeSettings.json
+++ b/cmake/CMakeSettings.json
@ -0,0 +1,62 @@
+{
+  "configurations": [
+    {
+      "name": "WinML-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\..\\build\\${name}",
+      "installRoot": "${projectDir}\\..\\build\\${name}\\install",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "onnxruntime_USE_WINML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_USE_DML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_BUILD_SHARED_LIB",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ],
+      "cmakeToolchain": "${projectDir}\\wcos_toolchain.cmake"
+    },
+    {
+      "name": "WinML-RelWithDebInfo",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\..\\build\\${name}",
+      "installRoot": "${projectDir}\\..\\build\\${name}\\install",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "onnxruntime_USE_WINML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_USE_DML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_BUILD_SHARED_LIB",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ],
+      "cmakeToolchain": "${projectDir}\\wcos_toolchain.cmake"
+    }
+  ]
+}
--- a/cmake/external/featurizers.cmake
+++ b/cmake/external/featurizers.cmake
@ -16,5 +16,9 @@ if (WIN32)
    # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include.
    set_target_properties(FeaturizersCode PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props)
 endif()
+if (WINDOWS_STORE)
+    # Library requires narrow version of APIs
+    target_compile_options(FeaturizersCode PRIVATE "/U UNICODE" "/U _UNICODE")
+endif()

 add_library(onnxruntime_featurizers ALIAS FeaturizersCode)
--- a/cmake/onnxruntime.cmake
+++ b/cmake/onnxruntime.cmake
@ -26,7 +26,7 @@ add_custom_command(OUTPUT ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_s

 add_custom_target(onnxruntime_generate_def ALL DEPENDS ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c)
 if(WIN32)
-    add_library(onnxruntime SHARED 
+    add_library(onnxruntime SHARED
      ${SYMBOL_FILE}
      "${ONNXRUNTIME_ROOT}/core/dll/dllmain.cc"
      "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc"
@ -108,13 +108,6 @@ install(TARGETS onnxruntime

 set_target_properties(onnxruntime PROPERTIES FOLDER "ONNXRuntime")

-if (onnxruntime_ENABLE_WCOS)
-  if (NOT onnxruntime_BUILD_SHARED_LIB)
-      message(
-        FATAL_ERROR 
-        "Option onnxruntime_ENABLE_WCOS can only be used when onnxruntime_BUILD_SHARED_LIB is also enabled")
-  endif()
-  target_link_libraries(onnxruntime PRIVATE windowsapp.lib)
-  target_link_options(onnxruntime PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
+if (WINDOWS_STORE)
  target_link_options(onnxruntime PRIVATE /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll)
-endif()
+endif()
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@ -176,7 +176,7 @@ set (onnxruntime_shared_lib_test_SRC
          ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_model_loading.cc)
 if(onnxruntime_RUN_ONNX_TESTS)
  list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_io_types.cc)
-endif() 
+endif()

 set (onnxruntime_global_thread_pools_test_SRC
          ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h
@ -397,6 +397,9 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
  # the default logger tests conflict with the need to have an overall default logger
  # so skip in this type of
  target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS)
+  if (onnxruntime_USE_FEATURIZERS)
+    target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src)
+  endif()

  if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
    target_link_libraries(onnxruntime_test_all PRIVATE onnxruntime_language_interop onnxruntime_pyop)
@ -461,7 +464,7 @@ add_library(onnx_test_data_proto ${TEST_SRC_DIR}/proto/tml.proto)
 add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})

 if(WIN32)
-  target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267")  
+  target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267")
 else()
  if(HAS_UNUSED_PARAMETER)
    target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-parameter")
@ -469,7 +472,7 @@ else()
  if(HAS_UNUSED_VARIABLE)
    target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-variable")
  endif()
-  if(HAS_UNUSED_BUT_SET_VARIABLE)    
+  if(HAS_UNUSED_BUT_SET_VARIABLE)
    target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-but-set-variable")
  endif()
 endif()
--- a/cmake/set_winapi_family_desktop.h
+++ b/cmake/set_winapi_family_desktop.h
@ -0,0 +1,2 @@
+#undef WINAPI_FAMILY
+#define WINAPI_FAMILY WINAPI_FAMILY_DESKTOP_APP
--- a/cmake/wcos_toolchain.cmake
+++ b/cmake/wcos_toolchain.cmake
@ -0,0 +1,6 @@
+set(CMAKE_SYSTEM_NAME WindowsStore)
+set(CMAKE_SYSTEM_VERSION 10.0)
+if (NOT DEFINED CMAKE_SYSTEM_PROCESSOR)
+  set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR})
+endif()
+set(CMAKE_VS_GLOBALS "AppContainerApplication=false")
--- a/cmake/winml.cmake
+++ b/cmake/winml.cmake
@ -1,6 +1,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.

+if (NOT WINDOWS_STORE)
+  message(FATAL_ERROR "WinML is only supported on WCOS")
+endif()
+
 include(precompiled_header.cmake)
 include(winml_sdk_helpers.cmake)
 include(winml_cppwinrt.cmake)
@ -204,7 +208,7 @@ list(APPEND winml_adapter_files
    ${winml_adapter_dir}/winml_adapter_model.h
    ${winml_adapter_dir}/winml_adapter_session.cpp
    )
-	
+
 if (onnxruntime_USE_DML)
  list(APPEND winml_adapter_files
    ${winml_adapter_dir}/abi_custom_registry_impl.cpp
@ -563,15 +567,6 @@ add_dependencies(winml_dll winml_sdk_cppwinrt)
 add_dependencies(winml_dll winml_api_native)
 add_dependencies(winml_dll winml_api_native_internal)

-# Any project that links in debug_alloc.obj needs this lib.
-# unresolved external symbol __imp_SymSetOptions
-# ...                        __imp_SymGetLineFromAddr64
-# ...                        __imp_SymInitialize
-# ...                        __imp_SymFromAddr
-if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-  set(DBGHELP dbghelp.lib)
-endif("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-
 # Link libraries
 target_link_libraries(winml_dll PRIVATE onnxruntime)
 target_link_libraries(winml_dll PRIVATE re2)
@ -581,7 +576,15 @@ target_link_libraries(winml_dll PRIVATE winml_lib_image)
 target_link_libraries(winml_dll PRIVATE winml_lib_ort)
 target_link_libraries(winml_dll PRIVATE winml_lib_telemetry)
 target_link_libraries(winml_dll PRIVATE delayimp.lib)
-target_link_libraries(winml_dll PRIVATE ${DBGHELP})
+
+# Any project that links in debug_alloc.obj needs this lib.
+# unresolved external symbol __imp_SymSetOptions
+# ...                        __imp_SymGetLineFromAddr64
+# ...                        __imp_SymInitialize
+# ...                        __imp_SymFromAddr
+if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
+  target_link_libraries(winml_dll PRIVATE dbghelp.lib)
+endif()

 # 1 of 3 projects that fail in link with 'failed to do memory mapped file I/O' (Only release)
 # when using x86 hosted architecture. When using the LKG compiler this becomes a problem
@ -602,6 +605,3 @@ endif()
 # However, there are no cuda imports in winml_dll, and the linker throws the 4199 warning.
 # This is needed to allow winml_dll build with cuda enabled.
 target_link_options(winml_dll PRIVATE /ignore:4199)
-
-target_link_libraries(winml_dll PRIVATE windowsapp.lib)
-target_link_options(winml_dll PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
--- a/cmake/winml_unittests.cmake
+++ b/cmake/winml_unittests.cmake
@ -39,6 +39,7 @@ function(add_winml_test)
  add_executable(${_UT_TARGET} ${_UT_SOURCES})
  source_group(TREE ${WINML_TEST_SRC_DIR} FILES ${_UT_SOURCES})
  set_winml_target_properties(${_UT_TARGET})
+  target_compile_definitions(${_UT_TARGET} PRIVATE BUILD_GOOGLE_TEST)

  if (_UT_DEPENDS)
    add_dependencies(${_UT_TARGET} ${_UT_DEPENDS})
@ -91,11 +92,8 @@ add_winml_test(
  SOURCES ${winml_test_api_src}
  LIBS winml_test_common delayimp.lib
 )
-target_compile_definitions(winml_test_api PRIVATE BUILD_GOOGLE_TEST)
 target_precompiled_header(winml_test_api testPch.h)

-target_link_libraries(winml_test_api PRIVATE windowsapp.lib)
-target_link_options(winml_test_api PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
 target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
 if (onnxruntime_USE_DML)
  target_link_options(winml_test_api PRIVATE /DELAYLOAD:directml.dll)
@ -111,10 +109,7 @@ add_winml_test(
  LIBS winml_test_common delayimp.lib ${winml_test_scenario_libs}
 )
 target_precompiled_header(winml_test_scenario testPch.h)
-target_compile_definitions(winml_test_scenario PRIVATE BUILD_GOOGLE_TEST)

-target_link_libraries(winml_test_scenario PRIVATE windowsapp.lib)
-target_link_options(winml_test_scenario PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
 target_link_options(winml_test_scenario PRIVATE /DELAYLOAD:d2d1.dll /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
 if (onnxruntime_USE_DML)
  target_link_options(winml_test_scenario PRIVATE /DELAYLOAD:directml.dll)
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -343,7 +343,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                 "-Donnxruntime_USE_DML=" + ("ON" if args.use_dml else "OFF"),
                 "-Donnxruntime_USE_WINML=" + ("ON" if args.use_winml else "OFF"),
                 "-Donnxruntime_USE_TELEMETRY=" + ("ON" if args.use_telemetry else "OFF"),
-                 "-Donnxruntime_ENABLE_WCOS=" + ("ON" if args.enable_wcos else "OFF"),
                 "-Donnxruntime_ENABLE_LTO=" + ("ON" if args.enable_lto else "OFF"),
                 ]

@ -936,37 +935,38 @@ def main():
        cmake_extra_args = []
        path_to_protoc_exe = None
        if(is_windows()):
-          if (args.x86):
-            cmake_extra_args = ['-A','Win32','-T','host=x64','-G', args.cmake_generator]
-          elif (args.arm or args.arm64):
-            # Cross-compiling for ARM(64) architecture
-            # First build protoc for host to use during cross-compilation
-            path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)
-            if args.arm:
-                cmake_extra_args = ['-A', 'ARM']
+            if (args.x86):
+                cmake_extra_args = ['-A','Win32','-T','host=x64','-G', args.cmake_generator]
+            elif (args.arm or args.arm64):
+                # Cross-compiling for ARM(64) architecture
+                # First build protoc for host to use during cross-compilation
+                path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)
+                if args.arm:
+                    cmake_extra_args = ['-A', 'ARM']
+                else:
+                    cmake_extra_args = ['-A', 'ARM64']
+                cmake_extra_args += ['-G', args.cmake_generator]
+                # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
+                if args.test:
+                    log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
+                    args.test = False
            else:
-                cmake_extra_args = ['-A', 'ARM64']
-            cmake_extra_args += ['-G', args.cmake_generator]
-            # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
-            if args.test:
-                log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
-                args.test = False
-          else:
-            if args.msvc_toolset == '14.16' and args.cmake_generator == 'Visual Studio 16 2019':
-                #CUDA 10.0 requires _MSC_VER >= 1700 and _MSC_VER < 1920, aka Visual Studio version in [2012, 2019)
-                #In VS2019, we have to use Side-by-side minor version MSVC toolsets from Visual Studio 2017
-                #14.16 is MSVC version
-                #141 is MSVC Toolset Version
-                #Cuda VS extension should be installed to C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations
-                toolset = 'v141,host=x64,version=' + args.msvc_toolset
-            elif args.msvc_toolset:
-                toolset = 'host=x64,version=' + args.msvc_toolset
-            else:
-                toolset = 'host=x64'
-            if (args.cuda_version):
-                toolset += ',cuda=' + args.cuda_version
-
-            cmake_extra_args = ['-A','x64','-T', toolset, '-G', args.cmake_generator]
+                if args.msvc_toolset == '14.16' and args.cmake_generator == 'Visual Studio 16 2019':
+                    #CUDA 10.0 requires _MSC_VER >= 1700 and _MSC_VER < 1920, aka Visual Studio version in [2012, 2019)
+                    #In VS2019, we have to use Side-by-side minor version MSVC toolsets from Visual Studio 2017
+                    #14.16 is MSVC version
+                    #141 is MSVC Toolset Version
+                    #Cuda VS extension should be installed to C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations
+                    toolset = 'v141,host=x64,version=' + args.msvc_toolset
+                elif args.msvc_toolset:
+                    toolset = 'host=x64,version=' + args.msvc_toolset
+                else:
+                    toolset = 'host=x64'
+                if (args.cuda_version):
+                    toolset += ',cuda=' + args.cuda_version
+                cmake_extra_args = ['-A','x64','-T', toolset, '-G', args.cmake_generator]
+            if args.enable_wcos:
+                cmake_extra_args.append('-DCMAKE_TOOLCHAIN_FILE=' + os.path.join(source_dir, 'cmake', 'wcos_toolchain.cmake'))
        if args.android:
            # Cross-compiling for Android
            path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml
@ -3,13 +3,13 @@ parameters:
  DoCompliance: 'false'
  PackageName: 'Microsoft.ML.ORTFeaturizers'

-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
  parameters:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-nuget'
    JobName: 'Windows_CI_Dev'
-    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'x64'
@ -30,7 +30,7 @@ jobs:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-win-x86-zip'
    JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --x86 --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x86'
    EnvSetupScript: 'setup_env_x86.bat'
    sln_platform: 'Win32'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
@ -3,13 +3,13 @@ parameters:
  PackageName: 'Microsoft.ML.OnnxRuntime'
  DoCompliance: 'false'

-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
  parameters:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-nuget'
    JobName: 'Windows_CI_Dev'
-    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'x64'
@ -29,7 +29,7 @@ jobs:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-win-x86-zip'
    JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x86'
    EnvSetupScript: 'setup_env_x86.bat'
    sln_platform: 'Win32'
@ -50,7 +50,7 @@ jobs:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-win-arm64-zip'
    JobName: 'Windows_CI_Dev_arm64'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'arm64'
@ -71,7 +71,7 @@ jobs:
    AgentPool : 'Win-CPU-2019'
    ArtifactName: 'drop-win-arm-zip'
    JobName: 'Windows_CI_Dev_arm'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
    BuildArch: 'x64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'arm'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
@ -2,21 +2,21 @@ parameters:
  DoEsrp: 'false'
  PackageName: 'Microsoft.ML.OnnxRuntime.Gpu'

-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
  parameters:
    AgentPool : 'Win-GPU-2019'
-    ArtifactName: 'drop-nuget'    
+    ArtifactName: 'drop-nuget'
    JobName: 'Windows_CI_GPU_CUDA_Dev'
    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda"
    BuildArch: 'x64'
    msbuildArchitecture: 'amd64'
-    EnvSetupScript: 'setup_env_cuda.bat'    
+    EnvSetupScript: 'setup_env_cuda.bat'
    sln_platform: 'x64'
    DoDebugBuild: 'false'
    DoNugetPack : 'true'
    DoCompliance: 'false'
-    DoEsrp: ${{ parameters.DoEsrp }}    
+    DoEsrp: ${{ parameters.DoEsrp }}
    CudaVersion: '10.1'
    OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
    NuPackScript: |
@ -29,17 +29,17 @@ jobs:
 - template: ../../templates/win-ci-2019.yml
  parameters:
    AgentPool : 'Win-GPU-2019'
-    ArtifactName: 'drop-nuget-dml'    
+    ArtifactName: 'drop-nuget-dml'
    JobName: 'Windows_CI_GPU_DML_Dev'
-    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
+    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
    BuildArch: 'x64'
    msbuildArchitecture: 'amd64'
-    EnvSetupScript: 'setup_env_cuda.bat'    
+    EnvSetupScript: 'setup_env_cuda.bat'
    sln_platform: 'x64'
    DoDebugBuild: 'false'
    DoNugetPack : 'true'
    DoCompliance: 'false'
-    DoEsrp: ${{ parameters.DoEsrp }}    
+    DoEsrp: ${{ parameters.DoEsrp }}
    CudaVersion: '10.0'
    OrtPackageId: 'Microsoft.ML.OnnxRuntime.DirectML'
    NuPackScript: |
@ -54,7 +54,7 @@ jobs:
    AgentPool : 'Win-GPU-2019'
    ArtifactName: 'drop-win-dml-x86-zip'
    JobName: 'Windows_CI_GPU_DML_Dev_x86'
-    BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
+    BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
    BuildArch: 'x86'
    EnvSetupScript: 'setup_env_x86.bat'
    sln_platform: 'Win32'
@ -75,7 +75,7 @@ jobs:
  workspace:
    clean: all
  pool: $(AgentPoolLinux)
-  steps:    
+  steps:
    - template: ../../templates/set-version-number-variables-step.yml
    - template: ../../templates/linux-set-variables-and-download.yml
    - task: CmdLine@2
@ -94,7 +94,7 @@ jobs:
       zip -r linux-x64.zip linux-x64
       cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory)
       mkdir $(Build.ArtifactStagingDirectory)/testdata
-       cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata 
+       cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
       ls -al $(Build.ArtifactStagingDirectory)
      displayName: 'Create Artifacts'
    - task: PublishPipelineArtifact@0
@ -117,7 +117,7 @@ jobs:
  - Windows_CI_GPU_DML_Dev_x86
  - Linux_CI_GPU_Dev
  condition: succeeded()
-  steps:  
+  steps:
  - task: DownloadPipelineArtifact@0
    displayName: 'Download Pipeline Artifact - NuGet CUDA'
    inputs:
@ -129,7 +129,7 @@ jobs:
    inputs:
      artifactName: 'drop-nuget-dml'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
-      
+
  - task: DownloadPipelineArtifact@0
    displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
    inputs:
@ -148,7 +148,7 @@ jobs:
     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
     set PATH=%CD%;%PATH%
-     SETLOCAL EnableDelayedExpansion 
+     SETLOCAL EnableDelayedExpansion
     FOR /R %%i IN (*.nupkg) do (
         set filename=%%~ni
         IF NOT "!filename:~25,7!"=="Managed" (
@ -175,7 +175,7 @@ jobs:
     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
     powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
     set PATH=%CD%;%PATH%
-     FOR /R %%i IN (*.nupkg) do ( 
+     FOR /R %%i IN (*.nupkg) do (
         rename %%~ni.nupkg %%~ni.zip
         unzip %%~ni.zip -d %%~ni
         del /Q %%~ni.zip
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml
@ -46,9 +46,9 @@ jobs:
      env:
        TELEMETRYGUID: $(TELEMETRYGUID)
    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.7'
-        addToPath: true
+      inputs: 
+        versionSpec: '3.7' 
+        addToPath: true 
        architecture: ${{ parameters.BuildArch }}

    - task: BatchScript@1
@ -59,7 +59,7 @@ jobs:
        workingFolder: '$(Build.BinariesDirectory)'

    - script: |
-       python -m pip install -q pyopenssl setuptools wheel numpy
+       python -m pip install -q pyopenssl setuptools wheel numpy     
      workingDirectory: '$(Build.BinariesDirectory)'
      displayName: 'Install python modules'

--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@ -56,7 +56,7 @@ jobs:
    displayName: 'Generate cmake config'
    inputs:
      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --build_java'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java'
      workingDirectory: '$(Build.BinariesDirectory)'

  - task: VSBuild@1
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@ -37,12 +37,12 @@ jobs:
     python -m pip install -q pyopenssl setuptools wheel numpy     
    workingDirectory: '$(Build.BinariesDirectory)'
    displayName: 'Install python modules'
-    
+
  - task: PythonScript@0
    displayName: 'Generate cmake config'
    inputs:
      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --enable_wcos --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0'
      workingDirectory: '$(Build.BinariesDirectory)'

  - task: VSBuild@1
@ -116,8 +116,8 @@ jobs:
     del wheel_filename_file
     python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
     set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH%
-     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0
-   
+     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --enable_wcos --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0
+
    workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
    displayName: 'Run tests'
 
--- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml
@ -34,17 +34,17 @@ jobs:
      workingFolder: '$(Build.BinariesDirectory)'

  - script: |
-     python -m pip install -q pyopenssl setuptools wheel numpy     
+     python -m pip install -q pyopenssl setuptools wheel numpy
    workingDirectory: '$(Build.BinariesDirectory)'
    displayName: 'Install python modules'
-    
+
  - task: PythonScript@0
    displayName: 'Generate cmake config'
    inputs:
      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos'
      workingDirectory: '$(Build.BinariesDirectory)'
-     
+
  - task: VSBuild@1
    displayName: 'Build'
    inputs:
@ -115,8 +115,8 @@ jobs:
     set /p WHEEL_FILENAME=<wheel_filename_file
     del wheel_filename_file
     python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
-     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --test --cmake_generator "Visual Studio 16 2019"  --x86 --build_wheel --enable_onnx_tests
-   
+     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --test --cmake_generator "Visual Studio 16 2019"  --x86 --build_wheel --enable_onnx_tests --enable_wcos
+
    workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
    displayName: 'Run tests'