Fix WCOS/Win32 linking bugs (#3126)

* Fix WCOS/Win32 linking bugs

* Remove unused NODEFAULTLIB flags

* Avoid plain target_link_libraries signature

* Avoid plain target_link_libraries signature

* Fix library list escaping

* Use library list instead of string

* Remove duplicate link to windowsapp.lib

* Remove Win32 build workarounds

* Specify CMake policies before initializing language

* Expose Win32 header definitions during build

* Force set API family

* Enable Win32 APIs in featurizer

* Use MT dynamic CRT

* Expose Win32 specific functions

* Disable app container globally

* Disable default wide functions in featurizers

* Add featurizers to test include path

* Workaround https://gitlab.kitware.com/cmake/cmake/issues/19428

* Revert pipeline debugging hacks

* Skip /FI in CUDA sources

* Default to Win32 builds

* Enable WCOS when using WinML

* Use generator expression to apply CMAKE_MSVC_RUNTIME_LIBRARY to C++ only

cmake/CMakeLists.txt

@@ -4,16 +4,14 @@
 
 # Minimum CMake required
 cmake_minimum_required(VERSION 3.13)
-
-# Project
-project(onnxruntime C CXX)
 cmake_policy(SET CMP0069 NEW)
-set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
-
 if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.15")
   cmake_policy(SET CMP0092 NEW)
 endif()
 
+# Project
+project(onnxruntime C CXX)
+
 include(CheckCXXCompilerFlag)
 include(CheckLanguage)
 
@@ -92,7 +90,6 @@ option(onnxruntime_USE_WINML "Build with WinML support" OFF)
 option(onnxruntime_USE_ACL "Build with ACL support" OFF)
 option(onnxruntime_ENABLE_INSTRUMENT "Enable Instrument with Event Tracing for Windows (ETW)" OFF)
 option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF)
-option(onnxruntime_ENABLE_WCOS "Build for Windows Core OS" OFF)
 #The onnxruntime_PREFER_SYSTEM_LIB is mainly designed for package managers like apt/yum/vcpkg.
 #Please note, by default Protobuf_USE_STATIC_LIBS is OFF but it's recommended to turn it ON on Windows. You should set it properly when onnxruntime_PREFER_SYSTEM_LIB is ON otherwise you'll hit linkage errors.
 #If you have already installed protobuf(or the others) in your system at the default system paths(like /usr/include), then it's better to set onnxruntime_PREFER_SYSTEM_LIB ON. Otherwise onnxruntime may see two different protobuf versions and we won't know which one will be used, the worst case could be onnxruntime picked up header files from one of them but the binaries from the other one.
@@ -122,6 +119,11 @@ else()
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /d2FH4-")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /d2FH4-")
   endif()
+  if(WINDOWS_STORE)
+    # cmake/external/protobuf/src/google/protobuf/compiler/subprocess.cc and onnxruntime/core/platform/windows/env.cc call a bunch of Win32 APIs.
+    # For now, we'll set the API family to desktop globally to expose Win32 symbols in headers; this must be fixed!
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:/FI${CMAKE_CURRENT_SOURCE_DIR}/set_winapi_family_desktop.h>")
+  endif()
 endif()
 
 if(onnxruntime_USE_OPENMP)
@@ -875,4 +877,3 @@ if (onnxruntime_BUILD_CSHARP)
 #  set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0")
   include(onnxruntime_csharp.cmake)
 endif()
-

cmake/CMakeSettings.json

@@ -0,0 +1,62 @@
+{
+  "configurations": [
+    {
+      "name": "WinML-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\..\\build\\${name}",
+      "installRoot": "${projectDir}\\..\\build\\${name}\\install",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "onnxruntime_USE_WINML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_USE_DML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_BUILD_SHARED_LIB",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ],
+      "cmakeToolchain": "${projectDir}\\wcos_toolchain.cmake"
+    },
+    {
+      "name": "WinML-RelWithDebInfo",
+      "generator": "Ninja",
+      "configurationType": "RelWithDebInfo",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\..\\build\\${name}",
+      "installRoot": "${projectDir}\\..\\build\\${name}\\install",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": "",
+      "variables": [
+        {
+          "name": "onnxruntime_USE_WINML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_USE_DML",
+          "value": "True",
+          "type": "BOOL"
+        },
+        {
+          "name": "onnxruntime_BUILD_SHARED_LIB",
+          "value": "True",
+          "type": "BOOL"
+        }
+      ],
+      "cmakeToolchain": "${projectDir}\\wcos_toolchain.cmake"
+    }
+  ]
+}

cmake/external/featurizers.cmake

@@ -16,5 +16,9 @@ if (WIN32)
     # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include.
     set_target_properties(FeaturizersCode PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props)
 endif()
+if (WINDOWS_STORE)
+    # Library requires narrow version of APIs
+    target_compile_options(FeaturizersCode PRIVATE "/U UNICODE" "/U _UNICODE")
+endif()
 
 add_library(onnxruntime_featurizers ALIAS FeaturizersCode)

cmake/onnxruntime.cmake

@@ -26,7 +26,7 @@ add_custom_command(OUTPUT ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_s
 
 add_custom_target(onnxruntime_generate_def ALL DEPENDS ${SYMBOL_FILE} ${CMAKE_CURRENT_BINARY_DIR}/generated_source.c)
 if(WIN32)
-    add_library(onnxruntime SHARED 
+    add_library(onnxruntime SHARED
       ${SYMBOL_FILE}
       "${ONNXRUNTIME_ROOT}/core/dll/dllmain.cc"
       "${ONNXRUNTIME_ROOT}/core/dll/onnxruntime.rc"
@@ -108,13 +108,6 @@ install(TARGETS onnxruntime
 
 set_target_properties(onnxruntime PROPERTIES FOLDER "ONNXRuntime")
 
-if (onnxruntime_ENABLE_WCOS)
+if (WINDOWS_STORE)
-  if (NOT onnxruntime_BUILD_SHARED_LIB)
-      message(
-        FATAL_ERROR 
-        "Option onnxruntime_ENABLE_WCOS can only be used when onnxruntime_BUILD_SHARED_LIB is also enabled")
-  endif()
-  target_link_libraries(onnxruntime PRIVATE windowsapp.lib)
-  target_link_options(onnxruntime PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
   target_link_options(onnxruntime PRIVATE /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll)
-endif()
+endif()

cmake/onnxruntime_unittests.cmake

@@ -176,7 +176,7 @@ set (onnxruntime_shared_lib_test_SRC
           ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_model_loading.cc)
 if(onnxruntime_RUN_ONNX_TESTS)
   list(APPEND onnxruntime_shared_lib_test_SRC ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_io_types.cc)
-endif() 
+endif()
 
 set (onnxruntime_global_thread_pools_test_SRC
           ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h
@@ -397,6 +397,9 @@ set(all_dependencies ${onnxruntime_test_providers_dependencies} )
   # the default logger tests conflict with the need to have an overall default logger
   # so skip in this type of
   target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS)
+  if (onnxruntime_USE_FEATURIZERS)
+    target_include_directories(onnxruntime_test_all PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/FeaturizersLibrary/src)
+  endif()
 
   if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS)
     target_link_libraries(onnxruntime_test_all PRIVATE onnxruntime_language_interop onnxruntime_pyop)
@@ -461,7 +464,7 @@ add_library(onnx_test_data_proto ${TEST_SRC_DIR}/proto/tml.proto)
 add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
 
 if(WIN32)
-  target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267")  
+  target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267")
 else()
   if(HAS_UNUSED_PARAMETER)
     target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-parameter")
@@ -469,7 +472,7 @@ else()
   if(HAS_UNUSED_VARIABLE)
     target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-variable")
   endif()
-  if(HAS_UNUSED_BUT_SET_VARIABLE)    
+  if(HAS_UNUSED_BUT_SET_VARIABLE)
     target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-but-set-variable")
   endif()
 endif()

cmake/set_winapi_family_desktop.h

@@ -0,0 +1,2 @@
+#undef WINAPI_FAMILY
+#define WINAPI_FAMILY WINAPI_FAMILY_DESKTOP_APP

cmake/wcos_toolchain.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_SYSTEM_NAME WindowsStore)
+set(CMAKE_SYSTEM_VERSION 10.0)
+if (NOT DEFINED CMAKE_SYSTEM_PROCESSOR)
+  set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR})
+endif()
+set(CMAKE_VS_GLOBALS "AppContainerApplication=false")

cmake/winml.cmake

@@ -1,6 +1,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+if (NOT WINDOWS_STORE)
+  message(FATAL_ERROR "WinML is only supported on WCOS")
+endif()
+
 include(precompiled_header.cmake)
 include(winml_sdk_helpers.cmake)
 include(winml_cppwinrt.cmake)
@@ -204,7 +208,7 @@ list(APPEND winml_adapter_files
     ${winml_adapter_dir}/winml_adapter_model.h
     ${winml_adapter_dir}/winml_adapter_session.cpp
     )
-	
+
 if (onnxruntime_USE_DML)
   list(APPEND winml_adapter_files
     ${winml_adapter_dir}/abi_custom_registry_impl.cpp
@@ -563,15 +567,6 @@ add_dependencies(winml_dll winml_sdk_cppwinrt)
 add_dependencies(winml_dll winml_api_native)
 add_dependencies(winml_dll winml_api_native_internal)
 
-# Any project that links in debug_alloc.obj needs this lib.
-# unresolved external symbol __imp_SymSetOptions
-# ...                        __imp_SymGetLineFromAddr64
-# ...                        __imp_SymInitialize
-# ...                        __imp_SymFromAddr
-if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-  set(DBGHELP dbghelp.lib)
-endif("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-
 # Link libraries
 target_link_libraries(winml_dll PRIVATE onnxruntime)
 target_link_libraries(winml_dll PRIVATE re2)
@@ -581,7 +576,15 @@ target_link_libraries(winml_dll PRIVATE winml_lib_image)
 target_link_libraries(winml_dll PRIVATE winml_lib_ort)
 target_link_libraries(winml_dll PRIVATE winml_lib_telemetry)
 target_link_libraries(winml_dll PRIVATE delayimp.lib)
-target_link_libraries(winml_dll PRIVATE ${DBGHELP})
+
+# Any project that links in debug_alloc.obj needs this lib.
+# unresolved external symbol __imp_SymSetOptions
+# ...                        __imp_SymGetLineFromAddr64
+# ...                        __imp_SymInitialize
+# ...                        __imp_SymFromAddr
+if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
+  target_link_libraries(winml_dll PRIVATE dbghelp.lib)
+endif()
 
 # 1 of 3 projects that fail in link with 'failed to do memory mapped file I/O' (Only release)
 # when using x86 hosted architecture. When using the LKG compiler this becomes a problem
@@ -602,6 +605,3 @@ endif()
 # However, there are no cuda imports in winml_dll, and the linker throws the 4199 warning.
 # This is needed to allow winml_dll build with cuda enabled.
 target_link_options(winml_dll PRIVATE /ignore:4199)
-
-target_link_libraries(winml_dll PRIVATE windowsapp.lib)
-target_link_options(winml_dll PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)

cmake/winml_unittests.cmake

@@ -39,6 +39,7 @@ function(add_winml_test)
   add_executable(${_UT_TARGET} ${_UT_SOURCES})
   source_group(TREE ${WINML_TEST_SRC_DIR} FILES ${_UT_SOURCES})
   set_winml_target_properties(${_UT_TARGET})
+  target_compile_definitions(${_UT_TARGET} PRIVATE BUILD_GOOGLE_TEST)
 
   if (_UT_DEPENDS)
     add_dependencies(${_UT_TARGET} ${_UT_DEPENDS})
@@ -91,11 +92,8 @@ add_winml_test(
   SOURCES ${winml_test_api_src}
   LIBS winml_test_common delayimp.lib
 )
-target_compile_definitions(winml_test_api PRIVATE BUILD_GOOGLE_TEST)
 target_precompiled_header(winml_test_api testPch.h)
 
-target_link_libraries(winml_test_api PRIVATE windowsapp.lib)
-target_link_options(winml_test_api PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
 target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
 if (onnxruntime_USE_DML)
   target_link_options(winml_test_api PRIVATE /DELAYLOAD:directml.dll)
@@ -111,10 +109,7 @@ add_winml_test(
   LIBS winml_test_common delayimp.lib ${winml_test_scenario_libs}
 )
 target_precompiled_header(winml_test_scenario testPch.h)
-target_compile_definitions(winml_test_scenario PRIVATE BUILD_GOOGLE_TEST)
 
-target_link_libraries(winml_test_scenario PRIVATE windowsapp.lib)
-target_link_options(winml_test_scenario PRIVATE /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:user32.lib /NODEFAULTLIB:gdi32.lib /NODEFAULTLIB:winspool.lib /NODEFAULTLIB:shell32.lib /NODEFAULTLIB:ole32.lib /NODEFAULTLIB:oleaut32.lib /NODEFAULTLIB:uuid.lib /NODEFAULTLIB:comdlg32.lib /NODEFAULTLIB:advapi32.lib)
 target_link_options(winml_test_scenario PRIVATE /DELAYLOAD:d2d1.dll /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
 if (onnxruntime_USE_DML)
   target_link_options(winml_test_scenario PRIVATE /DELAYLOAD:directml.dll)

tools/ci_build/build.py

@@ -343,7 +343,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
                  "-Donnxruntime_USE_DML=" + ("ON" if args.use_dml else "OFF"),
                  "-Donnxruntime_USE_WINML=" + ("ON" if args.use_winml else "OFF"),
                  "-Donnxruntime_USE_TELEMETRY=" + ("ON" if args.use_telemetry else "OFF"),
-                 "-Donnxruntime_ENABLE_WCOS=" + ("ON" if args.enable_wcos else "OFF"),
                  "-Donnxruntime_ENABLE_LTO=" + ("ON" if args.enable_lto else "OFF"),
                  ]
 
@@ -936,37 +935,38 @@ def main():
         cmake_extra_args = []
         path_to_protoc_exe = None
         if(is_windows()):
-          if (args.x86):
+            if (args.x86):
-            cmake_extra_args = ['-A','Win32','-T','host=x64','-G', args.cmake_generator]
+                cmake_extra_args = ['-A','Win32','-T','host=x64','-G', args.cmake_generator]
-          elif (args.arm or args.arm64):
+            elif (args.arm or args.arm64):
-            # Cross-compiling for ARM(64) architecture
+                # Cross-compiling for ARM(64) architecture
-            # First build protoc for host to use during cross-compilation
+                # First build protoc for host to use during cross-compilation
-            path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)
+                path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)
-            if args.arm:
+                if args.arm:
-                cmake_extra_args = ['-A', 'ARM']
+                    cmake_extra_args = ['-A', 'ARM']
+                else:
+                    cmake_extra_args = ['-A', 'ARM64']
+                cmake_extra_args += ['-G', args.cmake_generator]
+                # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
+                if args.test:
+                    log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
+                    args.test = False
             else:
-                cmake_extra_args = ['-A', 'ARM64']
+                if args.msvc_toolset == '14.16' and args.cmake_generator == 'Visual Studio 16 2019':
-            cmake_extra_args += ['-G', args.cmake_generator]
+                    #CUDA 10.0 requires _MSC_VER >= 1700 and _MSC_VER < 1920, aka Visual Studio version in [2012, 2019)
-            # Cannot test on host build machine for cross-compiled builds (Override any user-defined behaviour for test if any)
+                    #In VS2019, we have to use Side-by-side minor version MSVC toolsets from Visual Studio 2017
-            if args.test:
+                    #14.16 is MSVC version
-                log.info("Cannot test on host build machine for cross-compiled ARM(64) builds. Will skip test running after build.")
+                    #141 is MSVC Toolset Version
-                args.test = False
+                    #Cuda VS extension should be installed to C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations
-          else:
+                    toolset = 'v141,host=x64,version=' + args.msvc_toolset
-            if args.msvc_toolset == '14.16' and args.cmake_generator == 'Visual Studio 16 2019':
+                elif args.msvc_toolset:
-                #CUDA 10.0 requires _MSC_VER >= 1700 and _MSC_VER < 1920, aka Visual Studio version in [2012, 2019)
+                    toolset = 'host=x64,version=' + args.msvc_toolset
-                #In VS2019, we have to use Side-by-side minor version MSVC toolsets from Visual Studio 2017
+                else:
-                #14.16 is MSVC version
+                    toolset = 'host=x64'
-                #141 is MSVC Toolset Version
+                if (args.cuda_version):
-                #Cuda VS extension should be installed to C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations
+                    toolset += ',cuda=' + args.cuda_version
-                toolset = 'v141,host=x64,version=' + args.msvc_toolset
+                cmake_extra_args = ['-A','x64','-T', toolset, '-G', args.cmake_generator]
-            elif args.msvc_toolset:
+            if args.enable_wcos:
-                toolset = 'host=x64,version=' + args.msvc_toolset
+                cmake_extra_args.append('-DCMAKE_TOOLCHAIN_FILE=' + os.path.join(source_dir, 'cmake', 'wcos_toolchain.cmake'))
-            else:
-                toolset = 'host=x64'
-            if (args.cuda_version):
-                toolset += ',cuda=' + args.cuda_version
-
-            cmake_extra_args = ['-A','x64','-T', toolset, '-G', args.cmake_generator]
         if args.android:
             # Cross-compiling for Android
             path_to_protoc_exe = build_protoc_for_host(cmake_path, source_dir, build_dir, args)

tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml

@@ -3,13 +3,13 @@ parameters:
   DoCompliance: 'false'
   PackageName: 'Microsoft.ML.ORTFeaturizers'
 
-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
   parameters:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-nuget'
     JobName: 'Windows_CI_Dev'
-    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x64'
     EnvSetupScript: 'setup_env.bat'
     sln_platform: 'x64'
@@ -30,7 +30,7 @@ jobs:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-win-x86-zip'
     JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --enable_wcos --x86 --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x86'
     EnvSetupScript: 'setup_env_x86.bat'
     sln_platform: 'Win32'

tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml

@@ -3,13 +3,13 @@ parameters:
   PackageName: 'Microsoft.ML.OnnxRuntime'
   DoCompliance: 'false'
 
-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
   parameters:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-nuget'
     JobName: 'Windows_CI_Dev'
-    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand:  '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x64'
     EnvSetupScript: 'setup_env.bat'
     sln_platform: 'x64'
@@ -29,7 +29,7 @@ jobs:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-win-x86-zip'
     JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x86'
     EnvSetupScript: 'setup_env_x86.bat'
     sln_platform: 'Win32'
@@ -50,7 +50,7 @@ jobs:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-win-arm64-zip'
     JobName: 'Windows_CI_Dev_arm64'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm64 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x64'
     EnvSetupScript: 'setup_env.bat'
     sln_platform: 'arm64'
@@ -71,7 +71,7 @@ jobs:
     AgentPool : 'Win-CPU-2019'
     ArtifactName: 'drop-win-arm-zip'
     JobName: 'Windows_CI_Dev_arm'
-    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
+    BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --arm --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"'
     BuildArch: 'x64'
     EnvSetupScript: 'setup_env.bat'
     sln_platform: 'arm'

tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml

@@ -2,21 +2,21 @@ parameters:
   DoEsrp: 'false'
   PackageName: 'Microsoft.ML.OnnxRuntime.Gpu'
 
-jobs: 
+jobs:
 - template: ../../templates/win-ci-2019.yml
   parameters:
     AgentPool : 'Win-GPU-2019'
-    ArtifactName: 'drop-nuget'    
+    ArtifactName: 'drop-nuget'
     JobName: 'Windows_CI_GPU_CUDA_Dev'
     BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda"
     BuildArch: 'x64'
     msbuildArchitecture: 'amd64'
-    EnvSetupScript: 'setup_env_cuda.bat'    
+    EnvSetupScript: 'setup_env_cuda.bat'
     sln_platform: 'x64'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: 'false'
-    DoEsrp: ${{ parameters.DoEsrp }}    
+    DoEsrp: ${{ parameters.DoEsrp }}
     CudaVersion: '10.1'
     OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
     NuPackScript: |
@@ -29,17 +29,17 @@ jobs:
 - template: ../../templates/win-ci-2019.yml
   parameters:
     AgentPool : 'Win-GPU-2019'
-    ArtifactName: 'drop-nuget-dml'    
+    ArtifactName: 'drop-nuget-dml'
     JobName: 'Windows_CI_GPU_DML_Dev'
-    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
+    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
     BuildArch: 'x64'
     msbuildArchitecture: 'amd64'
-    EnvSetupScript: 'setup_env_cuda.bat'    
+    EnvSetupScript: 'setup_env_cuda.bat'
     sln_platform: 'x64'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: 'false'
-    DoEsrp: ${{ parameters.DoEsrp }}    
+    DoEsrp: ${{ parameters.DoEsrp }}
     CudaVersion: '10.0'
     OrtPackageId: 'Microsoft.ML.OnnxRuntime.DirectML'
     NuPackScript: |
@@ -54,7 +54,7 @@ jobs:
     AgentPool : 'Win-GPU-2019'
     ArtifactName: 'drop-win-dml-x86-zip'
     JobName: 'Windows_CI_GPU_DML_Dev_x86'
-    BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
+    BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019"
     BuildArch: 'x86'
     EnvSetupScript: 'setup_env_x86.bat'
     sln_platform: 'Win32'
@@ -75,7 +75,7 @@ jobs:
   workspace:
     clean: all
   pool: $(AgentPoolLinux)
-  steps:    
+  steps:
     - template: ../../templates/set-version-number-variables-step.yml
     - template: ../../templates/linux-set-variables-and-download.yml
     - task: CmdLine@2
@@ -94,7 +94,7 @@ jobs:
        zip -r linux-x64.zip linux-x64
        cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory)
        mkdir $(Build.ArtifactStagingDirectory)/testdata
-       cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata 
+       cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
        ls -al $(Build.ArtifactStagingDirectory)
       displayName: 'Create Artifacts'
     - task: PublishPipelineArtifact@0
@@ -117,7 +117,7 @@ jobs:
   - Windows_CI_GPU_DML_Dev_x86
   - Linux_CI_GPU_Dev
   condition: succeeded()
-  steps:  
+  steps:
   - task: DownloadPipelineArtifact@0
     displayName: 'Download Pipeline Artifact - NuGet CUDA'
     inputs:
@@ -129,7 +129,7 @@ jobs:
     inputs:
       artifactName: 'drop-nuget-dml'
       targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
-      
+
   - task: DownloadPipelineArtifact@0
     displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
     inputs:
@@ -148,7 +148,7 @@ jobs:
      powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
      powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
      set PATH=%CD%;%PATH%
-     SETLOCAL EnableDelayedExpansion 
+     SETLOCAL EnableDelayedExpansion
      FOR /R %%i IN (*.nupkg) do (
          set filename=%%~ni
          IF NOT "!filename:~25,7!"=="Managed" (
@@ -175,7 +175,7 @@ jobs:
      powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
      powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
      set PATH=%CD%;%PATH%
-     FOR /R %%i IN (*.nupkg) do ( 
+     FOR /R %%i IN (*.nupkg) do (
          rename %%~ni.nupkg %%~ni.zip
          unzip %%~ni.zip -d %%~ni
          del /Q %%~ni.zip

tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml

@@ -46,9 +46,9 @@ jobs:
       env:
         TELEMETRYGUID: $(TELEMETRYGUID)
     - task: UsePythonVersion@0
-      inputs:
+      inputs: 
-        versionSpec: '3.7'
+        versionSpec: '3.7' 
-        addToPath: true
+        addToPath: true 
         architecture: ${{ parameters.BuildArch }}
 
     - task: BatchScript@1
@@ -59,7 +59,7 @@ jobs:
         workingFolder: '$(Build.BinariesDirectory)'
 
     - script: |
-       python -m pip install -q pyopenssl setuptools wheel numpy
+       python -m pip install -q pyopenssl setuptools wheel numpy     
       workingDirectory: '$(Build.BinariesDirectory)'
       displayName: 'Install python modules'
 

tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml

@@ -56,7 +56,7 @@ jobs:
     displayName: 'Generate cmake config'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --build_java'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java'
       workingDirectory: '$(Build.BinariesDirectory)'
 
   - task: VSBuild@1

tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml

@@ -37,12 +37,12 @@ jobs:
      python -m pip install -q pyopenssl setuptools wheel numpy     
     workingDirectory: '$(Build.BinariesDirectory)'
     displayName: 'Install python modules'
-    
+
   - task: PythonScript@0
     displayName: 'Generate cmake config'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --enable_wcos --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0'
       workingDirectory: '$(Build.BinariesDirectory)'
 
   - task: VSBuild@1
@@ -116,8 +116,8 @@ jobs:
      del wheel_filename_file
      python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
      set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH%
-     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0
+     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --build_shared_lib --enable_onnx_tests --enable_wcos --use_dml --use_winml --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0
-   
+
     workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
     displayName: 'Run tests'
  

tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml

@@ -34,17 +34,17 @@ jobs:
       workingFolder: '$(Build.BinariesDirectory)'
 
   - script: |
-     python -m pip install -q pyopenssl setuptools wheel numpy     
+     python -m pip install -q pyopenssl setuptools wheel numpy
     workingDirectory: '$(Build.BinariesDirectory)'
     displayName: 'Install python modules'
-    
+
   - task: PythonScript@0
     displayName: 'Generate cmake config'
     inputs:
       scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests'
+      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos'
       workingDirectory: '$(Build.BinariesDirectory)'
-     
+
   - task: VSBuild@1
     displayName: 'Build'
     inputs:
@@ -115,8 +115,8 @@ jobs:
      set /p WHEEL_FILENAME=<wheel_filename_file
      del wheel_filename_file
      python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
-     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --test --cmake_generator "Visual Studio 16 2019"  --x86 --build_wheel --enable_onnx_tests
+     python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --use_winml --test --cmake_generator "Visual Studio 16 2019"  --x86 --build_wheel --enable_onnx_tests --enable_wcos
-   
+
     workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
     displayName: 'Run tests'