Merged PR 9200816: Add support for Windows undocked pipeline build

This PR adds the necessary YAML pipeline files for building SymCrypt via the Windows undocked pipeline. It also includes minor changes to existing files to support the tooling used by the pipeline. Currently, the pipeline only exists for pull requests. The next step is to create an official pipeline, including the option to package the binaries and ingest them into Windows. This will come in a subsequent PR.
2023-06-28 20:38:13 +00:00 · 2023-06-28 20:38:13 +00:00 · 9b760abcf3
--- a/.pipelines/OneBranch.WindowsUndocked.PullRequest.yml
+++ b/.pipelines/OneBranch.WindowsUndocked.PullRequest.yml
@ -0,0 +1,44 @@
+################################################################################
+#                  OneBranch Pipelines - PR Build                              #
+################################################################################
+#                                                                              #
+# This pipeline is for validation of pull requests. Useful links:              #
+#                                                                              #
+#   https://www.osgwiki.com/wiki/Windows_Undocked_Template                     #
+#   https://aka.ms/obpipelines                                                 #
+#   https://aka.ms/obpipelines/yaml/schema                                     #
+#   https://aka.ms/obpipelines/tasks                                           #
+#   https://aka.ms/onebranchsup                                                #
+#                                                                              #
+################################################################################
+
+trigger: none # https://aka.ms/obpipelines/triggers
+
+pr:
+- main
+
+variables:
+  # https://aka.ms/obpipelines/containers
+  WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
+
+resources:
+  repositories:
+  - repository: templates
+    type: git
+    name: OneBranch.Pipelines/GovernedTemplates
+    ref: refs/heads/main
+
+extends:
+  # https://aka.ms/obpipelines/templates
+  template: v2/Microsoft.NonOfficial.yml@templates
+  parameters:
+    platform:
+      name: 'windows_undocked'
+    featureFlags:
+      WindowsHostVersion: '1ESWindows2022'
+    stages:
+    - stage: Build
+      jobs:
+      - template: .pipelines/templates/build-windows-undocked.yml@self
+        parameters:
+          sln: SymCrypt.sln
--- a/.pipelines/templates/build-linux.yml
+++ b/.pipelines/templates/build-linux.yml
@ -42,6 +42,10 @@ jobs:
  variables:
    ob_outputDirectory: $(Build.SourcesDirectory)/bin
    ob_sdl_binskim_break: ${{ eq(parameters.config, 'Release') }} # Only break on binskim failures for Release binaries https://aka.ms/obpipelines/sdl
+    ${{ if eq(variables['system.debug'], true) }}:
+      verbose_build_flag: '--verbose'
+    ${{ else }}:
+      verbose_build_flag: ''

  steps:

@ -93,7 +97,7 @@ jobs:
      inputs:
        scriptSource: 'filePath'
        scriptPath: scripts/build.py
-        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }}'
+        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }} $(verbose_build_flag)'
        workingDirectory: $(Build.SourcesDirectory)

    - ${{ if ne(parameters.skipTests, true) }}:
--- a/.pipelines/templates/build-windows-undocked.yml
+++ b/.pipelines/templates/build-windows-undocked.yml
@ -0,0 +1,133 @@
+# This template is for building, signing, packaging and ingesting undocked Windows binaries.
+parameters:
+  # Build args
+  sln: ''
+  msbuildArgs: ''
+  restoreNugetPackages: false
+  targetOsBranch: 'official/main'
+  config: 'Debug,Release'
+  platform: 'x86,x64,arm64'
+  nativeCompiler: true
+  sign: false # Only signs UM binaries, for external (to Windows repo) release
+
+jobs:
+# Build the solution for all the necessary configurations.
+- job: build_sln
+  displayName: ${{ parameters.sln }}
+  ${{ if eq(parameters.package, true) }}:
+    dependsOn: set_version
+  strategy:
+    matrix:
+      ${{ if and(contains(parameters.config, 'Debug'), contains(parameters.platform, 'x86')) }}:
+        x86chk:
+          ob_build_platform: x86
+          ob_build_platform_win: x86
+          ob_build_config: Debug
+          ob_build_config_win: chk
+      ${{ if and(contains(parameters.config, 'Release'), contains(parameters.platform, 'x86')) }}:
+        x86fre:
+          ob_build_platform: x86
+          ob_build_platform_win: x86
+          ob_build_config: Release
+          ob_build_config_win: fre
+      ${{ if and(contains(parameters.config, 'Debug'), contains(parameters.platform, 'x64')) }}:
+        amd64chk:
+          ob_build_platform: x64
+          ob_build_platform_win: amd64
+          ob_build_config: Debug
+          ob_build_config_win: chk
+      ${{ if and(contains(parameters.config, 'Release'), contains(parameters.platform, 'x64')) }}:
+        amd64fre:
+          ob_build_platform: x64
+          ob_build_platform_win: amd64
+          ob_build_config: Release
+          ob_build_config_win: fre
+      ${{ if and(contains(parameters.config, 'Debug'), contains(parameters.platform, 'arm64')) }}:
+        arm64chk:
+          ob_build_platform: arm64
+          ob_build_platform_win: arm64
+          ob_build_config: Debug
+          ob_build_config_win: chk
+      ${{ if and(contains(parameters.config, 'Release'), contains(parameters.platform, 'arm64')) }}:
+        arm64fre:
+          ob_build_platform: arm64
+          ob_build_platform_win: arm64
+          ob_build_config: Release
+          ob_build_config_win: fre
+      # Note: must specify "arm32" for ARM because "arm" is a substring of "arm64" and ADO YAML
+      # does not support true arrays, so we can't use "contains" to check for "arm" in the platform.
+      ${{ if and(contains(parameters.config, 'Debug'), contains(parameters.platform, 'arm32')) }}:
+        armchk:
+          ob_build_platform: arm
+          ob_build_platform_win: arm
+          ob_build_config: Debug
+          ob_build_config_win: chk
+      ${{ if and(contains(parameters.config, 'Release'), contains(parameters.platform, 'arm32')) }}:
+        armfre:
+          ob_build_platform: arm
+          ob_build_platform_win: arm
+          ob_build_config: Release
+          ob_build_config_win: fre
+
+  pool:
+    type: windows
+
+  variables:
+    # Build variables
+    ob_outputDirectory: $(Build.SourcesDirectory)\build\bin\$(ob_build_platform_win)$(ob_build_config_win)
+    ob_artifactSuffix: _$(ob_build_platform_win)$(ob_build_config_win)
+    # https://aka.ms/obpipelines/sdl
+    ob_sdl_tsa_enabled: false # When TSA is disabled all SDL tools will forced into 'break' build mode.
+    ob_sdl_binskim_break: true
+    ob_sdl_policheck_break: true
+    ${{ if eq(parameters.sign, true) }}:
+      ob_sdl_codeSignValidation_excludes: -|**\*.sys # Signing is not supported for KM drivers
+    ${{ if eq(parameters.sign, false) }}:
+      ob_sdl_codeSignValidation_excludes: -|**\*.sys;-|**\*.dll;-|**\*.exe # Disable signing requirements for UM & KM builds
+    # https://eng.ms/docs/cloud-ai-platform/azure-edge-platform-aep/aep-engineering-systems/productivity-and-experiences/onebranch-windows-undocked/onebranch-windows-undocked/test/onebranchwindowsundocked
+    ob_NativeCompiler_enabled: ${{ parameters.nativeCompiler }}
+    ob_NativeCompiler_TaskVerbosity: 'Detailed'
+    ob_NativeCompiler_UseOSBranchVersion: true
+    ob_NativeCompiler_TargetOsBranch: ${{ parameters.targetOsBranch }}
+    ob_NativeCompiler_UcrtPlatform: $(ob_build_platform_win)
+
+  steps:
+  - task: VSBuild@1
+    displayName: Build
+    target: windows_build_container
+    inputs:
+      solution: ${{ parameters.sln }}
+      platform: $(ob_build_platform)
+      configuration: $(ob_build_config)
+      maximumCpuCount: true
+      restoreNugetPackages: ${{ parameters.restoreNugetPackages }}
+      msbuildArgs: '-p:UndockedOfficial=${{ parameters.nativeCompiler }} -p:UndockedBuildId=$(Build.BuildId) ${{ parameters.msbuildArgs }}'
+
+  # Only run tests on x86/x64 builds for now
+  - task: PythonScript@0
+    displayName: 'Run unit tests'
+    condition: or(eq(variables['ob_build_platform'], 'x64'), eq(variables['ob_build_platform'], 'x86')) 
+    inputs:
+      scriptSource: 'filePath'
+      scriptPath: scripts\test.py
+      arguments: 'build\bin noperftests'
+      workingDirectory: $(Build.SourcesDirectory)
+
+  - task: PythonScript@0
+    displayName: 'Run dynamic unit tests'
+    condition: or(eq(variables['ob_build_platform'], 'x64'), eq(variables['ob_build_platform'], 'x86')) 
+    inputs:
+      scriptSource: 'filePath'
+      scriptPath: scripts\test.py
+      arguments: 'build\bin dynamic:build\bin\$(ob_build_platform_win)$(ob_build_config_win)\symcrypttestmodule.dll noperftests'
+      workingDirectory: $(Build.SourcesDirectory)      
+
+  - ${{ if eq(parameters.sign, true) }}:
+    - task: onebranch.pipeline.signing@1 # https://aka.ms/obpipelines/signing
+      displayName: 'Sign Binaries'
+      target: windows_build_container
+      inputs:
+        command: 'sign'
+        signing_profile: 'external_distribution'
+        files_to_sign: '**\*.exe;**\*.dll' # Only supports user mode binaries
+        search_root: 'build/bin'
--- a/.pipelines/templates/build-windows.yml
+++ b/.pipelines/templates/build-windows.yml
@ -31,6 +31,10 @@ jobs:
  variables:
    ob_outputDirectory: $(Build.SourcesDirectory)\bin
    ob_sdl_binskim_break: ${{ eq(parameters.config, 'Release') }} # Only break on binskim failures for Release binaries https://aka.ms/obpipelines/sdl
+    ${{ if eq(variables['system.debug'], true) }}:
+      verbose_build_flag: '--verbose'
+    ${{ else }}:
+      verbose_build_flag: ''

  steps:
  
@ -49,7 +53,7 @@ jobs:
    inputs:
      scriptSource: 'filePath'
      scriptPath: scripts\build.py
-      arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} ${{ parameters.additionalArgs }}'
+      arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} ${{ parameters.additionalArgs }} $(verbose_build_flag)'
      workingDirectory: $(Build.SourcesDirectory)

  - ${{ if ne(parameters.skipTests, true) }}:
@ -65,7 +69,7 @@ jobs:
      displayName: 'Run dynamic unit tests'
      inputs:
        scriptSource: 'filePath'
-        scriptPath: scripts/test.py
+        scriptPath: scripts\test.py
        arguments: 'bin dynamic:bin\exe\symcrypttestmodule.dll noperftests'
        workingDirectory: $(Build.SourcesDirectory)      

--- a/lib/sc_lib.h
+++ b/lib/sc_lib.h
@ -48,6 +48,8 @@ typedef int                 BOOL;
 #pragma prefast( disable:28113 )
 #pragma warning( disable: 4702 )        // unreachable code. The compilers are not equally smart, and some complain
                                        // about 'function must return a value' and some about 'unreachable code'
+#pragma warning( disable: 4296 )        // expression is always false - this warning is forced to be an error by a
+                                        // pragma in the SDK warning.h, but we don't consider it useful


 //
--- a/msbuild/symcrypt.undocked.props
+++ b/msbuild/symcrypt.undocked.props
@ -46,10 +46,18 @@
      </AdditionalOptions>
    </Link>
  </ItemDefinitionGroup>
+  <!-- osgvsowi/44690219 - MultiThreadedDebug CRT is blocked on architectures other than AMD64
  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
    <ClCompile>
      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+  -->
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
    </ClCompile>
  </ItemDefinitionGroup>
  <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
@ -62,13 +70,9 @@
      <CallingConvention>StdCall</CallingConvention>
      <PreprocessorDefinitions>_X86_;%(PreprocessorDefinitions)</PreprocessorDefinitions>
    </ClCompile>
-    <Link>
-      <!-- Workaround for LNK2026: module unsafe for SAFESEH image -->
-      <AdditionalOptions>
-        /SAFESEH:NO
-        %(AdditionalOptions)
-      </AdditionalOptions>
-    </Link>
+    <MASM>
+      <UseSafeExceptionHandlers>true</UseSafeExceptionHandlers>
+    </MASM>
  </ItemDefinitionGroup>
  <ItemDefinitionGroup Condition="'$(Platform)'=='ARM64'">
    <ClCompile>
--- a/scripts/test.py
+++ b/scripts/test.py
@ -30,7 +30,7 @@ def run_unittest(build_dir : pathlib.Path, emulator : str,
            print("Warning: --glibc-disable-ymm is not supported on Windows.", file = sys.stderr)
            disable_ymm = False

-        # Build the path to the executable
+    # Build the path to the executable
    unittest_search_path = pathlib.Path(build_dir)
    unittest_candidates = unittest_search_path.rglob("**/{}{}".format(
        UNITTEST_FILENAME, UNITTEST_EXTENSION_WINDOWS if sys.platform == "win32" else ""))
@ -65,7 +65,7 @@ def run_unittest(build_dir : pathlib.Path, emulator : str,
        subprocess.run(unittest_invocation, env = env, check = True)
    except subprocess.CalledProcessError as e:
        print("Unit test exited unsuccessfully with code " + str(e.returncode), file = sys.stderr)
-        return e.returncode
+        exit(e.returncode)

    return 0

@ -84,7 +84,6 @@ def main() -> int:
    args = parser.parse_args()

    result = run_unittest(args.build_dir, args.emulator, args.emulator_lib_dir, args.glibc_disable_ymm, *args.additional_args)
-    return result

 if __name__ == "__main__":
    main()
--- a/unittest/exe_test/SymCryptUnitTest.vcxproj
+++ b/unittest/exe_test/SymCryptUnitTest.vcxproj
@ -38,7 +38,6 @@
      <WholeProgramOptimization>true</WholeProgramOptimization>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <ExceptionHandling>Sync</ExceptionHandling>
-      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
    </ClCompile>
    <Link>
      <SubSystem>Console</SubSystem>
@ -46,7 +45,6 @@
      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
      <AdditionalDependencies>%(AdditionalDependencies);rsa32.lib;msbignum.lib;ntdll.lib;Bcrypt.lib</AdditionalDependencies>
      <AdditionalLibraryDirectories>..\SymCryptDependencies\$(ProcessorArchitecture)</AdditionalLibraryDirectories>
-      <AdditionalOptions>/nodefaultlib:libcmt.lib %(AdditionalOptions)</AdditionalOptions>
    </Link>
  </ItemDefinitionGroup>
  <ItemGroup>
--- a/unittest/inc/test_lib.h
+++ b/unittest/inc/test_lib.h
@ -291,7 +291,9 @@ extern "C" {
 #pragma warning( disable: 4127 )        // conditional expression is constant
 #pragma warning( disable: 6262 )        // excessive stack usage. This is test code, I don't care.
 #pragma warning( disable: 4702 )        // unreachable code. The compilers are not equally smart, and some complain
-                                        // aobut 'function must return a value' and some about 'unreachable code'
+                                        // about 'function must return a value' and some about 'unreachable code'
+#pragma warning( disable: 4296 )        // expression is always false - this warning is forced to be an error by a
+                                        // pragma in the SDK warning.h, but we don't consider it useful

 //
 // Macros for different environments
--- a/unittest/lib/main.cpp
+++ b/unittest/lib/main.cpp
@ -1369,7 +1369,7 @@ initTestInfrastructure( int argc, _In_reads_( argc ) char * argv[] )
        moduleLoadStart = GET_PERF_CLOCK();
        g_dynamicSymCryptModuleHandle = loadDynamicModuleFromPath(g_dynamicModulePath.c_str());
        moduleLoadEnd = GET_PERF_CLOCK();
-        CHECK(g_dynamicSymCryptModuleHandle != NULL, "!");
+        CHECK(g_dynamicSymCryptModuleHandle != NULL, "Failed to load dynamic module. Is the path correct?");

        iprint("\nLoaded %s to %llx\nTook ~%d cycles.\n", g_dynamicModulePath.c_str(), (UINT64)g_dynamicSymCryptModuleHandle, moduleLoadEnd-moduleLoadStart);

--- a/unittest/lib/symcryptunittest_lib.vcxproj
+++ b/unittest/lib/symcryptunittest_lib.vcxproj
@ -17,7 +17,7 @@
    <ClCompile>
      <AdditionalIncludeDirectories>..\;..\inc;..\SymCryptDependencies\inc;..\..\inc;..\..\lib;</AdditionalIncludeDirectories>
      <LanguageStandard>stdcpp17</LanguageStandard>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <AdditionalOptions>/Zc:strictStrings- /bigobj /wd5252 %(AdditionalOptions)</AdditionalOptions>
      <WholeProgramOptimization>true</WholeProgramOptimization>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
--- a/unittest/module_windows/symcrypttestmodule.vcxproj
+++ b/unittest/module_windows/symcrypttestmodule.vcxproj
@ -47,7 +47,6 @@
      <AdditionalDependencies>%(AdditionalDependencies);rsa32.lib;msbignum.lib;ntdll.lib;Bcrypt.lib</AdditionalDependencies>
      <AdditionalLibraryDirectories>..\SymCryptDependencies\$(ProcessorArchitecture)</AdditionalLibraryDirectories>
      <ModuleDefinitionFile>exports.def</ModuleDefinitionFile>
-      <AdditionalOptions>/nodefaultlib:libcmt.lib %(AdditionalOptions)</AdditionalOptions>
    </Link>
  </ItemDefinitionGroup>
  <ItemGroup>