Merged PR 6422353: Enable AMD64->Arm64 cross compilation and QEMU test run in ADO pipeline

+ Add option to skip performance tests in unit tests and use the option when running Arm64 unit tests in QEMU emulator + Only run quicker functional tests (skip DH, DSA, and RSA) in the QEMU emulator for now to avoid timing out + Enable Generic builds in PRs, only run the Generic unit tests in CI builds and PRs to publish + Update README to specify toolchain files in double-quoted paths + Check for NaNs closer to measurement in performance unit tests in continuing search for source of sporadic NaNs on x86 test runs Related work items: #35613708, #35714927
2021-09-02 18:08:26 +00:00 · 2021-09-02 18:08:26 +00:00 · 5d592a4602
--- a/README.md
+++ b/README.md
@ -32,9 +32,9 @@ The integrity check additionally requires pip and pyelftools: `pip3 install -r .
    2. In step 4 below, add the additional cmake argument `-DSYMCRYPT_INTERNAL_BUILD=1`
 3. `mkdir bin; cd bin`
 4. Configure CMake compilation:
-    * For 32-bit Windows targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/WindowsUserMode-X86.cmake -A Win32`
-    * For 64-bit Windows targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/WindowsUserMode-AMD64.cmake`
-    * For 64-bit Linux targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/LinuxUserMode-AMD64.cmake`
+    * For 32-bit Windows targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE="../cmake-toolchain/WindowsUserMode-X86.cmake" -A Win32`
+    * For 64-bit Windows targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE="../cmake-toolchain/WindowsUserMode-AMD64.cmake"`
+    * For 64-bit Linux targets: `cmake .. -DCMAKE_TOOLCHAIN_FILE="../cmake-toolchain/LinuxUserMode-AMD64.cmake"`
    * For no CPU optimizations: `cmake ..`
    * Optionally, for a release build, specify `-DCMAKE_BUILD_TYPE=Release`
 5. `cmake --build .`
--- a/azure-build-template.yml
+++ b/azure-build-template.yml
@ -45,6 +45,12 @@ parameters:
 - name: additionalCMakeArgs
  type: string
  default: ''
+- name: emulator
+  type: string
+  default: native
+  values:
+  - native
+  - qemu-aarch64

 steps:
  - checkout: self  # self represents the repo where the initial Pipelines YAML file was found
@ -57,6 +63,12 @@ steps:
        python -m pip install --upgrade pip setuptools wheel
        pip install -r $(Build.SourcesDirectory)/scripts/requirements.txt
      displayName: 'Install Python requirements'
+  - ${{ if eq(parameters.emulator, 'qemu-aarch64') }}:
+    - script: |
+        sudo apt-get update
+        sudo apt-get install qemu-user binutils-aarch64-linux-gnu
+        sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+      displayName: 'Install arm64 cross-compilation and emulation tools'
  # Specify no toolchain file for generic build
  - ${{ if eq(parameters.env, 'Generic') }}:
    - task: CMake@1
@ -92,12 +104,14 @@ steps:
          .\symcryptunittest.exe
        displayName: 'Execute unit tests'
        name: '${{parameters.env}}UnitTest_${{parameters.buildType}}'
+    # Execute Generic unit tests in CI and in PRs to publish
    - ${{ if eq(parameters.env, 'Generic') }}:
      - script: |
          cd bin\exe\%PROCESSOR_ARCHITECTURE%\${{parameters.env}}\${{parameters.buildType}}
          .\symcryptunittest.exe
        displayName: 'Execute unit tests'
        name: '${{parameters.env}}UnitTest_${{parameters.buildType}}'
+        condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
  - ${{ if eq(parameters.hostos, 'Linux') }}:
    - task: CMake@1
      inputs:
@ -107,23 +121,31 @@ steps:
    - script: |
        ulimit -c unlimited
      displayName: 'Enable core dumps'
-    # Execute module and unit test using the inline script
    - ${{ if ne(parameters.env, 'Generic') }}:
-      - script: |
-          cd bin/exe/${{parameters.arch}}/${{parameters.env}}
-          ulimit -c unlimited
-          ./symcryptunittest
-        displayName: 'Execute unit tests'
-        name: '${{parameters.env}}UnitTest_${{parameters.buildType}}'
-      - ${{ if ne(parameters.buildType, 'Sanitize') }}:
-        # Only run module test if rdseed is present on the CPU
+      # Execute module and unit test using the inline script
+      - ${{ if eq(parameters.emulator, 'native') }}:
        - script: |
            cd bin/exe/${{parameters.arch}}/${{parameters.env}}
-            if lshw -c cpu | grep -q rdseed
-            then ./symcryptmoduletest
-            fi
-          displayName: 'Execute module test'
-    # Execute unit test using the inline script
+            ./symcryptunittest
+          displayName: 'Execute unit tests'
+          name: '${{parameters.env}}UnitTest_${{parameters.buildType}}'
+        - ${{ if ne(parameters.buildType, 'Sanitize') }}:
+          # Only run module test if rdseed is present on the CPU
+          - script: |
+              cd bin/exe/${{parameters.arch}}/${{parameters.env}}
+              if lshw -c cpu | grep -q rdseed
+              then ./symcryptmoduletest
+              fi
+            displayName: 'Execute module test'
+      # Execute unit test using the inline script (Arm64 module not yet supported)
+      # Currently skip slower algorithms (DH/RSA), as otherwise we time out the pipeline
+      - ${{ if eq(parameters.emulator, 'qemu-aarch64') }}:
+        - script: |
+            cd bin/exe/${{parameters.arch}}/${{parameters.env}}
+            qemu-aarch64 -L /usr/aarch64-linux-gnu/ ./symcryptunittest noperftests +symcrypt -dh -dsa -rsa
+          displayName: 'Execute unit tests'
+          name: '${{parameters.env}}UnitTest_${{parameters.buildType}}_QEMU_AARCH64'
+    # Execute Generic unit tests in CI and in PRs to publish
    - ${{ if eq(parameters.env, 'Generic') }}:
      - script: |
          archName=`uname -m`
@ -131,6 +153,7 @@ steps:
          ./symcryptunittest
        displayName: 'Execute unit tests'
        name: '${{parameters.env}}UnitTest_${{parameters.buildType}}'
+        condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
  # Publish artifacts so they're available in the pipeline results
  - publish: $(System.DefaultWorkingDirectory)/bin
    artifact: 'drop-${{parameters.hostos}}-${{parameters.env}}-${{parameters.arch}}-${{parameters.cc}}-${{parameters.buildType}}-${{parameters.additionalCMakeArgs}}'
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -24,11 +24,16 @@ pr:
 # 8. Linux AMD64 with CPU optimizations using clang in Debug mode
 # 9. Linux AMD64 with CPU optimizations using clang in Sanitize mode
 # 10. Linux AMD64 with CPU optimizations using clang in Release mode
-# The following jobs run only on CI and PRs to publish
-# 11. Windows 64b with no CPU optimizations in Release mode
-# 12. Windows 32b with no CPU optimizations in Release mode
-# 13. Linux with no CPU optimizations using GCC in Release mode
-# 14. Linux with no CPU optimizations using clang in Release mode
+
+# The following 2 jobs assume running on an AMD64 host, install cross compilation tools for Arm64 and runs a subset of units tests in QEMU
+# 11. Linux Arm64 with CPU optimizations using clang in Debug mode
+# 12. Linux Arm64 with CPU optimizations using clang in Release mode
+
+# The following 4 jobs only run the unit tests on CI and PRs to publish (they are always built)
+# 13. Windows 64b with no CPU optimizations in Release mode
+# 14. Windows 32b with no CPU optimizations in Release mode
+# 15. Linux with no CPU optimizations using GCC in Release mode
+# 16. Linux with no CPU optimizations using clang in Release mode

 jobs:
 - job: Windows_AMD64_Debug
@ -171,6 +176,35 @@ jobs:
      buildType: Release
      additionalCMakeArgs:

+- job: Linux_QEMU_ARM64_clang_Debug
+  pool:
+    vmImage: 'ubuntu-20.04'
+  steps:
+  - template: azure-build-template.yml
+    parameters:
+      hostos: Linux
+      env: LinuxUserMode
+      arch: ARM64
+      cc: clang
+      cxx: clang++
+      buildType: Debug
+      additionalCMakeArgs:
+      emulator: qemu-aarch64
+
+- job: Linux_QEMU_ARM64_clang_Release
+  pool:
+    vmImage: 'ubuntu-20.04'
+  steps:
+  - template: azure-build-template.yml
+    parameters:
+      hostos: Linux
+      env: LinuxUserMode
+      arch: ARM64
+      cc: clang
+      cxx: clang++
+      buildType: Release
+      additionalCMakeArgs:
+      emulator: qemu-aarch64

 - job: Generic_Windows_Win64_Release
  pool:
@ -185,7 +219,6 @@ jobs:
      cxx: cl
      buildType: Release
      additionalCMakeArgs:
-  condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))

 - job: Generic_Windows_Win32_Release
  pool:
@ -200,7 +233,6 @@ jobs:
      cxx: cl
      buildType: Release
      additionalCMakeArgs: -A Win32
-  condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))

 - job: Generic_Linux_gcc_Release
  pool:
@ -215,7 +247,6 @@ jobs:
      cxx: g++
      buildType: Release
      additionalCMakeArgs:
-  condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))

 - job: Generic_Linux_clang_Release
  pool:
@ -230,4 +261,3 @@ jobs:
      cxx: clang++
      buildType: Release
      additionalCMakeArgs:
-  condition: or(eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/publish'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
--- a/cmake-toolchain/LinuxUserMode-ARM64.cmake
+++ b/cmake-toolchain/LinuxUserMode-ARM64.cmake
@ -13,6 +13,20 @@ set(CMAKE_C_COMPILER_TARGET ${TARGET_TRIPLE})
 set(CMAKE_CXX_COMPILER clang++)
 set(CMAKE_CXX_COMPILER_TARGET ${TARGET_TRIPLE})

+# Point clang sysroot to cross compilation toolchain when cross compiling
+if(NOT CMAKE_HOST_SYSTEM_PROCESSOR EQUAL CMAKE_SYSTEM_PROCESSOR)
+    # C/C++ toolchain (installed on Ubuntu using apt-get gcc-aarch64-linux-gnu g++-aarch64-linux-gnu)
+    set(CMAKE_SYSROOT_COMPILE /usr/${TARGET_TRIPLE})
+
+    # We would expect setting SYSROOT to be sufficient for clang to cross-compile with the gcc-aarch64-linux-gnu
+    # toolchain, but it seems that this misses a few key header files for C++...
+    # Hacky solution which seems to work for Ubuntu + clang:
+    # Get CMake to find the appropriate include directory and explicitly include it
+    # Seems like there should be a better way to install cross-compilation tools, or specify search paths to clang
+    find_path(CXX_CROSS_INCLUDE_DIR NAMES ${TARGET_TRIPLE} PATHS /usr/${TARGET_TRIPLE}/include/c++/ PATH_SUFFIXES 15 14 13 12 11 10 9 8 7 6 5 NO_DEFAULT_PATH)
+    add_compile_options(-I${CXX_CROSS_INCLUDE_DIR}/${TARGET_TRIPLE})
+endif()
+
 # For now this is just used for separating the output directories
 set(SYMCRYPT_TARGET_ENV LinuxUserMode)

--- a/unittest/lib/main.cpp
+++ b/unittest/lib/main.cpp
@ -259,6 +259,12 @@ BOOL g_showPerfRangeInfo = FALSE;
 //
 BOOL g_verbose = FALSE;

+//
+// Option to skip running performance tests
+// Helpful when running in an emulated environment where we just want to test the functionality
+//
+BOOL g_noPerfTests = FALSE;
+
 //
 // Profiling options to run an algorithm in a tight loop
 //
@ -429,6 +435,7 @@ updateNameSet( _In_z_ PCSTR * names, _Inout_ StringSet * set, CHAR op, _In_ PSTR
                    }
                }
                set->erase( names[i] );
+                nameMatch = TRUE;
            }
        }
    }
@ -451,7 +458,8 @@ isAlgorithmPresent( String algName, BOOL isPrefix )
        } else {
            if( (*i)->m_algorithmName == algName )
            {
-                return TRUE;            }
+                return TRUE;
+            }
        }
    }
    return FALSE;
@ -591,6 +599,7 @@ usage()
            "                           a single .csv for postprocessing."
            "  kernel            Run the kernel-mode tests \n"
            "  verbose           Print detailed information for some algorithms\n"
+            "  noperftests       Skip running the performance tests - only run functional tests\n"
            "  profile:xxx [key=yyy]    Run one or more algorithms in a tight loop, xxx times for\n"
            "                           each key/datasize combination. The algorithms to run are specified\n"
            "                           by the +/- options. The optional key parameter can specify\n"
@ -747,6 +756,12 @@ processSingleOption( _In_ PSTR option )
            optionHandled = TRUE;
        }

+        if( STRICMP( &option[0], "noperftests" ) == 0 )
+        {
+            g_noPerfTests = TRUE;
+            optionHandled = TRUE;
+        }
+
        if( STRNICMP( &option[0], "profile:", 8 ) == 0 )
        {
            char * endptr;
@ -1598,7 +1613,7 @@ runFunctionalTests()
 VOID
 runPerfTests()
 {
-    if( g_runKernelmodeTest )
+    if( g_runKernelmodeTest || g_noPerfTests )
    {
        return;
    }
--- a/unittest/lib/perf.cpp
+++ b/unittest/lib/perf.cpp
@ -573,6 +573,9 @@ double measureDataPerfGivenStack(
        {
            durations[i] *= measurementScaleFactor;
        }
+
+        CHECK3( !isnan(measurementScaleFactor), "NaN result in measureDataPerfGivenStack measurementScaleFactor: %f", measurementScaleFactor );
+        CHECK4( !isnan(durations[i]), "NaN result in measureDataPerfGivenStack durations[%d]: %f", i, durations[i] );
        //average[i+1] = fixedAverage; // Helpful when debugging

        ++i;