From 60b7e5e5fa1d3d6e0feb2652aaf825c7280b49be Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:38:11 -0700 Subject: [PATCH] Refactor OneBranch CI pipelines to use templates (#548) Added some templates to make it easier to reuse build definition logic between the two OneBranch pipelines. --- .pipelines/OneBranch.Official.yml | 91 ++---------------- .pipelines/OneBranch.PullRequest.yml | 91 ++---------------- .../templates/onebranch-common-variables.yml | 15 +++ .../onebranch-windows-build-stage.yml | 92 +++++++++++++++++++ 4 files changed, 119 insertions(+), 170 deletions(-) create mode 100644 .pipelines/templates/onebranch-common-variables.yml create mode 100644 .pipelines/templates/onebranch-windows-build-stage.yml diff --git a/.pipelines/OneBranch.Official.yml b/.pipelines/OneBranch.Official.yml index b4eda506..c6de0380 100644 --- a/.pipelines/OneBranch.Official.yml +++ b/.pipelines/OneBranch.Official.yml @@ -22,18 +22,9 @@ parameters: # parameters are shown up in ADO UI in a build queue time default: 'ExampleEnvVar=ON' variables: - CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] - system.debug: ${{ parameters.debug }} - ENABLE_PRS_DELAYSIGN: 1 - ROOT: $(Build.SourcesDirectory) - REPOROOT: $(Build.SourcesDirectory) - OUTPUTROOT: $(REPOROOT)\out - NUGET_XMLDOC_MODE: none - CIBW_BUILD: cp3{8,9,10,11}-*amd64 - CIBW_ARCHS: AMD64 - CIBW_BUILD_VERBOSITY: 1 - DEBIAN_FRONTEND: noninteractive - CIBW_ENVIRONMENT: "${{ parameters.ExtraEnv }}" +- template: /.pipelines/templates/onebranch-common-variables.yml@self + parameters: + debug: ${{ parameters.debug }} resources: repositories: @@ -62,76 +53,6 @@ extends: exclusionsFile: '$(REPOROOT)\.config\policheck_exclusions.xml' stages: - - stage: Windows_Build - jobs: - - job: main - pool: - type: windows - - variables: - ob_outputDirectory: '$(REPOROOT)\out' - ob_sdl_binskim_break: true - steps: - - script: | - @echo off - set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" - for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( - set vslatest="%%i" - if exist "%%i\Common7\Tools\vsdevcmd.bat" ( - set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat" - ) - if exist "%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" ( - set vscmake="%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" - ) - if exist "%%i\MSBuild\Current\Bin\amd64\msbuild.exe" ( - set vsmsbuild="%%i\MSBuild\Current\Bin\amd64\msbuild.exe" - ) - ) - - @echo vslatest %vslatest% - @echo vsdevcmd %vsdevcmd% - @echo vscmake %vscmake% - @echo vsmsbuild %vsmsbuild% - - @echo ##vso[task.setvariable variable=vslatest]%vslatest% - @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd% - @echo ##vso[task.setvariable variable=vscmake]%vscmake% - @echo ##vso[task.setvariable variable=vsmsbuild]%vsmsbuild% - displayName: 'locate vsdevcmd via vswhere' - - script: | - call $(vsdevcmd) -arch=amd64 - set PYTHONPATH= - set PYTHONHOME= - python -m pip install --upgrade pip - python -m pip install cibuildwheel numpy - python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out - displayName: Build wheels - - task: SDLNativeRules@3 - inputs: - msBuildArchitecture: amd64 - setupCommandlines: '"$(vscmake)" $(REPOROOT) -A x64 -B $(REPOROOT)\windows_out -DOCOS_BUILD_PYTHON=ON -DOCOS_ENABLE_CTEST=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo' - msBuildCommandline: '"$(vsmsbuild)" "$(REPOROOT)\windows_out\onnxruntime_extensions.sln" /p:RunCodeAnalysis=true /p:platform=x64 /p:configuration=RelWithDebInfo /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(REPOROOT)\windows_out#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - - task: PostAnalysis@2 - inputs: - GdnBreakGdnToolSDLNativeRulesSeverity: Warning - GdnBreakGdnToolSDLNativeRules: true - displayName: 'Guardian Break' - - script: | - 7z x $(REPOROOT)\out\*.whl -o$(REPOROOT)\out\* - displayName: unzip the package - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: 'external_distribution' - files_to_sign: '**/**/*.pyd;**/**/*.dll' - search_root: '$(REPOROOT)\out' - displayName: 'Sign runtime DLLs' - - script: | - for /D %%i in ("out\onnxruntime_extensions*-win*") do ( - 7z u %%i.whl .\%%i\* - rmdir /s /q %%i - ) - workingDirectory: '$(REPOROOT)' - displayName: zip package + - template: /.pipelines/templates/onebranch-windows-build-stage.yml@self + parameters: + CibwEnv: ${{ parameters.ExtraEnv }} diff --git a/.pipelines/OneBranch.PullRequest.yml b/.pipelines/OneBranch.PullRequest.yml index 27a155dd..8c5e2735 100644 --- a/.pipelines/OneBranch.PullRequest.yml +++ b/.pipelines/OneBranch.PullRequest.yml @@ -22,18 +22,9 @@ parameters: # parameters are shown up in ADO UI in a build queue time default: 'ExampleEnvVar=ON' variables: - CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] - system.debug: ${{ parameters.debug }} - ENABLE_PRS_DELAYSIGN: 1 - ROOT: $(Build.SourcesDirectory) - REPOROOT: $(Build.SourcesDirectory) - OUTPUTROOT: $(REPOROOT)\out - NUGET_XMLDOC_MODE: none - CIBW_BUILD: cp3{8,9,10,11}-*amd64 - CIBW_ARCHS: AMD64 - CIBW_BUILD_VERBOSITY: 1 - DEBIAN_FRONTEND: noninteractive - CIBW_ENVIRONMENT: "${{ parameters.ExtraEnv }}" +- template: /.pipelines/templates/onebranch-common-variables.yml@self + parameters: + debug: ${{ parameters.debug }} resources: repositories: @@ -62,76 +53,6 @@ extends: exclusionsFile: '$(REPOROOT)\.config\policheck_exclusions.xml' stages: - - stage: Windows_Build - jobs: - - job: main - pool: - type: windows - - variables: - ob_outputDirectory: '$(REPOROOT)\out' - ob_sdl_binskim_break: true - steps: - - script: | - @echo off - set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" - for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( - set vslatest="%%i" - if exist "%%i\Common7\Tools\vsdevcmd.bat" ( - set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat" - ) - if exist "%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" ( - set vscmake="%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" - ) - if exist "%%i\MSBuild\Current\Bin\amd64\msbuild.exe" ( - set vsmsbuild="%%i\MSBuild\Current\Bin\amd64\msbuild.exe" - ) - ) - - @echo vslatest %vslatest% - @echo vsdevcmd %vsdevcmd% - @echo vscmake %vscmake% - @echo vsmsbuild %vsmsbuild% - - @echo ##vso[task.setvariable variable=vslatest]%vslatest% - @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd% - @echo ##vso[task.setvariable variable=vscmake]%vscmake% - @echo ##vso[task.setvariable variable=vsmsbuild]%vsmsbuild% - displayName: 'locate vsdevcmd via vswhere' - - script: | - call $(vsdevcmd) -arch=amd64 - set PYTHONPATH= - set PYTHONHOME= - python -m pip install --upgrade pip - python -m pip install cibuildwheel numpy - python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out - displayName: Build wheels - - task: SDLNativeRules@3 - inputs: - msBuildArchitecture: amd64 - setupCommandlines: '"$(vscmake)" $(REPOROOT) -A x64 -B $(REPOROOT)\windows_out -DOCOS_BUILD_PYTHON=ON -DOCOS_ENABLE_CTEST=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo' - msBuildCommandline: '"$(vsmsbuild)" "$(REPOROOT)\windows_out\onnxruntime_extensions.sln" /p:RunCodeAnalysis=true /p:platform=x64 /p:configuration=RelWithDebInfo /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' - excludedPaths: '$(REPOROOT)\windows_out#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - - task: PostAnalysis@2 - inputs: - GdnBreakGdnToolSDLNativeRulesSeverity: Warning - GdnBreakGdnToolSDLNativeRules: true - displayName: 'Guardian Break' - - script: | - 7z x $(REPOROOT)\out\*.whl -o$(REPOROOT)\out\* - displayName: unzip the package - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: 'external_distribution' - files_to_sign: '**/**/*.pyd;**/**/*.dll' - search_root: '$(REPOROOT)\out' - displayName: 'Sign runtime DLLs' - - script: | - for /D %%i in ("out\onnxruntime_extensions*-win*") do ( - 7z u %%i.whl .\%%i\* - rmdir /s /q %%i - ) - workingDirectory: '$(REPOROOT)' - displayName: zip package + - template: /.pipelines/templates/onebranch-windows-build-stage.yml@self + parameters: + CibwEnv: ${{ parameters.ExtraEnv }} diff --git a/.pipelines/templates/onebranch-common-variables.yml b/.pipelines/templates/onebranch-common-variables.yml new file mode 100644 index 00000000..b7b4859c --- /dev/null +++ b/.pipelines/templates/onebranch-common-variables.yml @@ -0,0 +1,15 @@ +parameters: +- name: debug + displayName: 'Enable debug output' + type: boolean + default: false + +variables: + CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] + system.debug: ${{ parameters.debug }} + ENABLE_PRS_DELAYSIGN: 1 + ROOT: $(Build.SourcesDirectory) + REPOROOT: $(Build.SourcesDirectory) + OUTPUTROOT: $(REPOROOT)\out + NUGET_XMLDOC_MODE: none + DEBIAN_FRONTEND: noninteractive diff --git a/.pipelines/templates/onebranch-windows-build-stage.yml b/.pipelines/templates/onebranch-windows-build-stage.yml new file mode 100644 index 00000000..137c21f8 --- /dev/null +++ b/.pipelines/templates/onebranch-windows-build-stage.yml @@ -0,0 +1,92 @@ +parameters: +- name: CibwEnv + displayName: 'Extra env variable set to CIBW_ENVIRONMENT, in form of "A=1 B=2 C=3"' + type: string + default: '' + +stages: +- stage: Windows_Build + dependsOn: [] + jobs: + - job: main + pool: + type: windows + + variables: + CIBW_BUILD: cp3{8,9,10,11}-*amd64 + CIBW_ARCHS: AMD64 + CIBW_ENVIRONMENT: "${{ parameters.CibwEnv }}" + CIBW_BUILD_VERBOSITY: 1 + ob_outputDirectory: '$(REPOROOT)\out' + ob_sdl_binskim_break: true + + steps: + - script: | + @echo off + set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" + for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( + set vslatest="%%i" + if exist "%%i\Common7\Tools\vsdevcmd.bat" ( + set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat" + ) + if exist "%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" ( + set vscmake="%%i\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" + ) + if exist "%%i\MSBuild\Current\Bin\amd64\msbuild.exe" ( + set vsmsbuild="%%i\MSBuild\Current\Bin\amd64\msbuild.exe" + ) + ) + + @echo vslatest %vslatest% + @echo vsdevcmd %vsdevcmd% + @echo vscmake %vscmake% + @echo vsmsbuild %vsmsbuild% + + @echo ##vso[task.setvariable variable=vslatest]%vslatest% + @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd% + @echo ##vso[task.setvariable variable=vscmake]%vscmake% + @echo ##vso[task.setvariable variable=vsmsbuild]%vsmsbuild% + displayName: 'locate vsdevcmd via vswhere' + + - script: | + call $(vsdevcmd) -arch=amd64 + set PYTHONPATH= + set PYTHONHOME= + python -m pip install --upgrade pip + python -m pip install cibuildwheel numpy + python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out + displayName: Build wheels + + - task: SDLNativeRules@3 + inputs: + msBuildArchitecture: amd64 + setupCommandlines: '"$(vscmake)" $(REPOROOT) -A x64 -B $(REPOROOT)\windows_out -DOCOS_BUILD_PYTHON=ON -DOCOS_ENABLE_CTEST=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo' + msBuildCommandline: '"$(vsmsbuild)" "$(REPOROOT)\windows_out\onnxruntime_extensions.sln" /p:RunCodeAnalysis=true /p:platform=x64 /p:configuration=RelWithDebInfo /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64' + excludedPaths: '$(REPOROOT)\windows_out#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' + displayName: 'Run the PREfast SDL Native Rules for MSBuild' + + - task: PostAnalysis@2 + inputs: + GdnBreakGdnToolSDLNativeRulesSeverity: Warning + GdnBreakGdnToolSDLNativeRules: true + displayName: 'Guardian Break' + + - script: | + 7z x $(REPOROOT)\out\*.whl -o$(REPOROOT)\out\* + displayName: unzip the package + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'external_distribution' + files_to_sign: '**/**/*.pyd;**/**/*.dll' + search_root: '$(REPOROOT)\out' + displayName: 'Sign runtime DLLs' + + - script: | + for /D %%i in ("out\onnxruntime_extensions*-win*") do ( + 7z u %%i.whl .\%%i\* + rmdir /s /q %%i + ) + workingDirectory: '$(REPOROOT)' + displayName: zip package