Add initial NuGet pipeline for Windows x64 build (#406)

* initial nuget pipeline * Update nuget.yml for Azure Pipelines * update nuget.yml for extensions specific packaging TODO: add certain template yml files * added component governance template yaml * change template yaml path * remove RoslynAnalyzers * Add packDestination to nuget pack task (change from default) * fix nuspec path * Update nuget.yml for Azure Pipelines * Update nuget.yml for Azure Pipelines * Update nuget.yml for Azure Pipelines * Update 2 nuget.yml for Azure Pipelines * Update NativeNuget.nuspec * Update nuget.yml for Azure Pipelines * update nuspec * Update 3 nuget.yml for Azure Pipelines * Update 4 nuget.yml for Azure Pipelines * Update 7 nuget.yml for Azure Pipelines * Remove unnecessary nupkg file and update nuspec (#405) * Add nuget pack to build.bat and small nuget changes for demo * Temporarily adding nuget.exe to build package until we can add to CI machine * Switch back from Release to RelWithDebInfo * Remove unnecessary changes --------- Co-authored-by: Sayan Shaw <sayanshaw@microsoft.com> * Update 8 nuget.yml for Azure Pipelines * Update 9 nuget.yml for Azure Pipelines * add DLL signing * Update nuget.yml for Azure Pipelines * fix indendation * Update 11 nuget.yml for Azure Pipelines * Update 12 nuget.yml for Azure Pipelines * Update 12 nuget.yml for Azure Pipelines * Revert some unneccesary changes on nuget.yml * clean up nuget.yml and update nuspec release notes * small changes * update commit id and release notes --------- Co-authored-by: Wenbing Li <wenbingl@outlook.com> Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com> Co-authored-by: Sayan Shaw <sayanshaw@microsoft.com>
2023-04-25 17:13:13 -07:00 · 2023-04-25 17:13:13 -07:00 · 4f481d23ac
--- a/.pipelines/nuget.yml
+++ b/.pipelines/nuget.yml
@ -0,0 +1,99 @@
+stages:
+- stage: NuGet_Packaging_CPU
+  dependsOn:
+  jobs:
+  - job:
+    workspace:
+      clean: all
+    # we need to use the 2022 pool to create the nuget package with both pre-net6+Xamarin and net6 targets.
+    # VS2019 has no support for net6 and we need to use msbuild (from the VS install) to do the packing
+    pool: 'Azure-Pipelines-EO-Windows2022-aiinfra'
+    variables:
+      OrtPackageId: 'Microsoft.ML.OnnxRuntime.Extensions'
+      breakCodesignValidationInjection: true
+      ort.version: '1.14.1'
+
+    steps:
+    - checkout: self
+      submodules: true
+    - task: DownloadGitHubRelease@0
+      inputs:
+        connection: 'microsoft'
+        userRepository: 'microsoft/onnxruntime'
+        defaultVersionType: 'specificTag'
+        version: 'v$(ort.version)'
+        itemPattern: '*-win-x64-$(ort.version)*'
+        downloadPath: '$(Build.SourcesDirectory)'
+      displayName: Download the ONNXRuntime prebuilt package.
+
+    - task: ExtractFiles@1
+      inputs:
+        archiveFilePatterns: '**/*.zip'
+        destinationFolder: '$(Build.SourcesDirectory)'
+        cleanDestinationFolder: false
+        overwriteExistingFiles: true
+      displayName: Unpack ONNXRuntime package.
+
+    - script: |
+        @echo off
+        set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
+        for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
+          if exist "%%i\Common7\Tools\vsdevcmd.bat" (
+            set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"
+          )
+        )
+
+        @echo %vsdevcmd% will be used as the VC compiler
+        @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%
+      displayName: 'locate vsdevcmd via vswhere'
+
+    # Build Extensions for Windows x64 - we will add more builds to the nuget pipeline in the future
+    - script: |
+        call $(vsdevcmd)
+        call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version) -DOCOS_ENABLE_BLINGFIRE=OFF
+      displayName: build the customop library with onnxruntime
+
+    - script: |
+        cd out/Windows
+        ctest -C RelWithDebInfo --output-on-failure
+      displayName: Run C++ native tests
+
+    # We need to sign twice: the first one happens before making the package, where we sign the DLLs.
+    # The second one is after making the package, where we sign the nuget package itself.
+    - template: templates/win-esrp-dll.yml
+      parameters:
+        FolderPath: '$(Build.SourcesDirectory)\out\Windows\bin\RelWithDebInfo'
+        DisplayName: 'Sign DLL'
+        DoEsrp: 'true'
+
+    - task: NuGetToolInstaller@0
+      displayName: Use Nuget 6.2.1
+      inputs:
+        versionSpec: 6.2.1
+
+    - task: NuGetCommand@2
+      displayName: Pack nuget package with nuspec
+      inputs:
+        command: 'pack'
+        packagesToPack: '$(Build.SourcesDirectory)/nuget/NativeNuget.nuspec'
+        packDestination: '$(Build.ArtifactStagingDirectory)'
+
+    - template: templates/esrp_nuget.yml
+      parameters:
+        DisplayName: 'ESRP - sign NuGet package'
+        FolderPath: '$(Build.ArtifactStagingDirectory)'
+        DoEsrp: 'true'
+
+    - task: PublishPipelineArtifact@0
+      displayName: 'Publish Pipeline NuGet Artifact'
+      inputs:
+        artifactName: 'drop-signed-nuget-CPU'
+        targetPath: '$(Build.ArtifactStagingDirectory)'
+
+    - template: templates/component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
+
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
--- a/.pipelines/templates/component-governance-component-detection-steps.yml
+++ b/.pipelines/templates/component-governance-component-detection-steps.yml
@ -0,0 +1,14 @@
+# component detection for component governance checks
+parameters:
+- name: condition
+  type: string
+  default: 'succeeded' # could be 'ci_only', 'always', 'succeeded'
+
+steps:
+- ${{ if eq(variables['System.TeamProject'], 'Lotus') }}: 
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    condition:
+      or(or(and(eq('${{parameters.condition}}', 'ci_only'), and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Scheduled'))),
+          and(eq('${{parameters.condition}}', 'always'), always())),
+          and(eq('${{parameters.condition}}', 'succeeded'), succeeded()))
--- a/.pipelines/templates/esrp_nuget.yml
+++ b/.pipelines/templates/esrp_nuget.yml
@ -0,0 +1,31 @@
+parameters:
+  FolderPath: ''
+  DisplayName: ''
+  DoEsrp: 'false'
+
+steps:
+- ${{ if eq(parameters['DoEsrp'], 'true') }}:
+  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@2
+    displayName: ${{ parameters.DisplayName }}
+    inputs:
+      ConnectedServiceName: 'OnnxRuntime CodeSign 20190817'
+      FolderPath: ${{ parameters.FolderPath }}
+      Pattern: '*.nupkg'
+      signConfigType: inlineSignParams
+      inlineOperation: |
+       [
+           {
+               "keyCode": "CP-401405",
+               "operationSetCode": "NuGetSign",
+               "parameters": [ ],
+               "toolName": "sign",
+               "toolVersion": "1.0"
+           },
+           {
+               "keyCode": "CP-401405",
+               "operationSetCode": "NuGetVerify",
+               "parameters": [ ],
+               "toolName": "sign",
+               "toolVersion": "1.0"
+           }
+       ]
--- a/.pipelines/templates/publish-nuget.yml
+++ b/.pipelines/templates/publish-nuget.yml
@ -0,0 +1,34 @@
+parameters:
+- name: PublishingNuget
+  displayName: Publishing Nuget Packages
+  type: boolean
+  default: true
+stages:
+- stage: Publish_NuGet_Package_And_Report
+  condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
+  jobs:
+  - job:
+    workspace:
+      clean: all
+    variables:
+    - name: GDN_CODESIGN_TARGETDIRECTORY
+      value: '$(Build.BinariesDirectory)/nuget-artifact/final-package'
+    pool: 'Win-CPU-2021'
+
+    steps:
+    - checkout: self
+      submodules: false
+    - template: set-version-number-variables-step.yml
+
+    - task: DownloadPipelineArtifact@0
+      displayName: 'Download Pipeline Artifact - Signed NuGet Package'
+      inputs:
+        artifactName: 'drop-signed-nuget-CPU'
+        targetPath: $(Build.BinariesDirectory)/nuget-artifact/final-package
+
+    - template: component-governance-component-detection-steps.yml
+      parameters :
+        condition : 'succeeded'
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
--- a/.pipelines/templates/win-esrp-dll.yml
+++ b/.pipelines/templates/win-esrp-dll.yml
@ -0,0 +1,57 @@
+parameters:
+- name: DoEsrp
+  type: boolean
+  default: true
+
+- name: FolderPath
+  type: string
+  default: ''
+
+- name: DisplayName
+  type: string
+  default: ''
+
+- name: Pattern
+  type: string
+  default: '*.dll'
+
+steps:
+- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@2
+  displayName: ${{ parameters.DisplayName }}
+  condition: and(succeeded(), eq('${{ parameters.DoEsrp }}', true))
+  inputs:
+    ConnectedServiceName: 'OnnxRuntime CodeSign 20190817'
+    FolderPath: ${{ parameters.FolderPath }}
+    Pattern: ${{ parameters.Pattern }}
+    signConfigType: inlineSignParams
+    inlineOperation: |
+      [
+        {
+          "keyCode": "CP-230012",
+          "operationSetCode": "SigntoolSign",
+          "parameters": [
+            {
+              "parameterName": "OpusName",
+              "parameterValue": "Microsoft"
+            },
+            {
+              "parameterName": "OpusInfo",
+              "parameterValue": "http://www.microsoft.com"
+            },
+            {
+              "parameterName": "PageHash",
+              "parameterValue": "/NPH"
+            },
+            {
+              "parameterName": "FileDigest",
+              "parameterValue": "/fd sha256"
+            },
+            {
+              "parameterName": "TimeStamp",
+              "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+            }
+          ],
+          "toolName": "signtool.exe",
+          "toolVersion": "6.2.9304.0"
+        }
+      ]
--- a/nuget/NativeNuget.nuspec
+++ b/nuget/NativeNuget.nuspec
@ -10,10 +10,10 @@
 	<!-- TODO: Update release notes prior to 0.8.0 release with new features here -->
 	<releaseNotes>
 		General
-		1. New custom operators: Whisper, YOLO, RobertaTokenizer, ClipTokenizer, EncodeImage, DecodeImage
-		2. ORT custom operator C++ stub generation tool
-		3. Operator implementation and documentation improved.
-		4. Python (3.7 - 3.10) and ORT (1.10 above) compatible.
+		1. New custom operators: Whisper, DrawBoundingBoxes, RobertaTokenizer, ClipTokenizer, EncodeImage, DecodeImage
+		2. Optional input/output support
+		3. ORT custom operator C++ stub generation tool
+		4. Operator implementation and documentation improved.

 		Mobile
 		1. Android package: Maven
@ -26,13 +26,13 @@
 	<icon>ORT_icon_for_light_bg.png</icon>
 	<license type="file">LICENSE.txt</license>
 	<projectUrl>https://github.com/Microsoft/onnxruntime-extensions</projectUrl>
-	<repository type="git" url="https://github.com/Microsoft/onnxruntime-extensions.git"/>
+	<repository type="git" url="https://github.com/Microsoft/onnxruntime-extensions.git" commit="94142d8391c9791ec71c38336436319a2d4ac7a0"/>
 	</metadata>
 	<!-- assemble files in the /nuget directory and run nuget pack from -->
 	<files>
 		<file src="LICENSE.txt" target="LICENSE.txt" />
 		<!-- Windows, Linux, macOS via netstandard -->
-		<file src="..\out\Windows\bin\Release\ortextensions.dll" target="runtimes\win-x64\native" />
+		<file src="..\out\Windows\bin\RelWithDebInfo\ortextensions.dll" target="runtimes\win-x64\native" />
 		<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\native" />
 		<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard1.1" />
 		<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard2.0" />
@ -42,4 +42,4 @@
 		<file src="..\ThirdPartyNotices.txt" target="ThirdPartyNotices.txt" />
 		<file src="ORT_icon_for_light_bg.png" target="ORT_icon_for_light_bg.png" />
 	</files>
-</package>
+</package>