[NuGet] All platform package pipeline (#408)

* nuget ci package
* disable macos arm64 build for err

* Get the iOS xcframework build working with the split build/pack approach. (#416)

* refine build_xcframework.py
Cleanup/clarify various things
- naming of parameters and files
- consistency
Make handling of additional build args more generic
Update the artifact download dir/extract dir to more intuitive names
Update scripts
- make usage from CI pipeline clearer (e.g. don't hide directory names inside script)
- keep comments in nuspec
- remove unused args
- make additional arg handling more
Co-authored-by: Scott McKay <skottmckay@gmail.com>
Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
This commit is contained in:
JiCheng 2023-05-05 18:22:54 +08:00 коммит произвёл GitHub
Родитель eb29d225a7
Коммит 571512aca4
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
26 изменённых файлов: 1518 добавлений и 287 удалений

Просмотреть файл

@ -39,7 +39,7 @@ jobs:
python ./tools/android/build_aar.py \
--output-dir $(Build.BinariesDirectory)/android_aar \
--config $(buildConfig) \
--cmake-extra-defines OCOS_ENABLE_SELECTED_OPLIST=ON
--cmake_extra_defines OCOS_ENABLE_SELECTED_OPLIST=ON
VERSION=$(cat ./version.txt)
AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"

Просмотреть файл

@ -424,8 +424,8 @@ jobs:
- script: |
python ./tools/ios/build_xcframework.py \
--output-dir $(Build.BinariesDirectory)/xcframework_out \
--platform-arch iphonesimulator x86_64 \
--output_dir $(Build.BinariesDirectory)/xcframework_out \
--platform_arch iphonesimulator x86_64 \
--config Release
displayName: Build xcframework for iphonesimulator x86_64

Просмотреть файл

@ -2,7 +2,7 @@
parameters:
- name: IsReleaseBuild
displayName: "Is this is a release build?"
displayName: "Is this a release build?"
type: boolean
default: false
@ -38,9 +38,9 @@ jobs:
- script: |
python ./tools/ios/build_xcframework.py \
--output-dir $(Build.BinariesDirectory)/xcframework_out \
--output_dir $(Build.BinariesDirectory)/xcframework_out \
--config Release \
--cmake-extra-defines OCOS_ENABLE_SELECTED_OPLIST=ON
--cmake_extra_defines OCOS_ENABLE_SELECTED_OPLIST=ON
displayName: "Build xcframework"
- script: |

Просмотреть файл

@ -1,107 +1,33 @@
parameters:
- name: DoCompliance
displayName: Run Compliance Tasks?
type: boolean
default: true
- name: DoEsrp
displayName: Run code sign tasks? Must be true if you are doing an OnnxRuntime extensions release.
type: boolean
default: true
- name: IsReleaseBuild
displayName: Is this a release build? Set it to true if you are doing an OnnxRuntime extensions release.
type: boolean
default: false
trigger:
branches:
exclude:
- "*"
pr:
branches:
include:
- "*"
stages:
- stage: NuGet_Packaging_CPU
dependsOn:
jobs:
- job:
workspace:
clean: all
# we need to use the 2022 pool to create the nuget package with both pre-net6+Xamarin and net6 targets.
# VS2019 has no support for net6 and we need to use msbuild (from the VS install) to do the packing
pool: 'Azure-Pipelines-EO-Windows2022-aiinfra'
variables:
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Extensions'
breakCodesignValidationInjection: true
ort.version: '1.14.1'
steps:
- checkout: self
submodules: true
- task: DownloadGitHubRelease@0
inputs:
connection: 'microsoft'
userRepository: 'microsoft/onnxruntime'
defaultVersionType: 'specificTag'
version: 'v$(ort.version)'
itemPattern: '*-win-x64-$(ort.version)*'
downloadPath: '$(Build.SourcesDirectory)'
displayName: Download the ONNXRuntime prebuilt package.
- task: ExtractFiles@1
inputs:
archiveFilePatterns: '**/*.zip'
destinationFolder: '$(Build.SourcesDirectory)'
cleanDestinationFolder: false
overwriteExistingFiles: true
displayName: Unpack ONNXRuntime package.
- script: |
@echo off
set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
if exist "%%i\Common7\Tools\vsdevcmd.bat" (
set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"
)
)
@echo %vsdevcmd% will be used as the VC compiler
@echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%
displayName: 'locate vsdevcmd via vswhere'
# Build Extensions for Windows x64 - we will add more builds to the nuget pipeline in the future
- script: |
call $(vsdevcmd)
call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version) -DOCOS_ENABLE_BLINGFIRE=OFF
displayName: build the customop library with onnxruntime
- script: |
cd out/Windows
ctest -C RelWithDebInfo --output-on-failure
displayName: Run C++ native tests
# We need to sign twice: the first one happens before making the package, where we sign the DLLs.
# The second one is after making the package, where we sign the nuget package itself.
- template: templates/win-esrp-dll.yml
parameters:
FolderPath: '$(Build.SourcesDirectory)\out\Windows\bin\RelWithDebInfo'
DisplayName: 'Sign DLL'
DoEsrp: 'true'
- task: PowerShell@2
displayName: Replace commit id token in nuspec
inputs:
targetType: 'inline'
script: |
(Get-Content NativeNuget.nuspec) -replace 'COMMIT_ID', '$(Build.SourceVersion)' | Set-Content NativeNuget.nuspec
workingDirectory: '$(Build.SourcesDirectory)/nuget'
- task: NuGetToolInstaller@0
displayName: Use Nuget 6.2.1
inputs:
versionSpec: 6.2.1
- task: NuGetCommand@2
displayName: Pack nuget package with nuspec
inputs:
command: 'pack'
packagesToPack: '$(Build.SourcesDirectory)/nuget/NativeNuget.nuspec'
packDestination: '$(Build.ArtifactStagingDirectory)'
- template: templates/esrp_nuget.yml
parameters:
DisplayName: 'ESRP - sign NuGet package'
FolderPath: '$(Build.ArtifactStagingDirectory)'
DoEsrp: 'true'
- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline NuGet Artifact'
inputs:
artifactName: 'drop-signed-nuget-CPU'
targetPath: '$(Build.ArtifactStagingDirectory)'
- template: templates/component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- template: templates/build-package-for-nuget.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Extensions'

Просмотреть файл

@ -0,0 +1,72 @@
# packaging pipeline for onnxruntime-extensions Android AAR in nuget package
parameters:
- name: AndroidABI
displayName: android abi,[arm64-v8a, x86_64, X86, armeabi-v7a]
values:
- 'x86_64'
- 'x86'
- 'arm64-v8a'
- 'armeabi-v7a'
type: string
- name: JobSuffix
displayName: android abi has dash `-`, which can't be part of job name.
type: string
- name: AdditionalBuildFlags
displayName: Additional build flags for /tools/android/build_aar.py
type: string
jobs:
- job: Android_C_API_Packaging_${{ parameters.JobSuffix }}
pool:
vmImage: "macOS-12"
timeoutInMinutes: 120
variables:
buildConfig: Release
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: "3.9"
addToPath: true
architecture: "x64"
displayName: "Use Python 3.9"
- task: JavaToolInstaller@0
displayName: Use jdk 11
inputs:
versionSpec: "11"
jdkArchitectureOption: "x64"
jdkSourceOption: "PreInstalled"
- script: brew install coreutils ninja
displayName: Install coreutils and ninja
- task: Bash@3
displayName: 'Build android shared library'
inputs:
targetType: 'inline'
script: |
set -e -x
python ./tools/android/build_aar.py \
--output-dir $(Build.BinariesDirectory)/android_so \
--mode build_so_only \
--abi ${{ parameters.AndroidABI }} \
--config $(buildConfig) \
${{parameters.AdditionalBuildFlags}}
workingDirectory: '$(Build.SourcesDirectory)'
- bash: |
set -e -x
SO_DIR=$(Build.BinariesDirectory)/android_so
ARTIFACTS_STAGING_DIR="$(Build.ArtifactStagingDirectory)"
mkdir ${ARTIFACTS_STAGING_DIR}/intermediates
cp -r ${SO_DIR}/intermediates/jnilibs ${ARTIFACTS_STAGING_DIR}/intermediates/
displayName: "Assemble artifacts"
- publish: "$(Build.ArtifactStagingDirectory)"
artifact: onnxruntime-extensions-android-${{ parameters.AndroidABI }}
displayName: "Publish artifacts"

Просмотреть файл

@ -0,0 +1,108 @@
# packaging pipeline for onnxruntime-extensions Android AAR package
parameters:
- name: IsReleaseBuild
displayName: "Is this a release build?"
type: boolean
default: false
- name: AdditionalBuildFlags
displayName: Additional build flags for /tools/android/build_aar.py
type: string
stages:
- stage: Android_Java_API_AAR_Packaging_Full
dependsOn: []
jobs:
- template: android-shared-lib-build.yml
parameters:
AndroidABI: 'x86_64'
JobSuffix: 'x86_64'
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- template: android-shared-lib-build.yml
parameters:
AndroidABI: 'x86'
JobSuffix: 'x86'
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- template: android-shared-lib-build.yml
parameters:
AndroidABI: 'armeabi-v7a'
JobSuffix: 'armeabi_v7a'
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- template: android-shared-lib-build.yml
parameters:
AndroidABI: 'arm64-v8a'
JobSuffix: 'arm64_v8a'
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- job: Android_Java_API_AAR_Package_Publish_All
dependsOn:
- Android_C_API_Packaging_x86_64
- Android_C_API_Packaging_x86
- Android_C_API_Packaging_armeabi_v7a
- Android_C_API_Packaging_arm64_v8a
pool:
vmImage: 'macOS-12'
timeoutInMinutes: 120
variables:
buildConfig: Release
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: "3.9"
addToPath: true
architecture: "x64"
displayName: "Use Python 3.9"
- task: JavaToolInstaller@0
displayName: Use jdk 11
inputs:
versionSpec: "11"
jdkArchitectureOption: "x64"
jdkSourceOption: "PreInstalled"
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-android-x86_64'
targetPath: '$(Build.BinariesDirectory)/android_aar'
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-android-x86'
targetPath: '$(Build.BinariesDirectory)/android_aar'
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-android-armeabi-v7a'
targetPath: '$(Build.BinariesDirectory)/android_aar'
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-android-arm64-v8a'
targetPath: '$(Build.BinariesDirectory)/android_aar'
- task: Bash@3
displayName: 'build android aar'
inputs:
targetType: 'inline'
script: |
set -e -x
python ./tools/android/build_aar.py \
--output-dir $(Build.BinariesDirectory)/android_aar \
--mode pack_aar_only \
--config $(buildConfig)
VERSION=$(cat ./version.txt)
AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
workingDirectory: '$(Build.SourcesDirectory)'
- bash: |
set -e -x
AAR_DIR=$(dirname "${ORT_EXTENSIONS_AAR_PATH}")
ARTIFACTS_STAGING_DIR="$(Build.ArtifactStagingDirectory)"
cp ${ORT_EXTENSIONS_AAR_PATH} ${ARTIFACTS_STAGING_DIR}
cp ${AAR_DIR}/*-javadoc.jar ${ARTIFACTS_STAGING_DIR}
cp ${AAR_DIR}/*-sources.jar ${ARTIFACTS_STAGING_DIR}
cp ${AAR_DIR}/*.pom ${ARTIFACTS_STAGING_DIR}
displayName: "Assemble artifacts"
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifact: 'onnxruntime-extensions-android-aar'
condition: 'succeededOrFailed()'

Просмотреть файл

@ -0,0 +1,146 @@
# packaging pipeline for iOS CocoaPods package
parameters:
- name: IsReleaseBuild
displayName: "Is this a release build?"
type: boolean
default: false
- name: AdditionalBuildFlags
displayName: Additional build flags for /tools/ios/build_xcframework.py
type: string
jobs:
- template: ios-framework-build.yml
parameters:
Platform: 'iphoneos'
IosArch: 'arm64'
IsReleaseBuild: ${{parameters.IsReleaseBuild}}
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- template: ios-framework-build.yml
parameters:
Platform: 'iphonesimulator'
IosArch: 'x86_64'
IsReleaseBuild: ${{parameters.IsReleaseBuild}}
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- template: ios-framework-build.yml
parameters:
Platform: 'iphonesimulator'
IosArch: 'arm64'
IsReleaseBuild: ${{parameters.IsReleaseBuild}}
AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
- job: IosPackaging
condition: not(or(failed(), canceled()))
displayName: "iOS Packaging"
dependsOn:
- IOS_C_API_Packaging_iphoneos_arm64
- IOS_C_API_Packaging_iphonesimulator_arm64
- IOS_C_API_Packaging_iphonesimulator_x86_64
pool:
vmImage: "macOS-12"
timeoutInMinutes: 120
steps:
- checkout: self
submodules: true
- task: UsePythonVersion@0
inputs:
versionSpec: "3.9"
addToPath: true
architecture: "x64"
# iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
- script: |
python -m pip install cmake==3.25.0
displayName: "Install CMake 3.25.0"
- template: set-package-version-variable-step.yml
parameters:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-ios-iphoneos-arm64-lib'
targetPath: '$(Build.BinariesDirectory)/'
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-ios-iphonesimulator-arm64-lib'
targetPath: '$(Build.BinariesDirectory)/'
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-ios-iphonesimulator-x86_64-lib'
targetPath: '$(Build.BinariesDirectory)/'
- script: |
set -e -x
ls
if ! ls onnxruntime-extensions-ios*.tgz &> /dev/null; then
echo "no tgz files found. It's not distributed building, skip!"
exit 0
fi
for tgz_lib in `ls onnxruntime-extensions-ios*.tgz`
do
echo "decompress $tgz_lib"
tar -xvzf $tgz_lib
done
ls
ls xcframework_out/intermediates/*
workingDirectory: $(Build.BinariesDirectory)
displayName: "Extract file from distributed build"
- script: |
python ./tools/ios/build_xcframework.py \
--output_dir $(Build.BinariesDirectory)/xcframework_out \
--config Release \
--mode pack_xcframework_only
displayName: "Pack xcframework for distributed building"
- script: |
python ./tools/ios/assemble_pod_package.py \
--staging-dir $(Build.BinariesDirectory)/pod_staging \
--xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
--pod-version ${ORT_EXTENSIONS_POD_VERSION}
displayName: "Assemble pod"
- script: |
ls -lR $(Build.BinariesDirectory)/pod_staging
pod lib lint
displayName: "Lint pod"
workingDirectory: $(Build.BinariesDirectory)/pod_staging
- script: |
ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
pod install
displayName: "Install pods for OrtExtensionsUsage"
workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
- script: |
set -e -x
POD_STAGING_DIR="$(Build.BinariesDirectory)/pod_staging"
ARTIFACTS_STAGING_DIR="$(Build.ArtifactStagingDirectory)"
POD_NAME="onnxruntime-extensions-c"
POD_ARCHIVE_BASENAME="pod-archive-${POD_NAME}-${ORT_EXTENSIONS_POD_VERSION}.zip"
PODSPEC_BASENAME="${POD_NAME}.podspec"
pushd ${POD_STAGING_DIR}
# assemble the files in the artifacts staging directory
zip -r ${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME} * --exclude ${PODSPEC_BASENAME}
cp ${PODSPEC_BASENAME} ${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME}
popd
displayName: "Assemble artifacts"
- publish: "$(Build.ArtifactStagingDirectory)"
artifact: onnxruntime-extensions-ios_package
displayName: "Publish artifacts"

Просмотреть файл

@ -0,0 +1,93 @@
# This file contains the ADO job that build so-file on Linux
parameters:
- name: IsReleaseBuild
displayName: "Is this a release build?"
type: boolean
default: false
- name: OrtExtensionsArch
type: string
- name: PoolName
type: string
default: 'aiinfra-Linux-CPU'
- name: OrtExtensionsCFlags
type: string
- name: OrtExtensionsCXXFlags
type: string
- name: AdditionalBuildFlags
displayName: Additional build flags for build_lib.sh
type: string
jobs:
- job: Linux_C_API_Packaging_CPU_${{parameters.OrtExtensionsArch}}
workspace:
clean: all
timeoutInMinutes: 210
pool: ${{parameters.PoolName}}
steps:
- checkout: self
clean: true
submodules: none
# Currently we can only run tests on x64 as the arm64 tests have a failure
# https://github.com/microsoft/onnxruntime-extensions/issues/417
- ${{ if eq(parameters.OrtExtensionsArch, 'x64') }}:
- bash: |
export CFLAGS="${{parameters.OrtExtensionsCFlags}}"
export CXXFLAGS="${{parameters.OrtExtensionsCXXFlags}}"
./build_lib.sh --build_dir $(Build.BinariesDirectory)/out/ --config RelWithDebInfo --parallel --enable_cxx_tests ${{parameters.AdditionalBuildFlags}}
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'build onnxruntime-extensions and run tests'
- ${{ else }}:
- bash: |
export CFLAGS="${{parameters.OrtExtensionsCFlags}}"
export CXXFLAGS="${{parameters.OrtExtensionsCXXFlags}}"
./build_lib.sh --build_dir $(Build.BinariesDirectory)/out/ --config RelWithDebInfo --parallel ${{parameters.AdditionalBuildFlags}}
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'build onnxruntime-extensions'
- template: set-package-version-variable-step.yml
parameters:
IsReleaseBuild: true
PackageVersionVariableName: ORT_EXTENSIONS_VERSION
- bash: |
set -e -x
lib_target=$(Build.BinariesDirectory)/onnxruntime-extensions-linux-${{parameters.OrtExtensionsArch}}-$(ORT_EXTENSIONS_VERSION)/lib
mkdir -p $lib_target
ls -l out/RelWithDebInfo/lib
cp out/RelWithDebInfo/lib/libortextensions.so $lib_target
cd $lib_target
objcopy --only-keep-debug libortextensions.so libortextensions.so.dbg
objcopy --strip-debug libortextensions.so
objcopy --add-gnu-debuglink=libortextensions.so.dbg libortextensions.so
displayName: create and copy artifacts.
workingDirectory: $(Build.BinariesDirectory)
- task: ArchiveFiles@2
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)/onnxruntime-extensions-linux-${{parameters.OrtExtensionsArch}}-$(ORT_EXTENSIONS_VERSION)'
includeRootFolder: true
archiveType: 'tar' # Options: zip, 7z, tar, wim
tarCompression: 'gz'
archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-extensions-linux-${{parameters.OrtExtensionsArch}}.tgz'
replaceExistingArchive: true
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifactName: 'onnxruntime-extensions-linux-${{parameters.OrtExtensionsArch}}'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

Просмотреть файл

@ -0,0 +1,54 @@
parameters:
- name: AdditionalBuildFlags
displayName: Additional build flags for build.sh
type: string
- name: IsReleaseBuild
displayName: "Is this a release build?"
type: boolean
default: false
stages:
- stage: MacOS_C_API_Packaging_CPU
dependsOn: []
jobs:
- template: mac-shared-lib-build.yml
parameters:
MacosArch: 'x86_64'
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }}
# error _png_do_read_transformations in liblibpng.a(pngrtran.c.o) not found for architecture arm64
# solution https://github.com/davisking/dlib/pull/2664/files
#- template: mac_shared_lib_build.yml
# parameters:
# MacosArch: 'arm64'
# AdditionalBuildFlags: "${{ parameters.AdditionalBuildFlags }} CMAKE_OSX_ARCHITECTURES=arm64"
#- template: mac_shared_lib_build.yml
# parameters:
# MacosArch: 'universal2'
# AdditionalBuildFlags: "${{ parameters.AdditionalBuildFlags }} CMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\""
- job: MacOS_C_API_Package_Publish_All
dependsOn:
- MacOS_C_API_Packaging_CPU_x86_64
#- MacOS_C_API_Packaging_CPU_arm64
#- MacOS_C_API_Packaging_CPU_universal2
pool:
vmImage: 'macOS-12'
steps:
- task: DownloadPipelineArtifact@2
inputs:
artifact: 'onnxruntime-extensions-osx-x86_64'
targetPath: '$(Build.ArtifactStagingDirectory)'
#- task: DownloadPipelineArtifact@2
# inputs:
# artifact: 'onnxruntime-extensions-osx-arm64'
# targetPath: '$(Build.ArtifactStagingDirectory)'
#- task: DownloadPipelineArtifact@2
# inputs:
# artifact: 'onnxruntime-extensions-osx-universal2'
# targetPath: '$(Build.ArtifactStagingDirectory)'
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifact: 'onnxruntime-extensions-osx'
condition: 'succeededOrFailed()'

Просмотреть файл

@ -0,0 +1,263 @@
parameters:
- name: DoCompliance
displayName: Run Compliance Tasks?
type: boolean
default: true
- name: DoEsrp
displayName: Run code sign tasks? Must be true if you are doing an OnnxRuntime extensions release.
type: boolean
default: false
- name: IsReleaseBuild
displayName: Is a release build?
type: boolean
default: false
- name: OrtNugetPackageId
displayName: Package name for nuget
type: string
default: 'Microsoft.ML.OnnxRuntime.Extensions'
- name: OperatorSelectionFlags
displayName: Flags to select operators included. Default is to disable OCOS_ENABLE_CV2 and OCOS_ENABLE_BLINGFIRE
type: string
default: '--cmake_extra_defines OCOS_ENABLE_CV2=OFF OCOS_ENABLE_BLINGFIRE=OFF'
- name: AdditionalBuildFlags
displayName: Additional build flags.
type: string
default: ''
stages:
# MacOS_C_API_Packaging_CPU
- template: build-package-for-macosx.yml
parameters:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- stage: Linux_C_API_Packaging_CPU
dependsOn: []
jobs:
- template: build-package-for-linux.yml
parameters:
OrtExtensionsArch: 'x64'
PoolName: 'aiinfra-Linux-CPU'
IsReleaseBuild: ${{parameters.IsReleaseBuild}}
OrtExtensionsCFlags: ''
OrtExtensionsCXXFlags: ''
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- template: build-package-for-linux.yml
parameters:
OrtExtensionsArch: 'aarch64'
PoolName: 'aiinfra-linux-ARM64-CPU-2019'
IsReleaseBuild: ${{parameters.IsReleaseBuild}}
OrtExtensionsCFlags: ''
OrtExtensionsCXXFlags: ''
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- stage: Windows_C_API_Packaging_CPU
dependsOn: []
jobs:
- template: build-package-for-windows.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
StageNameSuffix: CPU_x86
# Win32
BuildPlatform: x86
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- template: build-package-for-windows.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
StageNameSuffix: CPU_arm
BuildPlatform: arm
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- template: build-package-for-windows.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
StageNameSuffix: CPU_arm64
BuildPlatform: arm64
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- template: build-package-for-windows.yml
parameters:
DoCompliance: ${{ parameters.DoCompliance }}
DoEsrp: ${{ parameters.DoEsrp }}
StageNameSuffix: CPU_x64
BuildPlatform: 'x64'
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
#Android_Java_API_AAR_Packaging_Full
- template: build-package-for-android-aar.yml
parameters:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- stage: iOS_Full_xcframework
dependsOn: []
jobs:
- template: build-package-for-ios-cocoapods.yml
parameters:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
AdditionalBuildFlags: ${{parameters.OperatorSelectionFlags}} ${{parameters.AdditionalBuildFlags}}
- stage: NuGet_Packaging_CPU
dependsOn:
- Linux_C_API_Packaging_CPU
- Windows_C_API_Packaging_CPU
- MacOS_C_API_Packaging_CPU
- Android_Java_API_AAR_Packaging_Full
- iOS_Full_xcframework
condition: succeeded()
jobs:
- job: Nuget_Packaging
workspace:
clean: all
# we need to use the 2022 pool to create the nuget package with both pre-net6+Xamarin and net6 targets.
# VS2019 has no support for net6 and we need to use msbuild (from the VS install) to do the packing
pool: 'Azure-Pipelines-EO-Windows2022-aiinfra'
variables:
OrtPackageId: ${{ parameters.OrtNugetPackageId }}
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
steps:
- checkout: self
submodules: true
- task: UsePythonVersion@0
inputs:
versionSpec: "3.9"
addToPath: true
architecture: "x64"
displayName: "Use Python 3.9"
- task: DownloadPipelineArtifact@0
displayName: 'Download win-x64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-win-x64'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download win-x86 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-win-x86'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download win-arm64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-win-arm64'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download win-arm Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-win-arm'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download osx-all Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-osx'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@2
displayName: 'Download iOS Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-ios_package'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@2
displayName: 'Download android-full-aar Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-android-aar'
patterns: '**/*.aar'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download linux-x64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-linux-x64'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- task: DownloadPipelineArtifact@0
displayName: 'Download linux-aarch64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-extensions-linux-aarch64'
targetPath: '$(Build.BinariesDirectory)/artifact-downloads'
- script: |
dir
workingDirectory: '$(Build.BinariesDirectory)/artifact-downloads'
displayName: 'List artifacts'
# Reconstruct the build dir
- task: PowerShell@2
displayName: 'Extract native libraries to artifact-downloads for addition to nuget native package'
inputs:
targetType: filePath
filePath: $(Build.SourcesDirectory)\tools\ci_build\extract_nuget_files.ps1
arguments: $(Build.BinariesDirectory)/artifact-downloads $(Build.BinariesDirectory)\nuget-artifacts
- task: NuGetToolInstaller@0
displayName: Use Nuget 6.2.1
inputs:
versionSpec: 6.2.1
- task: PowerShell@2
displayName: 'Updating NuSpec'
inputs:
targetType: 'inline'
script: |
$OrtExtVersion=(cat ./version.txt)
python $(Build.SourcesDirectory)\tools\ci_build\update_nuspec_for_native_nuget.py `
--package_version $OrtExtVersion `
--commit_id $(Build.SourceVersion) `
--is_release_build ${{ parameters.IsReleaseBuild }}
cat $(Build.SourcesDirectory)\nuget\NativeNuget.nuspec
workingDirectory: '$(Build.SourcesDirectory)'
- task: PowerShell@2
displayName: 'Setup for packing'
inputs:
# copy nuget folder to binaries dir so we can refer to the assembled artifacts in
# $(Build.BinariesDirectory)/nuget-artifacts using a deterministic relative path in the nuspec.
targetType: 'inline'
script: |
cp -r $(Build.SourcesDirectory)/nuget $(Build.BinariesDirectory)
cp $(Build.SourcesDirectory)/ThirdPartyNotices.txt $(Build.BinariesDirectory)
workingDirectory: '$(Build.SourcesDirectory)'
- task: NuGetCommand@2
displayName: Packing onnxruntime extensions NuGet-package
inputs:
command: 'pack'
packagesToPack: '$(Build.BinariesDirectory)\nuget\NativeNuget.nuspec'
packDestination: $(Build.ArtifactStagingDirectory)
- template: esrp_nuget.yml
parameters:
DisplayName: 'ESRP - sign NuGet package'
FolderPath: '$(Build.ArtifactStagingDirectory)'
DoEsrp: 'true'
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifactName: 'drop-signed-nuget-CPU'
- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

Просмотреть файл

@ -0,0 +1,109 @@
parameters:
- name: DoCompliance
displayName: Run Compliance Tasks?
type: boolean
default: true
- name: DoEsrp
displayName: Run code sign tasks? Must be true if you are doing an OnnxRuntime extensions release.
type: boolean
default: false
- name: BuildPlatform
type: string
values:
- 'x86'
- 'x64'
- 'arm64'
- 'arm'
- name: StageNameSuffix
displayName: job name for nuget
type: string
default: ''
- name: BuildPool
type: string
default: 'Win-CPU-2021'
- name: AdditionalBuildFlags
displayName: Additional build flags for build_lib.bat
type: string
jobs:
- job: Windows_Packaging_${{ parameters.StageNameSuffix }}
workspace:
clean: all
pool: ${{ parameters.BuildPool }}
timeoutInMinutes: 300
steps:
- checkout: self
clean: true
submodules: none
# Currently we can only run tests on x64. x86 tests faile. arm/arm64 tests can't be run as the build machine is amd64
# TODO: Fix these issues so the nuget build is properly tested.
- ${{ if eq(parameters.BuildPlatform, 'x64') }}:
- task: BatchScript@1
displayName: 'build onnxruntime-extensions and run tests'
inputs:
filename: '.\build_lib.bat'
arguments: '--${{parameters.BuildPlatform}} --build_dir $(Build.BinariesDirectory)/out --config RelWithDebInfo --cmake_generator "Visual Studio 16 2019" --enable_cxx_tests ${{parameters.AdditionalBuildFlags}}'
modifyEnvironment: true
workingFolder: $(Build.SourcesDirectory)
- ${{ else }}:
- task: BatchScript@1
displayName: 'build onnxruntime-extensions'
inputs:
filename: '.\build_lib.bat'
arguments: '--${{parameters.BuildPlatform}} --build_dir $(Build.BinariesDirectory)/out --config RelWithDebInfo --cmake_generator "Visual Studio 16 2019" ${{parameters.AdditionalBuildFlags}}'
modifyEnvironment: true
workingFolder: $(Build.SourcesDirectory)
- script: |
dir $(Build.BinariesDirectory)\out\RelWithDebInfo\lib\RelWithDebInfo
dir $(Build.BinariesDirectory)\out\RelWithDebInfo\bin\RelWithDebInfo
displayName: 'List built DLLs'
workingDirectory: $(Build.BinariesDirectory)
- task: PowerShell@2
displayName: 'Set version'
inputs:
targetType: 'inline'
script: |
$_OrtExtVersion=(cat version.txt)
echo "##vso[task.setvariable variable=OrtExtVersion;]$_OrtExtVersion"
workingDirectory: '$(Build.SourcesDirectory)'
- task: PowerShell@2
displayName: 'Copy and Create artifacts'
inputs:
targetType: 'inline'
script: |
$target_lib_path="$(Build.BinariesDirectory)/onnxruntime-extensions-win-${{parameters.BuildPlatform}}-$(OrtExtVersion)/lib"
New-Item $target_lib_path -ItemType Directory
cp bin/RelWithDebInfo/ortextensions.* $target_lib_path
cp lib/RelWithDebInfo/ortextensions.* $target_lib_path
workingDirectory: '$(Build.BinariesDirectory)/out/RelWithDebInfo'
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)/onnxruntime-extensions-win-${{parameters.BuildPlatform}}-$(OrtExtVersion)'
DisplayName: 'Sign DLL'
DoEsrp: 'true'
- task: ArchiveFiles@2
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)/onnxruntime-extensions-win-${{parameters.BuildPlatform}}-$(OrtExtVersion)'
includeRootFolder: true
archiveType: 'tar' # Options: zip, 7z, tar, wim
tarCompression: 'gz'
archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-extensions-win-${{parameters.BuildPlatform}}.tgz'
replaceExistingArchive: true
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifactName: 'onnxruntime-extensions-win-${{parameters.BuildPlatform}}'

Просмотреть файл

@ -0,0 +1,66 @@
# packaging pipeline for onnxruntime-extensions ios cocoapods package
parameters:
- name: Platform
displayName: ios Platform
type: string
- name: IosArch
displayName: IosArch
type: string
- name: IsReleaseBuild
displayName: "Is this a release build?"
type: boolean
- name: AdditionalBuildFlags
displayName: Additional build flags for /tools/ios/build_xcframework.py
type: string
jobs:
- job: IOS_C_API_Packaging_${{ parameters.Platform }}_${{ parameters.IosArch }}
condition: succeeded()
pool:
vmImage: "macOS-12"
timeoutInMinutes: 120
variables:
buildConfig: Release
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: "3.9"
addToPath: true
architecture: "x64"
# iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
- script: |
python -m pip install cmake==3.25.0
displayName: "Install CMake 3.25.0"
- template: set-package-version-variable-step.yml
parameters:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION
- script: |
python ./tools/ios/build_xcframework.py \
--output_dir $(Build.BinariesDirectory)/xcframework_out \
--config $(buildConfig) \
--platform_arch ${{ parameters.Platform }} ${{ parameters.IosArch }} \
--mode build_platform_arch_frameworks_only \
${{parameters.AdditionalBuildFlags}}
displayName: "Build xcframework"
- bash: |
set -e -x
package_name=onnxruntime-extensions-ios-${{ parameters.Platform }}-${{ parameters.IosArch }}-lib.tgz
target_folder=xcframework_out/intermediates/${{ parameters.Platform }}/${{ parameters.IosArch }}
framework_info=${target_folder}/framework_info.json
tar -cvzf $(Build.ArtifactStagingDirectory)/${package_name} -C $(Build.BinariesDirectory)/ ${target_folder}/static_framework ${framework_info}
displayName: "Assemble artifacts"
- publish: "$(Build.ArtifactStagingDirectory)"
artifact: onnxruntime-extensions-ios-${{ parameters.Platform }}-${{ parameters.IosArch }}-lib
displayName: "Publish artifacts"

Просмотреть файл

@ -0,0 +1,70 @@
parameters:
- name: MacosArch
type: string
values:
- 'x86_64'
- 'arm64'
- 'universal2'
default: 'x86_64'
- name: AdditionalBuildFlags
displayName: Additional build flags for build_lib.sh
type: string
jobs:
- job: MacOS_C_API_Packaging_CPU_${{ parameters.MacosArch }}
workspace:
clean: all
variables:
MACOSX_DEPLOYMENT_TARGET: '10.14'
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
pool:
vmImage: 'macOS-12'
timeoutInMinutes: 300
steps:
- checkout: self
clean: true
submodules: none
- task: NodeTool@0
inputs:
versionSpec: '16.x'
- template: set-package-version-variable-step.yml
parameters:
IsReleaseBuild: true
PackageVersionVariableName: ORT_EXTENSIONS_VERSION
# Can't run tests on macos due to build error
# https://github.com/microsoft/onnxruntime-extensions/issues/418
# Add --enable_cxx_tests to the build_lib.sh command line when fixed
- script: |
set -e -x
./build_lib.sh --build_dir $(Build.BinariesDirectory)/out/ --config RelWithDebInfo --parallel ${{parameters.AdditionalBuildFlags}}
displayName: 'Build ${{ parameters.MacosArch }}'
- task: Bash@3
displayName: 'Copy build artifacts for zipping'
inputs:
targetType: 'inline'
# TODO: Save debug symbols instead of just stripping
script: |
set -e -x
target_lib_path=onnxruntime-extensions-osx-${{ parameters.MacosArch }}-$(ORT_EXTENSIONS_VERSION)/lib
mkdir -p $target_lib_path
cp out/RelWithDebInfo/lib/libortextensions.dylib ${target_lib_path}/
strip -S ${target_lib_path}/libortextensions.dylib
workingDirectory: $(Build.BinariesDirectory)
- task: ArchiveFiles@2
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)/onnxruntime-extensions-osx-${{ parameters.MacosArch }}-$(ORT_EXTENSIONS_VERSION)'
includeRootFolder: true
archiveType: 'tar' # Options: zip, 7z, tar, wim
tarCompression: 'gz'
archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-extensions-osx-${{ parameters.MacosArch }}.tgz'
replaceExistingArchive: true
- publish: '$(Build.ArtifactStagingDirectory)'
artifact: 'onnxruntime-extensions-osx-${{ parameters.MacosArch }}'

Просмотреть файл

@ -5,16 +5,29 @@ elseif(ONNXRUNTIME_PKG_DIR)
set(ONNXRUNTIME_INCLUDE_DIR ${ONNXRUNTIME_PKG_DIR}/include)
set(ONNXRUNTIME_LIB_DIR ${ONNXRUNTIME_PKG_DIR}/lib)
else()
message(STATUS "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
message(STATUS "CMAKE_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}")
# default to 1.11.1 if not specified
set(ONNXRUNTIME_VER "1.11.1" CACHE STRING "ONNX Runtime version")
if(CMAKE_HOST_APPLE)
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-osx-universal2-${ONNXRUNTIME_VER}.tgz")
elseif(CMAKE_HOST_WIN32)
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-x64-${ONNXRUNTIME_VER}.zip")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-arm64-${ONNXRUNTIME_VER}.zip")
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64")
if (CMAKE_GENERATOR_PLATFORM STREQUAL "Win32")
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-x86-${ONNXRUNTIME_VER}.zip")
else()
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-x64-${ONNXRUNTIME_VER}.zip")
endif()
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64")
if (CMAKE_GENERATOR_PLATFORM STREQUAL "ARM")
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-arm-${ONNXRUNTIME_VER}.zip")
else()
set(ONNXRUNTIME_URL "v${ONNXRUNTIME_VER}/onnxruntime-win-arm64-${ONNXRUNTIME_VER}.zip")
endif()
else()
message(FATAL_ERROR "Unexpected CMAKE_SYSTEM_PROCESSOR of ${CMAKE_SYSTEM_PROCESSOR}.")
endif()
else()
# Linux or other, using Linux package to retrieve the headers

Просмотреть файл

@ -1,47 +1,72 @@
<?xml version="1.0"?>
<!-- ORT Extensions nuspec -->
<?xml version='1.0' encoding='utf-8'?>
<package>
<metadata>
<id>Microsoft.ML.OnnxRuntime.Extensions</id>
<version>0.8.0-alpha</version>
<authors>Microsoft</authors>
<owners>Microsoft</owners>
<description>ONNX Runtime Extensions NuGet Package</description>
<!-- TODO: Update release notes prior to 0.8.0 release with new features here -->
<releaseNotes>
General
1. New custom operators: Whisper, DrawBoundingBoxes, RobertaTokenizer, ClipTokenizer, EncodeImage, DecodeImage
2. Optional input/output support
3. ORT custom operator C++ stub generation tool
4. Operator implementation and documentation improved.
<metadata>
<id>Microsoft.ML.OnnxRuntime.Extensions</id>
<version>0.8.0-alpha</version>
<authors>Microsoft</authors>
<owners>Microsoft</owners>
<description>ONNX Runtime Extensions NuGet Package</description>
<releaseNotes>
General
1. New custom operators: Whisper, DrawBoundingBoxes, RobertaTokenizer, ClipTokenizer, EncodeImage, DecodeImage
2. Optional input/output support
3. ORT custom operator C++ stub generation tool
4. Operator implementation and documentation improved.
Mobile
1. Android package: Maven
2. iOS package: CocoaPods
3. PrePostProcessor tool for mobile model
4. Super-resolution model pre- / post- processing end-to-end examples
</releaseNotes>
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
<tags>ONNX ONNXRuntime AI Machine Learning</tags>
<icon>ORT_icon_for_light_bg.png</icon>
<license type="file">LICENSE.txt</license>
<readme>README.md</readme>
<projectUrl>https://github.com/Microsoft/onnxruntime-extensions</projectUrl>
<repository type="git" url="https://github.com/Microsoft/onnxruntime-extensions.git" commit="COMMIT_ID"/>
</metadata>
<!-- assemble files in the /nuget directory and run nuget pack from -->
<files>
<file src="LICENSE.txt" target="LICENSE.txt" />
<file src="README.md" target="README.md" />
<!-- Windows, Linux, macOS via netstandard -->
<file src="..\out\Windows\bin\RelWithDebInfo\ortextensions.dll" target="runtimes\win-x64\native" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\native" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard1.1" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard2.0" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\native" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\netstandard1.1" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\netstandard2.0" />
<file src="..\ThirdPartyNotices.txt" target="ThirdPartyNotices.txt" />
<file src="ORT_icon_for_light_bg.png" target="ORT_icon_for_light_bg.png" />
</files>
Mobile
1. Android package: Maven
2. iOS package: CocoaPods
3. PrePostProcessor tool for mobile model
4. Super-resolution model pre- / post- processing end-to-end examples
</releaseNotes>
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
<tags>ONNX ONNXRuntime AI Machine Learning</tags>
<icon>ORT_icon_for_light_bg.png</icon>
<license type="file">LICENSE.txt</license>
<readme>README.md</readme>
<projectUrl>https://github.com/Microsoft/onnxruntime-extensions</projectUrl>
<repository type="git" url="https://github.com/Microsoft/onnxruntime-extensions.git" commit="COMMIT_ID" />
</metadata>
<files>
<file src="LICENSE.txt" target="LICENSE.txt" />
<file src="README.md" target="README.md" />
<file src="ORT_icon_for_light_bg.png" target="ORT_icon_for_light_bg.png" />
<file src="..\ThirdPartyNotices.txt" target="ThirdPartyNotices.txt" />
<!--
NOTE: If you wish to create a local nuget package for testing, update the relevant src paths below
to point to the local build you wish to test, and comment out the entries for any platforms you don't
need in the nuget package.
Run `nuget pack NativeNuget.nuspec` from this directory to produce the package for local testing.
-->
<!-- Windows and Linux via netstandard -->
<file src="..\nuget-artifacts\onnxruntime-extensions-win-x64\lib\ortextensions.dll" target="runtimes\win-x64\native" />
<file src="..\nuget-artifacts\onnxruntime-extensions-win-x86\lib\ortextensions.dll" target="runtimes\win-x86\native" />
<file src="..\nuget-artifacts\onnxruntime-extensions-win-arm64\lib\ortextensions.dll" target="runtimes\win-arm64\native" />
<file src="..\nuget-artifacts\onnxruntime-extensions-win-arm\lib\ortextensions.dll" target="runtimes\win-arm\native" />
<file src="..\nuget-artifacts\onnxruntime-extensions-linux-aarch64\lib\libortextensions.so" target="runtimes\linux-arm64\native" />
<file src="..\nuget-artifacts\onnxruntime-extensions-linux-x64\lib\libortextensions.so" target="runtimes\linux-x64\native" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard1.1" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.props" target="build\netstandard2.0" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\netstandard1.1" />
<file src="targets\netstandard\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\netstandard2.0" />
<!-- Android via Xamarin/MAUI -->
<file src="..\nuget-artifacts\onnxruntime-extensions-android\onnxruntime-extensions.aar" target="runtimes\android\native" />
<file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\monoandroid11.0" />
<file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\monoandroid11.0" />
<file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-android31.0" />
<file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-android31.0" />
<!-- iOS via Xamarin/MAUI. -->
<file src="..\nuget-artifacts\onnxruntime_extensions.xcframework\**" target="runtimes\ios\native\onnxruntime_extensions.xcframework" />
<file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\xamarinios10" />
<file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\xamarinios10" />
<file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-ios15.4" />
<file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-ios15.4" />
<!-- .net 6 macOS -->
<file src="..\nuget-artifacts\onnxruntime-extensions-osx-x86_64\lib\libortextensions.dylib" target="runtimes\osx.10.14-x64\native" />
<!-- TODO: Fix build error to include arm target
<file src="..\nuget-artifacts\onnxruntime-extensions-osx-arm64\lib\libortextensions.dylib" target="runtimes\osx.10.14-arm64\native"/>
-->
<file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-macos12.3" />
<file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-macos12.3" />
</files>
</package>

Просмотреть файл

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Condition=" '$(AndroidApplication)'=='true' ">
<AndroidLibrary Include="$(MSBuildThisFileDirectory)..\..\runtimes\android\native\*">
<Link>%(Filename)%(Extension)</Link>
</AndroidLibrary>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Condition=" '$(AndroidApplication)'=='true' ">
<AndroidLibrary Bind="false" Include="$(MSBuildThisFileDirectory)..\..\runtimes\android\native\*">
<Link>%(Filename)%(Extension)</Link>
</AndroidLibrary>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Condition="('$(OutputType)'!='Library' OR '$(IsAppExtension)'=='True')">
<NativeReference Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\onnxruntime_extensions.xcframework">
<Kind>Static</Kind>
<IsCxx>True</IsCxx>
<SmartLink>True</SmartLink>
<ForceLoad>True</ForceLoad>
<LinkerFlags>-lc++</LinkerFlags>
<WeakFrameworks>CoreML</WeakFrameworks>
</NativeReference>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Condition="('$(OutputType)'!='Library' OR '$(IsAppExtension)'=='True')">
<NativeReference Condition="'$(Platform)' == 'arm64'" Include="$(MSBuildThisFileDirectory)..\..\runtimes\osx.10.14-arm64\native\libonnxruntime_extensions.dylib">
<Kind>Framework</Kind>
<IsCxx>True</IsCxx>
<SmartLink>True</SmartLink>
<ForceLoad>True</ForceLoad>
<LinkerFlags>-lc++</LinkerFlags>
<WeakFrameworks>CoreML</WeakFrameworks>
</NativeReference>
<NativeReference Condition="'$(Platform)' == 'x64'" Include="$(MSBuildThisFileDirectory)..\..\runtimes\osx.10.14-x64\native\libonnxruntime_extensions.dylib">
<Kind>Framework</Kind>
<IsCxx>True</IsCxx>
<SmartLink>True</SmartLink>
<ForceLoad>True</ForceLoad>
<LinkerFlags>-lc++</LinkerFlags>
<WeakFrameworks>CoreML</WeakFrameworks>
</NativeReference>
</ItemGroup>
</Project>

Просмотреть файл

@ -1,18 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<EnginePlatform Condition="'$(Platform)' == 'Win32'">x86</EnginePlatform>
<EnginePlatform Condition="'$(Platform)' != 'Win32' AND '$(Platform)' != 'ARM64'">$(Platform)</EnginePlatform>
</PropertyGroup>
<PropertyGroup>
<OnnxRuntimeBinary>$(MSBuildThisFileDirectory)../../runtimes/win-$(EnginePlatform)/native/ortextensions.dll</OnnxRuntimeBinary>
</PropertyGroup>
<ItemGroup>
<!-- TODO: Should be able to have a single entry and use something like OnnxRuntimeBinary in it.
Need to handle AnyCPU with Prefer32Bit. -->
<!-- x64 -->
<None Include="$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\ortextensions.dll"
Condition="'$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')">
@ -28,5 +16,21 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<!-- arm64 -->
<None Include="$(MSBuildThisFileDirectory)..\..\runtimes\win-arm64\native\ortextensions.dll"
Condition="'$(PlatformTarget)' == 'ARM64'">
<Link>ortextensions.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<!-- arm -->
<None Include="$(MSBuildThisFileDirectory)..\..\runtimes\win-arm\native\ortextensions.dll"
Condition="'$(PlatformTarget)' == 'ARM'">
<Link>ortextensions.dll</Link>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Condition="('$(OutputType)'!='Library' OR '$(IsAppExtension)'=='True')">
<NativeReference Condition="'$(Platform)' == 'iPhoneSimulator'" Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\onnxruntime_extensions.xcframework\ios-arm64_x86_64-simulator\onnxruntime_extensions.framework">
<Kind>Framework</Kind>
<IsCxx>True</IsCxx>
<SmartLink>True</SmartLink>
<ForceLoad>True</ForceLoad>
<LinkerFlags>-lc++</LinkerFlags>
<WeakFrameworks>CoreML</WeakFrameworks>
</NativeReference>
<NativeReference Condition="'$(Platform)' == 'iPhone'" Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\onnxruntime_extensions.xcframework\ios-arm64\onnxruntime_extensions.framework">
<Kind>Framework</Kind>
<IsCxx>True</IsCxx>
<SmartLink>True</SmartLink>
<ForceLoad>True</ForceLoad>
<LinkerFlags>-lc++</LinkerFlags>
<WeakFrameworks>CoreML</WeakFrameworks>
</NativeReference>
</ItemGroup>
</Project>

Просмотреть файл

@ -12,26 +12,22 @@ import shutil
import sys
from typing import List
_script_dir = Path(__file__).resolve().parent
_repo_dir = _script_dir.parents[1]
_repo_dir = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(_repo_dir / "tools"))
from utils import get_logger, is_windows, run
from utils import get_logger, is_windows, run # noqa
_supported_abis = ["armeabi-v7a", "arm64-v8a", "x86", "x86_64"]
_log = get_logger("build_aar")
def build_for_abi(
build_dir: Path,
config: str,
abi: str,
api_level: int,
sdk_path: Path,
ndk_path: Path,
cmake_extra_defines: List[str],
):
def build_for_abi(build_dir: Path,
config: str,
abi: str,
api_level: int,
sdk_path: Path,
ndk_path: Path,
other_args: List[str]):
build_cmd = [
sys.executable,
str(_repo_dir / "tools" / "build.py"),
@ -47,20 +43,19 @@ def build_for_abi(
f"--android_api={api_level}",
f"--android_home={sdk_path}",
f"--android_ndk_path={ndk_path}",
] + ((["--cmake_extra_defines"] + cmake_extra_defines) if cmake_extra_defines else [])
] + other_args
run(*build_cmd)
def build_aar(
output_dir: Path,
config: str,
abis: List[str],
api_level: int,
sdk_path: Path,
ndk_path: Path,
cmake_extra_defines: List[str],
):
def do_build_by_mode(output_dir: Path,
config: str,
mode: str,
abis: List[str],
api_level: int,
sdk_path: Path,
ndk_path: Path,
other_args: List[str]):
output_dir = output_dir.resolve()
sdk_path = sdk_path.resolve(strict=True)
@ -72,17 +67,24 @@ def build_aar(
intermediates_dir = output_dir / "intermediates"
base_jnilibs_dir = intermediates_dir / "jnilibs" / config
for abi in abis:
build_dir = intermediates_dir / abi
build_for_abi(build_dir, config, abi, api_level, sdk_path, ndk_path, cmake_extra_defines)
if mode in ["build_so_only", "build_aar"]:
for abi in abis:
build_dir = intermediates_dir / abi
build_for_abi(build_dir, config, abi, api_level, sdk_path, ndk_path, other_args)
# copy JNI library files to jnilibs_dir
jnilibs_dir = base_jnilibs_dir / abi
jnilibs_dir.mkdir(parents=True, exist_ok=True)
# copy JNI library files to jnilibs_dir
jnilibs_dir = base_jnilibs_dir / abi
jnilibs_dir.mkdir(parents=True, exist_ok=True)
jnilib_names = ["libonnxruntime_extensions4j_jni.so"]
for jnilib_name in jnilib_names:
shutil.copyfile(build_dir / config / "java" / "android" / abi / jnilib_name, jnilibs_dir / jnilib_name)
jnilib_names = ["libonnxruntime_extensions4j_jni.so"]
for jnilib_name in jnilib_names:
shutil.copyfile(build_dir / config / "java" / "android" / abi / jnilib_name, jnilibs_dir / jnilib_name)
# early return if only building JNI libraries
# To accelerate the build pipeline, we can build the JNI libraries first in parallel for different abi,
# and then build the AAR package.
if mode == "build_so_only":
return
java_root = _repo_dir / "java"
gradle_build_file = java_root / "build-android.gradle"
@ -114,7 +116,10 @@ def build_aar(
def parse_args():
parser = argparse.ArgumentParser(description="Builds the Android AAR package for onnxruntime-extensions.")
parser = argparse.ArgumentParser(
description="""Builds the Android AAR package for onnxruntime-extensions.
Any additional arguments provided will be passed through to build.py.
""")
def path_from_env_var(env_var: str):
env_var_value = os.environ.get(env_var)
@ -126,6 +131,7 @@ def parse_args():
required=True,
help="Path to output directory.",
)
parser.add_argument(
"--config",
choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
@ -133,6 +139,24 @@ def parse_args():
help="CMake build configuration.",
)
# this one is used in the ci pipeline for accelerating the build process,
# we have 4 archs to be built. It's in sequence by default, but we can build them in parallel.
# The parallel build works as:
# 1. build the so files for each arch in different ci jobs
# 2. download all the so files in tasks
# 3. pack the aar package
parser.add_argument(
"--mode",
type=str,
choices=["build_aar", "build_so_only", "pack_aar_only"],
default="build_aar",
help="""Build mode:
'build_aar' builds the AAR package.
'build_so_only' builds the so libraries.
'pack_aar_only' only pack aar from existing so files.
""",
)
parser.add_argument(
"--abi",
action="append",
@ -145,8 +169,8 @@ def parse_args():
parser.add_argument(
"--api-level",
type=int,
default=24,
help="Android API Level. E.g., 24.",
default=21,
help="Android API Level. E.g., 21.",
)
parser.add_argument(
@ -163,16 +187,8 @@ def parse_args():
help="Path to the Android NDK. Typically `<Android SDK>/ndk/<ndk_version>`.",
)
parser.add_argument(
"--cmake-extra-defines",
action="append",
nargs="+",
default=[],
help="Extra definition(s) to pass to CMake (with the CMake -D option) during build system generation. "
"E.g., `--cmake-extra-defines OPTION1=ON OPTION2=OFF --cmake-extra-defines OPTION3=ON`.",
)
args = parser.parse_args()
args, unknown_args = parser.parse_known_args()
args.other_args = unknown_args
args.abis = args.abis or _supported_abis.copy()
@ -184,9 +200,6 @@ def parse_args():
args.ndk_path is not None
), "Android NDK path must be provided with --ndk-path or environment variable ANDROID_NDK_HOME."
# convert from List[List[str]] to List[str]
args.cmake_extra_defines = [element for inner_list in args.cmake_extra_defines for element in inner_list]
return args
@ -195,14 +208,15 @@ def main():
_log.info(f"Building AAR for ABIs: {args.abis}")
build_aar(
do_build_by_mode(
output_dir=args.output_dir,
config=args.config,
mode=args.mode,
abis=args.abis,
api_level=args.api_level,
sdk_path=args.sdk_path,
ndk_path=args.ndk_path,
cmake_extra_defines=args.cmake_extra_defines,
other_args=args.other_args,
)
_log.info("AAR build complete.")

Просмотреть файл

@ -162,6 +162,9 @@ def _parse_arguments():
parser.add_argument("--x86", action="store_true",
help="[cross-compiling] Create Windows x86 makefiles. Requires --update and no existing cache "
"CMake setup. Delete CMakeCache.txt if needed")
# x64 args
parser.add_argument("--x64", action="store_true",
help="doing nothing, just for compatibility")
# Arguments needed by CI
parser.add_argument("--cmake_path", default="cmake", type=Path, help="Path to the CMake program.")

Просмотреть файл

@ -0,0 +1,79 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# This file is used by the Nuget Packaging Pipeline
# TODO: Setup params properly for the arguments
# arg0: Source directory that build artifacts for the various platforms are downloaded to
# arg1: Target directory that we extract the relevant files to and setup the directory structure expected by
# /nuget/NativeNuget.nuspec
if ($args.Count -ne 2) {
throw "Specify source and output directories respectively."
}
$artifact_download_dir = $args[0]
$nuget_sources_dir = $args[1]
# Create a directory to extract files from the various artifacts to be packed into the nuget package.
# All 'src' paths in the nuspec 'files' section for binary files should be valid once we finish extraction.
New-Item -Path $nuget_sources_dir -ItemType directory
## .zip files
# unzip directly
Get-ChildItem $artifact_download_dir -Filter *.zip |
Foreach-Object {
$cmd = "7z.exe x $($_.FullName) -y -o$nuget_sources_dir"
Write-Output $cmd
Invoke-Expression -Command $cmd
}
## .tgz files
# first extract the tar file from the tgz
Get-ChildItem $artifact_download_dir -Filter *.tgz |
Foreach-Object {
$cmd = "7z.exe x $($_.FullName) -y -o$artifact_download_dir"
Write-Output $cmd
Invoke-Expression -Command $cmd
}
## .tar files
Get-ChildItem $artifact_download_dir -Filter *.tar |
Foreach-Object {
$cmd = "7z.exe x $($_.FullName) -y -o$nuget_sources_dir"
Write-Output $cmd
Invoke-Expression -Command $cmd
}
# copy android AAR.
# target file structure:
# nuget-artifact
# onnxruntime-extensions-android
# onnxruntime-extensions-android-x.y.z.aar <-- this is the file we want
# to match any {version}
$aars = Get-ChildItem $artifact_download_dir -Filter onnxruntime-extensions-android-*.aar
# could be empty if no android build
if ($aars.Count -eq 1) {
$aar = $aars[0]
$target_dir = "$nuget_sources_dir\onnxruntime-extensions-android-aar"
$target_file = "$target_dir\onnxruntime-extensions.aar" # remove -version info from filename
New-Item -Path $target_dir -ItemType directory
Write-Output "Copy-Item $($aar.FullName) $target_file"
Copy-Item $aar.FullName $target_file
}
Write-Output "Get-ChildItem -Directory -Path $nuget_sources_dir\onnxruntime-extensions-*"
$ort_dirs = Get-ChildItem -Directory -Path $nuget_sources_dir\onnxruntime-extensions-*
foreach ($ort_dir in $ort_dirs)
{
# remove the last '-xxx' segment from the dir name. typically that's the architecture.
$dirname = Split-Path -Path $ort_dir -Leaf
$dirname = $dirname.SubString(0,$dirname.LastIndexOf('-'))
Write-Output "Renaming $ort_dir to $dirname"
Rename-Item -Path $ort_dir -NewName $nuget_sources_dir\$dirname
}
# List artifacts
"Post copy artifacts"
Get-ChildItem -Recurse $nuget_sources_dir\

Просмотреть файл

@ -0,0 +1,69 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import xml.etree.ElementTree as ElementTree
import argparse
import sys
from pathlib import Path
def update_nuspec(args):
print(f"Updating {args.nuspec_path}")
# preserve comments
tree = ElementTree.parse(args.nuspec_path,
parser = ElementTree.XMLParser(target=ElementTree.TreeBuilder(insert_comments=True)))
root = tree.getroot()
# update version and commit id
packages_node = root.findall('metadata')[0]
for package_item in packages_node:
if package_item.tag == "version" and args.package_version:
if args.is_release_build:
package_item.text = args.package_version
else:
import datetime
now = datetime.datetime.now().strftime('%Y%m%d-%H%M')
package_item.text = f"{args.package_version}-dev-{now}-{args.commit_id[:8]}"
elif package_item.tag == "repository" and args.commit_id:
package_item.attrib['commit'] = args.commit_id
# format of indent
py_version = sys.version_info
if py_version > (3, 9):
ElementTree.indent(root)
tree.write(args.nuspec_path, encoding='utf-8', xml_declaration=True)
return
def parse_arguments():
parser = argparse.ArgumentParser(
description="ONNXRuntime extensions create nuget spec script (for hosting native shared library artifacts)",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
default_nuspec = Path(__file__).resolve().parents[2] / "nuget" / "NativeNuget.nuspec"
# Main arguments
parser.add_argument("--package_version", default='', help="ORT extensions package version. e.g.: 1.0.0")
parser.add_argument("--nuspec_path", type=Path, default=default_nuspec,
help="Path to nuspec file to update.")
parser.add_argument("--commit_id", required=True, help="The last commit id included in this package.")
parser.add_argument("--is_release_build", default="False", type=str, help="If it's a release build.")
args = parser.parse_args()
args.nuspec_path = args.nuspec_path.resolve(strict=True)
args.is_release_build = args.is_release_build.lower() == "true"
print("used args:", args)
return args
def main():
args = parse_arguments()
update_nuspec(args)
if __name__ == "__main__":
sys.exit(main())

176
tools/ios/build_xcframework.py Executable file → Normal file
Просмотреть файл

@ -54,7 +54,7 @@ def build_framework_for_platform_and_arch(
opencv_dir: Path,
ios_deployment_target: str,
cmake_extra_defines: List[str],
) -> Path:
):
build_dir.mkdir(parents=True, exist_ok=True)
# generate build files
@ -88,12 +88,11 @@ def build_framework_for_platform_and_arch(
# build
_run([_cmake, f"--build", f"{build_dir}", f"--config={config}", "--parallel"])
return build_dir / "static_framework/onnxruntime_extensions.framework"
def build_xcframework(
output_dir: Path,
platform_archs: Dict[str, List[str]],
mode: str,
config: str,
opencv_dir: Path,
ios_deployment_target: str,
@ -105,67 +104,91 @@ def build_xcframework(
assert len(platform_archs) > 0, "no platforms specified"
# the public headers and framework_info.json should be the same across platform/arch builds
# select them from one of the platform/arch build directories to copy to the output directory
headers_dir = None
framework_info_file = None
platform_fat_framework_dirs = []
for platform, archs in platform_archs.items():
assert len(archs) > 0, f"no arch specified for platform {platform}"
arch_framework_dirs = []
for arch in archs:
arch_framework_dir = build_framework_for_platform_and_arch(
intermediate_build_dir / f"{platform}/{arch}/{config}",
platform,
arch,
config,
opencv_dir,
ios_deployment_target,
cmake_extra_defines,
)
arch_framework_dirs.append(arch_framework_dir)
def platform_arch_framework_build_dir(platform, arch):
return intermediate_build_dir / f"{platform}/{arch}"
build_platform_arch_frameworks = mode in ["build_platform_arch_frameworks_only", "build_xcframework"]
if build_platform_arch_frameworks:
for platform, archs in platform_archs.items():
for arch in archs:
build_framework_for_platform_and_arch(
platform_arch_framework_build_dir(platform, arch),
platform,
arch,
config,
opencv_dir,
ios_deployment_target,
cmake_extra_defines,
)
pack_xcframework = mode in ["pack_xcframework_only", "build_xcframework"]
if pack_xcframework:
# the public headers and framework_info.json should be the same across platform/arch builds
# select them from one of the platform/arch build directories to copy to the output directory
headers_dir = None
framework_info_file = None
# create per-platform fat framework from platform/arch frameworks
platform_fat_framework_dirs = []
for platform, archs in platform_archs.items():
arch_framework_dirs = [
platform_arch_framework_build_dir(platform, arch) / "static_framework/onnxruntime_extensions.framework"
for arch in archs
]
if not build_platform_arch_frameworks:
# if they weren't just built, check that the expected platform/arch framework directories are present
for arch_framework_dir in arch_framework_dirs:
assert (
arch_framework_dir.is_dir()
), f"platform/arch framework directory not found: {arch_framework_dir}"
first_arch_framework_dir = arch_framework_dirs[0]
if headers_dir is None:
headers_dir = arch_framework_dir / "Headers"
framework_info_file = arch_framework_dir.parents[1] / "framework_info.json"
headers_dir = first_arch_framework_dir / "Headers"
framework_info_file = first_arch_framework_dir.parents[1] / "framework_info.json"
platform_fat_framework_dir = intermediate_build_dir / f"{platform}/onnxruntime_extensions.framework"
_rmtree_if_existing(platform_fat_framework_dir)
platform_fat_framework_dir.mkdir()
platform_fat_framework_dir = intermediate_build_dir / f"{platform}/onnxruntime_extensions.framework"
_rmtree_if_existing(platform_fat_framework_dir)
platform_fat_framework_dir.mkdir()
# copy over files from arch-specific framework
for framework_file_relative_path in [Path("Headers"), Path("Info.plist")]:
src = arch_framework_dirs[0] / framework_file_relative_path
dst = platform_fat_framework_dir / framework_file_relative_path
if src.is_dir():
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
# copy over files from arch-specific framework to fat framework
for framework_file_relative_path in [Path("Headers"), Path("Info.plist")]:
src = first_arch_framework_dir / framework_file_relative_path
dst = platform_fat_framework_dir / framework_file_relative_path
if src.is_dir():
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
# combine arch-specific framework libraries
arch_libs = [str(framework_dir / "onnxruntime_extensions") for framework_dir in arch_framework_dirs]
_run([_lipo, "-create", "-output", str(platform_fat_framework_dir / "onnxruntime_extensions")] + arch_libs)
# combine arch-specific framework libraries
arch_libs = [str(framework_dir / "onnxruntime_extensions") for framework_dir in arch_framework_dirs]
_run([_lipo, "-create", "-output", str(platform_fat_framework_dir / "onnxruntime_extensions")] + arch_libs)
platform_fat_framework_dirs.append(platform_fat_framework_dir)
platform_fat_framework_dirs.append(platform_fat_framework_dir)
# create xcframework
xcframework_dir = output_dir / "onnxruntime_extensions.xcframework"
_rmtree_if_existing(xcframework_dir)
# create xcframework
xcframework_dir = output_dir / "onnxruntime_extensions.xcframework"
_rmtree_if_existing(xcframework_dir)
create_xcframework_args = [_xcrun, "xcodebuild", "-create-xcframework", "-output", str(xcframework_dir)]
for platform_fat_framework_dir in platform_fat_framework_dirs:
create_xcframework_args += ["-framework", str(platform_fat_framework_dir)]
_run(create_xcframework_args)
create_xcframework_args = [_xcrun, "xcodebuild", "-create-xcframework", "-output", str(xcframework_dir)]
for platform_fat_framework_dir in platform_fat_framework_dirs:
create_xcframework_args += ["-framework", str(platform_fat_framework_dir)]
_run(create_xcframework_args)
# copy public headers
output_headers_dir = output_dir / "Headers"
_rmtree_if_existing(output_headers_dir)
shutil.copytree(headers_dir, output_headers_dir)
# copy public headers
output_headers_dir = output_dir / "Headers"
_rmtree_if_existing(output_headers_dir)
shutil.copytree(headers_dir, output_headers_dir)
# copy framework_info.json
shutil.copyfile(framework_info_file, output_dir / "framework_info.json")
# copy framework_info.json
shutil.copyfile(framework_info_file, output_dir / "framework_info.json")
def parse_args():
@ -174,24 +197,30 @@ def parse_args():
)
parser.add_argument(
"--output-dir",
"--output_dir",
type=Path,
required=True,
help="Path to output directory.",
)
# This option is used in CI pipelines to accelerate the build process,
# We have multiple platform/archs to build for. We can build them in parallel.
# The parallel build works like this:
# 1. build the platform/arch frameworks in different jobs, in parallel
# 2. download the platform/arch framework files from the previous jobs and pack them into the xcframework
parser.add_argument(
"--config",
choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
default="Debug",
help="CMake build configuration.",
"--mode",
default="build_xcframework",
choices=["build_xcframework", "build_platform_arch_frameworks_only", "pack_xcframework_only"],
help="Build mode. "
"'build_xcframework' builds the whole package. "
"'build_platform_arch_frameworks_only' builds the platform/arch frameworks only. "
"'pack_xcframework_only' packs the xcframework from existing lib files only. "
"Note: 'pack_xcframework_only' assumes previous invocation(s) with mode 'build_platform_arch_frameworks_only'.",
)
parser.add_argument(
"--ios-deployment-target",
default="11.0",
help="iOS deployment target.",
)
parser.add_argument(
"--platform-arch",
"--platform_arch",
nargs=2,
action="append",
metavar=("PLATFORM", "ARCH"),
@ -199,16 +228,32 @@ def parse_args():
help="Specify a platform/arch pair to build. Repeat to specify multiple pairs. "
"If no pairs are specified, all supported pairs will be built.",
)
# platform/arch framework build-related options
parser.add_argument(
"--cmake-extra-defines",
"--config",
choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
default="Debug",
help="CMake build configuration.",
)
parser.add_argument(
"--ios_deployment_target",
default="11.0",
help="iOS deployment target.",
)
parser.add_argument(
"--cmake_extra_defines",
action="append",
nargs="+",
default=[],
help="Extra definition(s) to pass to CMake (with the CMake -D option) during build system generation. "
"E.g., `--cmake-extra-defines OPTION1=ON OPTION2=OFF --cmake-extra-defines OPTION3=ON`.",
"e.g., `--cmake_extra_defines OPTION1=ON OPTION2=OFF --cmake_extra_defines OPTION3=ON`.",
)
args = parser.parse_args()
# ignore unknown args which may be present in a CI build for the nuget package as we pass through
# all additional build flags
args, unknown_args = parser.parse_known_args()
# convert from [[platform1, arch1], [platform1, arch2], ...] to {platform1: [arch1, arch2, ...], ...}
def platform_archs_from_args(platform_archs_arg: Optional[List[List[str]]]) -> Dict[str, List[str]]:
@ -245,6 +290,7 @@ def main():
build_xcframework(
output_dir=args.output_dir,
platform_archs=args.platform_archs,
mode=args.mode,
config=args.config,
opencv_dir=_repo_dir / "cmake/externals/opencv",
ios_deployment_target=args.ios_deployment_target,