update Python pipelines for release (#353)

* re-enable Python pipelines

* deprecate the direct setup.py call

* pipeline fixing

* run cmake from visual studio

* remove the self check

* support different ort versions

* remove ort 1.9 and add 1.13

* ci pipeline fixing

* fix the test with latest pytorch

* code refinement

* torch version detection

* make package python version more consistent.
This commit is contained in:
Wenbing Li 2023-02-06 18:23:56 -08:00 коммит произвёл GitHub
Родитель 95ac242e7c
Коммит 1af7a56b28
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
17 изменённых файлов: 119 добавлений и 130 удалений

4
.gitignore поставляемый
Просмотреть файл

@ -35,7 +35,7 @@ nuget_root/
__pycache__
out/
*.egg-info/
.setuptools-cmake-build/
.scb/
onnxruntime_extensions/_version.py
onnxruntime-*-*-*/
temp_*.onnx
@ -50,4 +50,4 @@ tutorials/*/app/libs
*.dylib
*.pyd
/test/data/ppp_vision/*.updated.onnx
/test/data/generated/
/test/data/generated/

Просмотреть файл

@ -58,11 +58,11 @@ extends:
jobs:
- job: main
pool:
type: windows
type: windows
variables:
ob_outputDirectory: '$(REPOROOT)\out'
ob_sdl_binskim_break: true
ob_sdl_binskim_break: true
steps:
- script: |
@echo off
@ -97,7 +97,6 @@ extends:
python -m pip install --upgrade pip
python -m pip install cibuildwheel numpy
python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out
del /s /q /f .setuptools-cmake-build\*onnxruntime.dll
displayName: Build wheels
- task: SDLNativeRules@3
inputs:

Просмотреть файл

@ -37,7 +37,7 @@ resources:
ref: refs/heads/main
extends:
template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
parameters:
cloudvault:
enabled: false
@ -45,7 +45,7 @@ extends:
tsa:
enabled: false
binskim:
break: true # always break the build on binskim issues in addition to TSA upload
break: false # always break the build on binskim issues in addition to TSA upload
analyzeTargetGlob: '**\RelWithDebInfo\ortextensions.dll' # avoid scanning the 3rd party DLLs.
codeql:
python:
@ -59,11 +59,11 @@ extends:
jobs:
- job: main
pool:
type: windows
type: windows
variables:
ob_outputDirectory: '$(REPOROOT)\out'
ob_sdl_binskim_break: true
ob_sdl_binskim_break: false
steps:
- script: |
@echo off
@ -98,7 +98,6 @@ extends:
python -m pip install --upgrade pip
python -m pip install cibuildwheel numpy
python -m cibuildwheel --platform windows --archs AMD64 --output-dir $(REPOROOT)\out
del /s /q /f .setuptools-cmake-build\*onnxruntime.dll
displayName: Build wheels
- task: SDLNativeRules@3
inputs:

Просмотреть файл

@ -15,29 +15,28 @@ jobs:
#######
# Linux
#######
- job: Linux
pool:
vmImage: 'ubuntu-latest'
strategy:
matrix:
py310-1121:
py310-1131:
python.version: '3.10'
torch.version: 'torch torchvision torchaudio'
ort.version: '1.13.1'
py39-1121:
python.version: '3.9'
torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.12.0'
py39-1110:
python.version: '3.9'
py38-1110:
python.version: '3.8'
torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.11.0'
py38-1100:
python.version: '3.8'
py37-1100:
python.version: '3.7'
torch.version: 'torch==1.10.0+cpu torchvision==0.11.1+cpu torchaudio==0.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.10.0'
py37-190:
python.version: '3.7'
torch.version: 'torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.9.0'
steps:
- task: DownloadGitHubRelease@0
@ -59,7 +58,7 @@ jobs:
displayName: Unpack ONNXRuntime package.
- script: |
CPU_NUMBER=2 sh ./build.sh -DOCOS_ENABLE_CTEST=ON
CPU_NUMBER=2 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/onnxruntime-linux-x64-$(ort.version)
displayName: build the customop library with onnxruntime
- script: |
@ -75,11 +74,12 @@ jobs:
- script: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools
python -m pip install onnxruntime=${ort.version}
python -m pip install -r requirements.txt
displayName: Install requirements.txt
- script: |
CPU_NUMBER=2 python setup.py develop
CPU_NUMBER=2 python -m pip install -e .
displayName: Build the library and tests
- script: python -m pip install $(torch.version)
@ -95,13 +95,15 @@ jobs:
###########
# macOS C++
###########
- job: MacOSX
pool:
vmImage: 'macOS-12'
strategy:
matrix:
ort-1131:
ort.version: '1.13.1'
ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
ort-1121:
ort.version: '1.12.1'
ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
@ -111,9 +113,6 @@ jobs:
ort-1100:
ort.version: '1.10.0'
ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
ort-190:
ort.version: '1.9.0'
ort.dirname: 'onnxruntime-osx-x64-$(ort.version)'
steps:
# needed for onnxruntime
@ -139,7 +138,7 @@ jobs:
displayName: Unpack ONNXRuntime package.
- script: |
sh ./build.sh -DOCOS_ENABLE_CTEST=ON
sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/$(ort.dirname)
displayName: build the customop library with onnxruntime
- script: |
@ -148,15 +147,18 @@ jobs:
displayName: Run C++ native tests
#############
# macOS Wheel
# macOS Python
#############
- job: MacOSWheel
- job: MacOSPython
pool:
vmImage: 'macOS-12'
strategy:
matrix:
py310-1131:
python.version: '3.10'
torch.version: 'torch torchvision torchaudio'
ort.version: '1.13.1'
py310-1121:
python.version: '3.10'
torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
@ -169,10 +171,6 @@ jobs:
python.version: '3.8'
torch.version: 'torch==1.10.0 torchvision==0.11.1 torchaudio==0.10.0 -f https://download.pytorch.org/whl/torch_stable.html'
ort.version: '1.10.0'
py37-190:
python.version: '3.7'
torch.version: 'torch==1.9.0 torchvision==0.10.0 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html'
ort.version: '1.9.0'
steps:
# needed for onnxruntime
@ -188,15 +186,16 @@ jobs:
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools
python -m pip install --upgrade wheel
python -m pip install onnxruntime=${ort.version}
python -m pip install -r requirements.txt
displayName: Install requirements.txt
- script: python -c "import onnxruntime;print(onnxruntime.__version__)"
- script: |
python -c "import onnxruntime;print(onnxruntime.__version__)"
displayName: Check installation
- script: |
python setup.py bdist_wheel
for file in dist/*.whl; do python -m pip install --upgrade $file; done
python -m pip install -e .
displayName: Build and install the wheel
- script: python -m pip install -r requirements-dev.txt
@ -208,29 +207,23 @@ jobs:
- script: cd test && python -m pytest . --verbose
displayName: Run python test
- task: PublishPipelineArtifact@0
inputs:
artifactName: 'wheel-macos-$(python.version)'
targetPath: 'dist'
#########
# Windows C++
#########
- job: WindowsC
pool:
vmImage: 'windows-2019'
strategy:
matrix:
ort-1131:
ort.version: '1.13.1'
ort-1121:
ort.version: '1.12.1'
ort-1110:
ort.version: '1.11.0'
ort-1100:
ort.version: '1.10.0'
ort-190:
ort.version: '1.9.0'
steps:
- task: DownloadGitHubRelease@0
@ -266,7 +259,7 @@ jobs:
- script: |
call $(vsdevcmd)
call .\build.bat -DOCOS_ENABLE_CTEST=ON
call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version)
displayName: build the customop library with onnxruntime
- script: |
@ -275,31 +268,30 @@ jobs:
displayName: Run C++ native tests
################
# Windows wheels
# Windows Python
################
- job: WindowsWheel
- job: WindowsPython
pool:
vmImage: 'windows-2019'
vmImage: 'windows-latest'
strategy:
matrix:
py310-1121:
py310-1131:
python.version: '3.10'
torch.version: 'torch torchvision torchaudio'
ort.version: '1.13.1'
py39-1121:
python.version: '3.9'
torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.12.1'
py39-1110:
python.version: '3.9'
py38-1110:
python.version: '3.8'
torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.11.0'
py38-1100:
python.version: '3.8'
py37-1100:
python.version: '3.7'
torch.version: 'torch==1.10.0+cpu torchvision==0.11.1+cpu torchaudio==0.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.10.0'
py37-190:
python.version: '3.7'
torch.version: 'torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html'
ort.version: '1.9.0'
steps:
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
@ -311,30 +303,16 @@ jobs:
- script: |
call activate pyenv
python -m pip install --upgrade pip
python -m pip install onnxruntime=${ort.version}
python -m pip install -r requirements.txt
python -m pip install -r requirements-dev.txt
displayName: Install requirements.txt
- script: |
call activate pyenv
@echo find where Visual Studio was installed and use the bundled cmake
set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
set vslatest=%%i
if exist "%%i\Common7\Tools\vsdevcmd.bat" (
set vsdevcmd=%%i\Common7\Tools\vsdevcmd.bat
)
)
@echo vslatest (%vslatest%)
@echo vsdevcmd (%vsdevcmd%)
python setup.py bdist_wheel
python -m pip install .
displayName: Build the wheel
- script: |
call activate pyenv
python -m pip install --find-links=dist\ onnxruntime-extensions
displayName: Install the wheel
- script: |
call activate pyenv
python -m pip install $(torch.version)
@ -345,15 +323,9 @@ jobs:
pytest test
displayName: Run python test
- task: PublishPipelineArtifact@0
inputs:
artifactName: 'wheel-window-$(python.version)'
targetPath: 'dist'
#############
# WebAssembly
#############
- job: WebAssembly
pool:
vmImage: 'ubuntu-latest'

Просмотреть файл

@ -3,7 +3,7 @@ jobs:
timeoutInMinutes: 120
pool: {vmImage: 'ubuntu-latest', name: 'Linux-CPU'}
variables:
CIBW_BUILD: "cp3{7,8,9}-*"
CIBW_BUILD: "cp3{7,8,9,10}-*"
steps:
- task: UsePythonVersion@0

Просмотреть файл

@ -3,7 +3,7 @@ jobs:
timeoutInMinutes: 120
pool: {vmImage: 'macOS-12'}
variables:
CIBW_BUILD: "cp3{8,9}-*"
CIBW_BUILD: "cp3{8,9,10}-*"
CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"
# Skip trying to test arm64 builds on Intel Macs
# CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"

Просмотреть файл

@ -3,7 +3,7 @@ jobs:
timeoutInMinutes: 120
pool: {vmImage: 'windows-latest', name: 'Win-CPU-2021'}
variables:
CIBW_BUILD: "cp3{7,8,9}-*amd64"
CIBW_BUILD: "cp3{7,8,9,10}-*amd64"
steps:
- task: UsePythonVersion@0

Просмотреть файл

@ -1,9 +1,11 @@
@ECHO OFF
SETLOCAL ENABLEDELAYEDEXPANSION
IF DEFINED VSINSTALLDIR GOTO :VSDEV_CMD
IF NOT DEFINED VSDEVCMD GOTO :NOT_FOUND
set _VSFINDER=%~dp0tools\get_vsdevcmd.ps1
for /f "tokens=* USEBACKQ" %%i in (
`powershell -NoProfile -ExecutionPolicy Bypass -File "%_VSFINDER%"`) do call "%%i"
CALL "%VSDEVCMD%"
IF NOT DEFINED VSINSTALLDIR GOTO :NOT_FOUND
:VSDEV_CMD
set GENERATOR="Visual Studio 16 2019"

Просмотреть файл

@ -1,6 +1,9 @@
if(_ONNXRUNTIME_EMBEDDED)
set(ONNXRUNTIME_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/../include/onnxruntime/core/session)
set(ONNXRUNTIME_LIB_DIR "")
elseif(ONNXRUNTIME_PKG_DIR)
set(ONNXRUNTIME_INCLUDE_DIR ${ONNXRUNTIME_PKG_DIR}/include)
set(ONNXRUNTIME_LIB_DIR ${ONNXRUNTIME_PKG_DIR}/lib)
else()
# default to 1.10.0 if not specified
set(ONNXRUNTIME_VER "1.10.0" CACHE STRING "ONNX Runtime version")

Просмотреть файл

@ -420,17 +420,17 @@ OrtStatusPtr RegisterPythonDomainAndOps(OrtSessionOptions* options, const OrtApi
continue; // Register this domain in the second iteration.
}
if (status = ortApi->CreateCustomOpDomain(val_pair.first.c_str(), &domain)) {
if (status = ortApi->CreateCustomOpDomain(val_pair.first.c_str(), &domain); status) {
return status;
}
for (auto const& cop: val_pair.second) {
if (status = ortApi->CustomOpDomain_Add(domain, &cop)) {
if (status = ortApi->CustomOpDomain_Add(domain, &cop); status) {
return status;
}
}
if (status = ortApi->AddCustomOpDomain(options, domain)) {
if (status = ortApi->AddCustomOpDomain(options, domain); status) {
return status;
}
}

Просмотреть файл

@ -1,6 +1,6 @@
[build-system]
# Minimum requirements for the build system to execute.
requires = ["setuptools", "wheel", "numpy>=1.18.5"] # PEP 508 specifications.
requires = ["setuptools", "wheel", "numpy>=1.18.5", "cmake"] # PEP 508 specifications.
[tool.black]
line-length = 120

Просмотреть файл

@ -1,2 +1,2 @@
onnx>=1.9.0
onnxruntime>=1.6.0
onnxruntime>=1.10.0

Просмотреть файл

@ -1,3 +1,3 @@
[build]
build_base = .setuptools-cmake-build
build_base = .scb
# debug = 1

Просмотреть файл

@ -22,12 +22,11 @@ PACKAGE_NAME = 'onnxruntime_extensions'
VSINSTALLDIR_NAME = 'VSINSTALLDIR'
def load_msvcvar():
if os.environ.get('vsdevcmd'):
# need put the quotation marks around the path to avoid popen massing with it.
os.environ['__vsdevcmd'] = '"{}"'.format(os.environ['vsdevcmd'])
def load_vsdevcmd():
if os.environ.get(VSINSTALLDIR_NAME) is None:
stdout, _ = subprocess.Popen([
'cmd', '/q', '/c', 'call %__vsdevcmd% && set {}'.format(VSINSTALLDIR_NAME)],
'powershell', ' -noprofile', '-executionpolicy',
'bypass', '-f', TOP_DIR+'/tools/get_vsdevcmd.ps1', '-outputEnv' '1'],
stdout=subprocess.PIPE, shell=False, universal_newlines=True).communicate()
for line in stdout.splitlines():
kv_pair = line.split('=')
@ -81,14 +80,17 @@ class BuildCMakeExt(_build_ext):
cmake_args = [
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(ext_fullpath.parent.absolute()),
'-DOCOS_BUILD_PYTHON=ON',
'-DOCOS_ENABLE_CTEST=OFF',
# Disabling openCV can drastically reduce the build time.
# '-DOCOS_ENABLE_OPENCV_CODECS=OFF',
# '-DOCOS_ENABLE_CV2=OFF',
# '-DOCOS_ENABLE_VISION=OFF',
'-DOCOS_EXTENTION_NAME=' + ext_fullpath.name,
'-DCMAKE_BUILD_TYPE=' + config
]
if os.environ.get('OCOS_NO_OPENCV') == '1':
# Disabling openCV can drastically reduce the build time.
cmake_args += [
'-DOCOS_ENABLE_CTEST=OFF',
'-DOCOS_ENABLE_OPENCV_CODECS=OFF',
'-DOCOS_ENABLE_CV2=OFF',
'-DOCOS_ENABLE_VISION=OFF']
# overwrite the Python module info if the auto-detection doesn't work.
# export Python3_INCLUDE_DIRS=/opt/python/cp38-cp38
# export Python3_LIBRARIES=/opt/python/cp38-cp38
@ -106,6 +108,8 @@ class BuildCMakeExt(_build_ext):
'--parallel' + ('' if cpu_number is None else ' ' + cpu_number)
]
cmake_exe = 'cmake'
# unlike Linux/MacOS, cmake python package on Windows fails to build some 3rd party dependencies.
# so we have to use the cmake installed with Visual Studio.
if os.environ.get(VSINSTALLDIR_NAME):
cmake_exe = os.environ[VSINSTALLDIR_NAME] +\
'Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake\\bin\\cmake.exe'
@ -166,7 +170,7 @@ def write_py_version(ortx_version):
if sys.platform == "win32":
load_msvcvar()
load_vsdevcmd()
ext_modules = [
setuptools.extension.Extension(

Просмотреть файл

@ -138,31 +138,6 @@ class TestPreprocessing(unittest.TestCase):
o_res = mfunc([_i.numpy() for _i in test_input])
numpy.testing.assert_allclose(res, o_res)
@unittest.skipIf(
LooseVersion(torch.__version__) < LooseVersion("1.11"),
"PythonOp bug fixing on Pytorch 1.11",
)
def test_functional_processing(self):
# load an image
img = Image.open(util.get_test_data_file("data", "pineapple.jpg")).convert(
"RGB"
)
img = torch.from_numpy(numpy.asarray(img))
pipeline = _MobileNetProcessingModule(
onnx.load_model(util.get_test_data_file("data", "mobilev2.onnx"))
)
ids, probabilities = pipeline.forward(img)
full_model_func = OrtPyFunction.from_model(
pnp.export(pipeline, img, opset_version=11, output_path="temp_func.onnx")
)
actual_ids, actual_result = full_model_func(img.numpy())
numpy.testing.assert_allclose(probabilities.numpy(), actual_result, rtol=1e-3)
self.assertEqual(
ids[0, 0].item(), 953
) # 953 is pineapple class id in the imagenet dataset
if __name__ == "__main__":
unittest.main()

Просмотреть файл

@ -21,7 +21,10 @@ def my_inverse(g, self):
return g.op("ai.onnx.contrib::Inverse", self)
register_custom_op_symbolic('::inverse', my_inverse, 1)
if LooseVersion(torch.__version__) >= LooseVersion("1.13"):
register_custom_op_symbolic('::linalg_inv', my_inverse, 1)
else:
register_custom_op_symbolic('::inverse', my_inverse, 1)
def my_all(g, self):

32
tools/get_vsdevcmd.ps1 Normal file
Просмотреть файл

@ -0,0 +1,32 @@
# Find all installed Visual Studio with Cmake, and only output the vsdevcmd path of the latest one
PARAM([int]$outputEnv = 0)
$latest_valid_vs = ""
function choose_latter_vs {
param([string]$path)
if ($global:latest_valid_vs -lt $path) {
$cmake_path = $path + "\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe"
if (Test-Path -Path $cmake_path) {
$global:latest_valid_vs = $path
}
}
}
$vswherepath=[Environment]::GetEnvironmentVariable(
"ProgramFiles(x86)") + "\Microsoft Visual Studio\Installer\vswhere.exe"
$vs_installed = & "$vswherepath" -latest -property installationPath
FOREACH ($line in $vs_installed) {
choose_latter_vs($line.Trim())
}
if ($latest_valid_vs.Length -eq 0) {
exit 1
}
$pathDevCmd = $latest_valid_vs + "\Common7\Tools\vsdevcmd.bat"
if ($outputEnv) {
cmd /q /c "$pathDevCmd & set"
}
else {
$pathDevCmd
}