Create webassembly pipeline (#78)
This commit is contained in:
Mojimi
GitHub
Родитель
87f4c234c7
Коммит
4e78e9651b
|
|
@ -13,7 +13,7 @@ dist/
|
|||
*.bin
|
||||
cmake_build
|
||||
.cmake_build
|
||||
cmake-build-debug
|
||||
cmake-build*
|
||||
gen
|
||||
.DS_Store
|
||||
*~
|
||||
|
|
|
|||
|
|
@ -15,15 +15,13 @@ include(CheckLanguage)
|
|||
|
||||
option(CC_OPTIMIZE "Allow compiler optimizations, Set to OFF to disable" ON)
|
||||
option(OCOS_ENABLE_PYTHON "Enable Python component building" OFF)
|
||||
option(OCOS_ENABLE_CTEST "Enable C++ test" ON)
|
||||
option(OCOS_ENABLE_TF_STRING "Enable String Operator Set" ON)
|
||||
option(OCOS_ENABLE_GPT2_TOKENIZER "Enable the GPT2 tokenizer building" ON)
|
||||
option(OCOS_ENABLE_SPM_TOKENIZER "Enable the SentencePiece tokenizer building" ON)
|
||||
option(OCOS_ENABLE_BERT_TOKENIZER "Enable the BertTokenizer building" ON)
|
||||
|
||||
find_library(ONNXRUNTIME onnxruntime HINTS "${ONNXRUNTIME_LIB_DIR}")
|
||||
if ((NOT OCOS_ENABLE_PYTHON) AND (NOT ONNXRUNTIME))
|
||||
message(FATAL_ERROR "Cannot find onnxruntime in the default library paths, please specify the ONNXRUNTIME_LIB_DIR.")
|
||||
endif()
|
||||
|
||||
|
||||
if(NOT CC_OPTIMIZE)
|
||||
message("!!!THE COMPILER OPTIMIZATION HAS BEEN DISABLED, DEBUG-ONLY!!!")
|
||||
|
|
@ -60,6 +58,9 @@ if (OCOS_ENABLE_TF_STRING)
|
|||
include(farmhash)
|
||||
FetchContent_GetProperties(googlere2)
|
||||
FetchContent_GetProperties(farmhash)
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
|
||||
set_property(TARGET re2 PROPERTY COMPILE_OPTIONS )
|
||||
endif()
|
||||
endif()
|
||||
|
||||
file(GLOB TARGET_SRC "./ocos/*.cc" "./ocos/*.h*" "./ocos/utils/*.h*" "./ocos/utils/*.cc")
|
||||
|
|
@ -98,6 +99,7 @@ endif()
|
|||
add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
|
||||
add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
|
||||
add_library(ortcustomops_static STATIC ${TARGET_SRC})
|
||||
target_include_directories(ortcustomops_static PUBLIC tokenizer)
|
||||
|
||||
set(ocos_libraries ortcustomops_static)
|
||||
if (OCOS_ENABLE_TF_STRING)
|
||||
|
|
@ -127,7 +129,7 @@ endif()
|
|||
|
||||
if (OCOS_ENABLE_SPM_TOKENIZER)
|
||||
# SentencePiece
|
||||
target_include_directories(ortcustomops_static PRIVATE ${PROJECT_SOURCE_DIR}/tokenizer ${sentencepieceproject_INCLUDE_DIRS})
|
||||
target_include_directories(ortcustomops_static PUBLIC ${sentencepieceproject_INCLUDE_DIRS})
|
||||
list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_SPM_TOKENIZER)
|
||||
list(APPEND ocos_libraries sentencepiece-static)
|
||||
endif()
|
||||
|
|
@ -159,15 +161,30 @@ if(OCOS_ENABLE_PYTHON)
|
|||
|
||||
Python3_add_library(ortcustomops SHARED ${TARGET_SRC_PYOPS} ${shared_TARGET_SRC})
|
||||
list(APPEND OCOS_COMPILE_DEFINITIONS PYTHON_OP_SUPPORT)
|
||||
elseif(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
|
||||
add_executable(ortcustomops ${shared_TARGET_SRC})
|
||||
set_target_properties(ortcustomops PROPERTIES LINK_FLAGS " \
|
||||
-s WASM=1 \
|
||||
-s NO_EXIT_RUNTIME=0 \
|
||||
-s ALLOW_MEMORY_GROWTH=1 \
|
||||
-s SAFE_HEAP=0 \
|
||||
-s MODULARIZE=1 \
|
||||
-s SAFE_HEAP_LOG=0 \
|
||||
-s STACK_OVERFLOW_CHECK=0 \
|
||||
-s EXPORT_ALL=0 \
|
||||
-s VERBOSE=0 \
|
||||
--no-entry")
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set_property(TARGET ortcustomops APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=1 -s DEMANGLE_SUPPORT=1")
|
||||
else()
|
||||
set_property(TARGET ortcustomops APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s DEMANGLE_SUPPORT=0")
|
||||
endif()
|
||||
else()
|
||||
list(APPEND shared_TARGET_SRC "${PROJECT_SOURCE_DIR}/shared/ortcustomops.def")
|
||||
add_library(ortcustomops SHARED ${shared_TARGET_SRC})
|
||||
endif()
|
||||
|
||||
target_compile_definitions(ortcustomops PRIVATE ${OCOS_COMPILE_DEFINITIONS})
|
||||
if (OCOS_ENABLE_SPM_TOKENIZER) # FIXME: this include path is not recommendeded.
|
||||
target_include_directories(ortcustomops PRIVATE ${PROJECT_SOURCE_DIR}/tokenizer ${sentencepieceproject_INCLUDE_DIRS})
|
||||
endif()
|
||||
target_link_libraries(ortcustomops PRIVATE ${ocos_libraries})
|
||||
|
||||
if(OCOS_ENABLE_PYTHON)
|
||||
|
|
@ -204,8 +221,13 @@ message(STATUS "fetch CPack")
|
|||
include(CPack)
|
||||
|
||||
# test section
|
||||
if (NOT OCOS_ENABLE_PYTHON)
|
||||
if (OCOS_ENABLE_CTEST)
|
||||
# Enable CTest
|
||||
find_library(ONNXRUNTIME onnxruntime HINTS "${ONNXRUNTIME_LIB_DIR}")
|
||||
if (NOT ONNXRUNTIME)
|
||||
message(FATAL_ERROR "Enable ctest should provide onnxruntime path, please specify the ONNXRUNTIME_LIB_DIR.")
|
||||
endif()
|
||||
|
||||
enable_testing()
|
||||
message(STATUS "fetch CTest")
|
||||
include(CTest)
|
||||
|
|
|
|||
|
|
@ -1,323 +1,346 @@
|
|||
jobs:
|
||||
|
||||
#######
|
||||
# Linux
|
||||
#######
|
||||
#######
|
||||
# Linux
|
||||
#######
|
||||
|
||||
- job: Linux
|
||||
pool:
|
||||
vmImage: 'ubuntu-latest'
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
build.lib: '1'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
build.lib: '0'
|
||||
maxParallel: 1
|
||||
- job: Linux
|
||||
pool:
|
||||
vmImage: 'ubuntu-latest'
|
||||
|
||||
steps:
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-linux-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
build.lib: '1'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
build.lib: '0'
|
||||
maxParallel: 1
|
||||
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.tgz'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
steps:
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-linux-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
|
||||
- script: |
|
||||
sh ./build.sh -DONNXRUNTIME_LIB_DIR=onnxruntime-linux-x64-$(ort.version)/lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.tgz'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
|
||||
- script: |
|
||||
cd out/Linux
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
- script: |
|
||||
sh ./build.sh -DONNXRUNTIME_LIB_DIR=onnxruntime-linux-x64-$(ort.version)/lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
addToPath: true
|
||||
- script: |
|
||||
cd out/Linux
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
|
||||
- script: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install --upgrade setuptools
|
||||
python -m pip install -r requirements.txt
|
||||
displayName: Install requirements.txt
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
addToPath: true
|
||||
|
||||
- script: |
|
||||
python setup.py develop
|
||||
displayName: Build the library and tests
|
||||
- script: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install --upgrade setuptools
|
||||
python -m pip install -r requirements.txt
|
||||
displayName: Install requirements.txt
|
||||
|
||||
- script: python -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
displayName: Install pytorch
|
||||
- script: |
|
||||
python setup.py develop
|
||||
displayName: Build the library and tests
|
||||
|
||||
- script: python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements-dev.txt
|
||||
- script: python -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
displayName: Install pytorch
|
||||
|
||||
- script: python -m pytest test --verbose --verbose
|
||||
displayName: Run python test
|
||||
- script: python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements-dev.txt
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
# python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
condition: eq(variables['build.lib'], '1')
|
||||
- script: python -m pytest test --verbose --verbose
|
||||
displayName: Run python test
|
||||
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-linux-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
condition: eq(variables['build.lib'], '1')
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
# python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
condition: eq(variables['build.lib'], '1')
|
||||
|
||||
###########
|
||||
# macOS C++
|
||||
###########
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-linux-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
condition: eq(variables['build.lib'], '1')
|
||||
|
||||
- job: MacOSC
|
||||
pool:
|
||||
vmImage: 'macOS-latest'
|
||||
###########
|
||||
# macOS C++
|
||||
###########
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
maxParallel: 1
|
||||
- job: MacOSC
|
||||
pool:
|
||||
vmImage: 'macOS-latest'
|
||||
|
||||
steps:
|
||||
# needed for onnxruntime
|
||||
- script: brew install libomp
|
||||
displayName: 'Install omp'
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
maxParallel: 1
|
||||
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-osx-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
steps:
|
||||
# needed for onnxruntime
|
||||
- script: brew install libomp
|
||||
displayName: 'Install omp'
|
||||
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.tgz'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-osx-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
|
||||
- script: |
|
||||
sh ./build.sh -DONNXRUNTIME_LIB_DIR=onnxruntime-osx-x64-$(ort.version)/lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.tgz'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
|
||||
- script: |
|
||||
cd out/Darwin
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
- script: |
|
||||
sh ./build.sh -DONNXRUNTIME_LIB_DIR=onnxruntime-osx-x64-$(ort.version)/lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
|
||||
#############
|
||||
# macOS Wheel
|
||||
#############
|
||||
- script: |
|
||||
cd out/Darwin
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
|
||||
- job: MacOSWheel
|
||||
pool:
|
||||
vmImage: 'macOS-latest'
|
||||
#############
|
||||
# macOS Wheel
|
||||
#############
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
py37-170:
|
||||
python.version: '3.7'
|
||||
ort.version: '1.7.0'
|
||||
py36-170:
|
||||
python.version: '3.6'
|
||||
ort.version: '1.7.0'
|
||||
maxParallel: 1
|
||||
- job: MacOSWheel
|
||||
pool:
|
||||
vmImage: 'macOS-latest'
|
||||
|
||||
steps:
|
||||
# needed for onnxruntime
|
||||
- script: brew install libomp
|
||||
displayName: 'Install omp'
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
py37-170:
|
||||
python.version: '3.7'
|
||||
ort.version: '1.7.0'
|
||||
py36-170:
|
||||
python.version: '3.6'
|
||||
ort.version: '1.7.0'
|
||||
maxParallel: 1
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
addToPath: true
|
||||
steps:
|
||||
# needed for onnxruntime
|
||||
- script: brew install libomp
|
||||
displayName: 'Install omp'
|
||||
|
||||
- script: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install --upgrade setuptools
|
||||
python -m pip install -r requirements.txt
|
||||
displayName: Install requirements.txt
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
addToPath: true
|
||||
|
||||
- script: python -c "import onnxruntime;print(onnxruntime.__version__)"
|
||||
displayName: Check installation
|
||||
- script: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install --upgrade setuptools
|
||||
python -m pip install -r requirements.txt
|
||||
displayName: Install requirements.txt
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py develop
|
||||
displayName: Build the library and tests
|
||||
- script: python -c "import onnxruntime;print(onnxruntime.__version__)"
|
||||
displayName: Check installation
|
||||
|
||||
- script: python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements-dev.txt
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py develop
|
||||
displayName: Build the library and tests
|
||||
|
||||
- script: python -m pip install torch torchvision torchaudio
|
||||
displayName: Install pytorch
|
||||
- script: python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements-dev.txt
|
||||
|
||||
- script: python -m pytest test --verbose
|
||||
displayName: Run python test
|
||||
- script: python -m pip install torch torchvision torchaudio
|
||||
displayName: Install pytorch
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
- script: python -m pytest test --verbose
|
||||
displayName: Run python test
|
||||
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-macos-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
|
||||
#########
|
||||
# Windows C++
|
||||
#########
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-macos-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
|
||||
- job: WindowsC
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
#########
|
||||
# Windows C++
|
||||
#########
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
build.lib: '0'
|
||||
maxParallel: 1
|
||||
- job: WindowsC
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
|
||||
steps:
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-win-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
ortlib.version: '38443267'
|
||||
py38-160:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.6.0'
|
||||
ortlib.version: '34858191'
|
||||
build.lib: '0'
|
||||
maxParallel: 1
|
||||
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.zip'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
steps:
|
||||
- task: DownloadGitHubRelease@0
|
||||
inputs:
|
||||
connection: 'microsoft'
|
||||
userRepository: 'microsoft/onnxruntime'
|
||||
defaultVersionType: 'specificVersion'
|
||||
version: '$(ortlib.version)'
|
||||
itemPattern: '*-win-x64-$(ort.version)*'
|
||||
downloadPath: '$(Build.sourcesdirectory)'
|
||||
displayName: Download the ONNXRuntime prebuilt package.
|
||||
|
||||
- script: |
|
||||
call .\build.bat -DONNXRUNTIME_LIB_DIR=.\onnxruntime-win-x64-$(ort.version)\lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: '**/*.zip'
|
||||
destinationFolder: '$(Build.sourcesdirectory)'
|
||||
cleanDestinationFolder: false
|
||||
overwriteExistingFiles: true
|
||||
displayName: Unpack ONNXRuntime package.
|
||||
|
||||
- script: |
|
||||
cd out/Windows
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
- script: |
|
||||
call .\build.bat -DONNXRUNTIME_LIB_DIR=.\onnxruntime-win-x64-$(ort.version)\lib
|
||||
displayName: build the customop library with onnxruntime
|
||||
|
||||
################
|
||||
# Windows wheels
|
||||
################
|
||||
- script: |
|
||||
cd out/Windows
|
||||
ctest -C RelWithDebInfo
|
||||
displayName: Run C++ native tests
|
||||
|
||||
- job: WindowsWheel
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
################
|
||||
# Windows wheels
|
||||
################
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
py37-170:
|
||||
python.version: '3.7'
|
||||
ort.version: '1.7.0'
|
||||
py36-170:
|
||||
python.version: '3.6'
|
||||
ort.version: '1.7.0'
|
||||
maxParallel: 1
|
||||
- job: WindowsWheel
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
|
||||
steps:
|
||||
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
|
||||
displayName: Add conda to PATH
|
||||
strategy:
|
||||
matrix:
|
||||
py38-170:
|
||||
python.version: '3.8'
|
||||
ort.version: '1.7.0'
|
||||
py37-170:
|
||||
python.version: '3.7'
|
||||
ort.version: '1.7.0'
|
||||
py36-170:
|
||||
python.version: '3.6'
|
||||
ort.version: '1.7.0'
|
||||
maxParallel: 1
|
||||
|
||||
- script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy
|
||||
displayName: Create Anaconda environment
|
||||
steps:
|
||||
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
|
||||
displayName: Add conda to PATH
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements.txt
|
||||
- script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy
|
||||
displayName: Create Anaconda environment
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
displayName: Install pytorch
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install -r requirements-dev.txt
|
||||
displayName: Install requirements.txt
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
echo Test numpy installation... && python -c "import numpy"
|
||||
python setup.py develop
|
||||
displayName: Build the custom-op library
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
|
||||
displayName: Install pytorch
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pytest test
|
||||
displayName: Run python test
|
||||
- script: |
|
||||
call activate pyenv
|
||||
echo Test numpy installation... && python -c "import numpy"
|
||||
python setup.py develop
|
||||
displayName: Build the custom-op library
|
||||
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
python setup.py sdist
|
||||
python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python -m pytest test
|
||||
displayName: Run python test
|
||||
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-window-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
- script: |
|
||||
call activate pyenv
|
||||
python setup.py bdist_wheel
|
||||
# cmake produces assemblies before setup.py makes the
|
||||
# the list of files to include, second call solves that
|
||||
python setup.py sdist
|
||||
python setup.py bdist_wheel
|
||||
displayName: Build wheel
|
||||
|
||||
- task: PublishPipelineArtifact@0
|
||||
inputs:
|
||||
artifactName: 'wheel-window-$(python.version)'
|
||||
targetPath: 'dist'
|
||||
|
||||
#############
|
||||
# WebAssembly
|
||||
#############
|
||||
|
||||
- job: WebAssembly
|
||||
pool:
|
||||
vmImage: 'ubuntu-latest'
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
cd cmake
|
||||
mkdir deps
|
||||
cd deps
|
||||
git clone https://github.com/emscripten-core/emsdk
|
||||
emsdk/emsdk install latest
|
||||
emsdk/emsdk activate latest
|
||||
displayName: Setup emscripten pipeline
|
||||
|
||||
- script: |
|
||||
sh ./build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/deps/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DOCOS_ENABLE_SPM_TOKENIZER=OFF -DOCOS_ENABLE_PYTHON=OFF -DOCOS_ENABLE_CTEST=OFF
|
||||
displayName: build the customop library with onnxruntime
|
||||
# TODO add unittest for webassembly
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 2e7eaf7233144e5e25b1c1338890bbab5d011815
|
||||
1
setup.py
1
setup.py
|
|
@ -69,6 +69,7 @@ class BuildCMakeExt(_build_ext):
|
|||
cmake_args = [
|
||||
'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(ext_fullpath.parent.absolute()),
|
||||
'-DOCOS_ENABLE_PYTHON=ON',
|
||||
'-DOCOS_ENABLE_CTEST=OFF',
|
||||
'-DOCOS_EXTENTION_NAME=' + pathlib.Path(self.get_ext_filename(extension.name)).name,
|
||||
'-DCMAKE_BUILD_TYPE=' + config
|
||||
]
|
||||
|
|
|
|||
|
|
@ -49,7 +49,8 @@ class SpecialTokenMap {
|
|||
auto it = str.first.begin();
|
||||
size_t search_pos = 0;
|
||||
while (it != str.first.end()) {
|
||||
#if defined(__APPLE__)
|
||||
// work fine for all clang-based platform: Mac OS, Android, WebAssembly
|
||||
#if defined(__clang__)
|
||||
auto search_it = std::search(it, str.first.end(), st.str.begin(), st.str.end());
|
||||
#else
|
||||
auto search_it = std::search(it, str.first.end(),
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче