diff --git a/cmake/externals/sentencepieceproject.cmake b/cmake/externals/sentencepieceproject.cmake index 391d5177..940bc1d3 100644 --- a/cmake/externals/sentencepieceproject.cmake +++ b/cmake/externals/sentencepieceproject.cmake @@ -1,27 +1,36 @@ +# spm is abbreviation of sentencepiece to meet the path length limits on Windows + +set(spm_patches) +if(_ONNXRUNTIME_EMBEDDED) + # use protobuf provided by ORT + list(APPEND spm_patches "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject.protobuf.patch") +endif() +if(IOS) + # fix for iOS build + list(APPEND spm_patches "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject.ios.patch") +endif() + +set(spm_patch_command) +if(spm_patches) + set(spm_patch_command git checkout . && git apply --ignore-space-change --ignore-whitespace ${spm_patches}) +endif() + FetchContent_Declare( spm GIT_REPOSITORY https://github.com/google/sentencepiece.git GIT_TAG v0.1.96 + PATCH_COMMAND ${spm_patch_command} ) -# spm is abbr. of sentencepiece to meet the MAX_PATH compiling requirement on Windows FetchContent_GetProperties(spm) if(NOT spm_POPULATED) FetchContent_Populate(spm) - # need to patch sentencepiece to use protobuf provided by ort. This is ifdef onnxruntime_BUILD_WEBASSEMBLY - # for now but there is no real reason to not use it for all builds. - # 'git apply' should be in FetchContent_Declare() but that creates issues in sucessive builds so for now it is using execute_process(). - if(onnxruntime_BUILD_WEBASSEMBLY) - set(SPM_PATCH "${PROJECT_SOURCE_DIR}/cmake/externals/sentencepieceproject.patch") - message("-- sentencepiece: patching with ${SPM_PATCH}") - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace ${SPM_PATCH} WORKING_DIRECTORY ${spm_SOURCE_DIR}) - endif() add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL) set_target_properties(sentencepiece-static PROPERTIES FOLDER externals/google/sentencepiece) endif() -if (onnxruntime_BUILD_WEBASSEMBLY) +if(_ONNXRUNTIME_EMBEDDED) set(SPM_USE_BUILTIN_PROTOBUF OFF) set(spm_INCLUDE_DIRS ${REPO_ROOT}/cmake/external/protobuf/src diff --git a/cmake/externals/sentencepieceproject.ios.patch b/cmake/externals/sentencepieceproject.ios.patch new file mode 100644 index 00000000..a71cdcbf --- /dev/null +++ b/cmake/externals/sentencepieceproject.ios.patch @@ -0,0 +1,28 @@ +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 8b7fb76..c08b322 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -285,10 +285,19 @@ endif() + list(APPEND SPM_INSTALLTARGETS + spm_encode spm_decode spm_normalize spm_train spm_export_vocab) + +-install(TARGETS ${SPM_INSTALLTARGETS} +- RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} +- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++if (CMAKE_SYSTEM_NAME STREQUAL "iOS") ++ install(TARGETS ${SPM_INSTALLTARGETS} ++ BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR} ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++else() ++ install(TARGETS ${SPM_INSTALLTARGETS} ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ++ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++endif() ++ + install(FILES sentencepiece_trainer.h sentencepiece_processor.h + DESTINATION ${CMAKE_INSTALL_INCDIR}) + if (NOT SPM_USE_BUILTIN_PROTOBUF) diff --git a/cmake/externals/sentencepieceproject.patch b/cmake/externals/sentencepieceproject.protobuf.patch similarity index 100% rename from cmake/externals/sentencepieceproject.patch rename to cmake/externals/sentencepieceproject.protobuf.patch diff --git a/tools/ios/build_xcframework.py b/tools/ios/build_xcframework.py index 6491e5c8..65202ef1 100755 --- a/tools/ios/build_xcframework.py +++ b/tools/ios/build_xcframework.py @@ -63,11 +63,6 @@ def build_framework_for_platform_and_arch( f"-DCMAKE_OSX_ARCHITECTURES={arch}", f"-DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO", "-DOCOS_BUILD_APPLE_FRAMEWORK=ON", - # our version of sentencepiece doesn't support iOS build - # CMake Error at out/ios/RelWithDebInfo/_deps/spm-src/src/CMakeLists.txt:288 (install): - # install TARGETS given no BUNDLE DESTINATION for MACOSX_BUNDLE executable - # target "spm_encode". - "-DOCOS_ENABLE_SPM_TOKENIZER=OFF", # use OpenCV's CMake toolchain file f"-DCMAKE_TOOLCHAIN_FILE={_get_opencv_toolchain_file(platform, opencv_dir)}", # required by OpenCV CMake toolchain file