Add extensions catalyst support (#684)

* save work * add flags to disable build that's currently not supported on macabi * update * update * fix arg type * update ext_test.cmake * fix * try add flags * exclude ext_test * add missing downloading steps * nameing minor fix * update ios version etc. * fix * minor fix * minor fix * clean up * nuget package changes for catalyst extensions (#686) Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net> * update to net7.0 and add help comments --------- Co-authored-by: rachguo <rachguo@rachguos-Mac-mini.local> Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net>
2024-04-17 10:43:35 -07:00 · 2024-04-17 10:43:35 -07:00 · fe8cd9ee8d
--- a/.pipelines/templates/build-package-for-ios-cocoapods.yml
+++ b/.pipelines/templates/build-package-for-ios-cocoapods.yml
@ -16,6 +16,7 @@ jobs:
  parameters:
    Platform: 'iphoneos'
    IosArch: 'arm64'
+    IosVersion: '12.0'
    IsReleaseBuild: ${{parameters.IsReleaseBuild}}
    AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}

@ -23,6 +24,7 @@ jobs:
  parameters:
    Platform: 'iphonesimulator'
    IosArch: 'x86_64'
+    IosVersion: '12.0'
    IsReleaseBuild: ${{parameters.IsReleaseBuild}}
    AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}

@ -30,6 +32,23 @@ jobs:
  parameters:
    Platform: 'iphonesimulator'
    IosArch: 'arm64'
+    IosVersion: '12.0'
+    IsReleaseBuild: ${{parameters.IsReleaseBuild}}
+    AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
+  
+- template: ios-framework-build.yml
+  parameters:
+    Platform: 'maccatalyst'
+    IosArch: 'arm64'
+    IosVersion: '14.0'
+    IsReleaseBuild: ${{parameters.IsReleaseBuild}}
+    AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}
+
+- template: ios-framework-build.yml
+  parameters:
+    Platform: 'maccatalyst'
+    IosArch: 'x86_64'
+    IosVersion: '14.0'
    IsReleaseBuild: ${{parameters.IsReleaseBuild}}
    AdditionalBuildFlags: ${{parameters.AdditionalBuildFlags}}

@ -40,6 +59,8 @@ jobs:
  - IOS_C_API_Packaging_iphoneos_arm64
  - IOS_C_API_Packaging_iphonesimulator_arm64
  - IOS_C_API_Packaging_iphonesimulator_x86_64
+  - IOS_C_API_Packaging_maccatalyst_x86_64
+  - IOS_C_API_Packaging_maccatalyst_arm64
  pool:
    vmImage: "macOS-13"

@ -82,6 +103,16 @@ jobs:
      artifact: 'onnxruntime-extensions-ios-iphonesimulator-x86_64-lib'
      targetPath: '$(Build.BinariesDirectory)/'

+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: 'onnxruntime-extensions-ios-maccatalyst-x86_64-lib'
+      targetPath: '$(Build.BinariesDirectory)/'
+  
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: 'onnxruntime-extensions-ios-maccatalyst-arm64-lib'
+      targetPath: '$(Build.BinariesDirectory)/'
+      
  - script: |
      set -e -x
      ls
@ -107,6 +138,8 @@ jobs:
        --platform_arch iphonesimulator x86_64 \
        --platform_arch iphonesimulator arm64 \
        --platform_arch iphoneos arm64 \
+        --platform_arch maccatalyst x86_64 \
+        --platform_arch maccatalyst arm64 \
        --mode pack_xcframework_only
    displayName: "Pack xcframework for distributed building"

@ -114,7 +147,8 @@ jobs:
      python ./tools/ios/assemble_pod_package.py \
        --staging-dir $(Build.BinariesDirectory)/pod_staging \
        --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
-        --pod-version ${ORT_EXTENSIONS_POD_VERSION}
+        --pod-version ${ORT_EXTENSIONS_POD_VERSION} \
+        --mac_catalyst_enabled
    displayName: "Assemble pod"

  - script: |
--- a/.pipelines/templates/ios-framework-build.yml
+++ b/.pipelines/templates/ios-framework-build.yml
@ -8,6 +8,10 @@ parameters:
  displayName: IosArch
  type: string

+- name: IosVersion
+  displayName: IosVersion
+  type: string
+
 - name: IsReleaseBuild
  displayName: "Is this a release build?"
  type: boolean
@ -50,6 +54,7 @@ jobs:
            --config $(buildConfig) \
            --platform_arch  ${{ parameters.Platform }} ${{ parameters.IosArch }} \
            --mode build_platform_arch_frameworks_only \
+            --ios_deployment_target ${{ parameters.IosVersion}} \
            -- \
            ${{parameters.AdditionalBuildFlags}}
        displayName: "Build xcframework"
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -842,6 +842,10 @@ endif()

 if(OCOS_BUILD_APPLE_FRAMEWORK)
  include(ext_apple_framework)
+  if (MAC_CATALYST)
+    add_compile_options(-Wno-overriding-t-option)
+    add_link_options(-Wno-overriding-t-option)
+  endif()
 endif()

 # clean up the requirements.txt files from 3rd party project folder to suppress the code security false alarms
--- a/cmake/ext_tests.cmake
+++ b/cmake/ext_tests.cmake
@ -1,3 +1,5 @@
+if (NOT MAC_CATALYST)
+
 if (OCOS_ENABLE_SELECTED_OPLIST)
  # currently the tests don't handle operator exclusion cleanly.
  message(FATAL_ERROR "Due to usage of OCOS_ENABLE_SELECTED_OPLIST excluding operators the tests are unable to be built and run")
@ -200,3 +202,5 @@ else()
    endif()
  endblock()
 endif()
+
+endif()
--- a/nuget/DummyNativeNuget.nuspec
+++ b/nuget/DummyNativeNuget.nuspec
@ -51,19 +51,22 @@
    <file src="..\nuget-artifacts\onnxruntime-extensions-android\onnxruntime-extensions.aar" target="runtimes\android\native" />
    <file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\monoandroid11.0" />
    <file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\monoandroid11.0" />
-    <file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net6.0-android31.0" />
-    <file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net6.0-android31.0" />
+    <file src="targets\net7.0-android\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net7.0-android31.0" />
+    <file src="targets\net7.0-android\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net7.0-android31.0" />
    <!-- iOS via Xamarin/MAUI. -->
    <file src="..\nuget-artifacts\onnxruntime_extensions.xcframework\**" target="runtimes\ios\native\onnxruntime_extensions.xcframework" />
    <file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\xamarinios10" />
    <file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\xamarinios10" />
-    <file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net6.0-ios15.4" />
-    <file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net6.0-ios15.4" />
-    <!-- .net 6 macOS -->
+    <file src="targets\net7.0-ios\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net7.0-ios15.4" />
+    <file src="targets\net7.0-ios\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net7.0-ios15.4" />
+    <!-- .net 7 macOS -->
    <file src="..\nuget-artifacts\onnxruntime-extensions-osx-x86_64\lib\libortextensions.dylib" target="runtimes\osx.10.14-x64\native" />
    <file src="..\nuget-artifacts\onnxruntime-extensions-osx-arm64\lib\libortextensions.dylib" target="runtimes\osx.10.14-arm64\native"/>

-    <file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net6.0-macos12.3" />
-    <file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net6.0-macos12.3" />
+    <file src="targets\net7.0-macos\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="build\net7.0-macos12.3" />
+    <file src="targets\net7.0-macos\Microsoft.ML.OnnxRuntime.Extensions.Dummy.targets" target="buildTransitive\net7.0-macos12.3" />
+    <!-- .net 7 macCatalyst -->
+    <file src="targets\net7.0-maccatalyst\_._" target="build\net7.0-maccatalyst14.0" />
+    <file src="targets\net7.0-maccatalyst\_._" target="buildTransitive\net7.0-maccatalyst14.0" />
  </files>
 </package>
--- a/nuget/NativeNuget.nuspec
+++ b/nuget/NativeNuget.nuspec
@ -48,19 +48,22 @@
    <file src="..\nuget-artifacts\onnxruntime-extensions-android\onnxruntime-extensions.aar" target="runtimes\android\native" />
    <file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\monoandroid11.0" />
    <file src="targets\monoandroid11.0\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\monoandroid11.0" />
-    <file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-android31.0" />
-    <file src="targets\net6.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-android31.0" />
+    <file src="targets\net7.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net7.0-android31.0" />
+    <file src="targets\net7.0-android\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net7.0-android31.0" />
    <!-- iOS via Xamarin/MAUI. -->
    <file src="..\nuget-artifacts\onnxruntime_extensions.xcframework\**" target="runtimes\ios\native\onnxruntime_extensions.xcframework" />
    <file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\xamarinios10" />
    <file src="targets\xamarinios10\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\xamarinios10" />
-    <file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-ios15.4" />
-    <file src="targets\net6.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-ios15.4" />
-    <!-- .net 6 macOS -->
+    <file src="targets\net7.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net7.0-ios15.4" />
+    <file src="targets\net7.0-ios\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net7.0-ios15.4" />
+    <!-- .net 7 macOS -->
    <file src="..\nuget-artifacts\onnxruntime-extensions-osx-x86_64\lib\libortextensions.dylib" target="runtimes\osx.10.14-x64\native" />
    <file src="..\nuget-artifacts\onnxruntime-extensions-osx-arm64\lib\libortextensions.dylib" target="runtimes\osx.10.14-arm64\native"/>
    
-    <file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net6.0-macos12.3" />
-    <file src="targets\net6.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net6.0-macos12.3" />
+    <file src="targets\net7.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="build\net7.0-macos12.3" />
+    <file src="targets\net7.0-macos\Microsoft.ML.OnnxRuntime.Extensions.targets" target="buildTransitive\net7.0-macos12.3" />
+    <!-- .net 7 macCatalyst -->
+    <file src="targets\net7.0-maccatalyst\_._" target="build\net7.0-maccatalyst14.0" />
+    <file src="targets\net7.0-maccatalyst\_._" target="buildTransitive\net7.0-maccatalyst14.0" />
  </files>
 </package>
--- a/nuget/targets/net6.0-maccatalyst/_._
+++ b/nuget/targets/net6.0-maccatalyst/_._
--- a/tools/build.py
+++ b/tools/build.py
@ -204,6 +204,7 @@ def _parse_arguments():
        "--build_apple_framework", action="store_true", help="Build a macOS/iOS framework for ONNX Runtime Extensions."
    )
    parser.add_argument("--ios", action="store_true", help="build for iOS")
+    parser.add_argument("--macos", choices=["MacOSX", "Catalyst"], help="build for MacOS (OSX/Catalyst)")
    parser.add_argument(
        "--apple_sysroot",
        default="",
@ -497,9 +498,8 @@ def _generate_build_tree(
            # if neither code signing development identity nor team id are provided, don't code sign
            cmake_args += ["-DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO"]

-        if args.build_apple_framework or args.ios:
+        if args.build_apple_framework or args.ios or args.macos:
            cmake_args += ["-DOCOS_BUILD_APPLE_FRAMEWORK=ON"]
-    
            required_args = [
                args.apple_sysroot,
                args.apple_deploy_target,
@ -521,6 +521,23 @@ def _generate_build_tree(
                    "-DCMAKE_SYSTEM_NAME=iOS",
                    "-DCMAKE_TOOLCHAIN_FILE=" + str(args.ios_toolchain_file.resolve(strict=True)),
                ]
+            if args.macos == "Catalyst":
+                macabi_target = f"{args.apple_arch}-apple-ios{args.apple_deploy_target}-macabi"
+                cmake_args += [
+                    "-DCMAKE_CXX_COMPILER_TARGET=" + macabi_target,
+                    "-DCMAKE_C_COMPILER_TARGET=" + macabi_target,
+                    "-DCMAKE_CC_COMPILER_TARGET=" + macabi_target,
+                    f"-DCMAKE_CXX_FLAGS=--target={macabi_target}",
+                    f"-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}",
+                    f"-DCMAKE_C_FLAGS=--target={macabi_target}",
+                    f"-DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}",
+                    f"-DCMAKE_CC_FLAGS=--target={macabi_target}",
+                    f"-DCMAKE_CC_FLAGS_RELEASE=-O3 -DNDEBUG --target={macabi_target}",
+                    "-DOCOS_ENABLE_CV2=OFF",
+                    "-DOCOS_ENABLE_VISION=OFF",
+                    "-DOCOS_ENABLE_CTESTS=OFF",
+                    "-DMAC_CATALYST=1",
+                ]

    if args.wasm:
        emsdk_toolchain = (
--- a/tools/ios/assemble_pod_package.py
+++ b/tools/ios/assemble_pod_package.py
@ -60,7 +60,7 @@ def gen_file_from_template(


 def assemble_pod_package(
-    staging_dir: Path, xcframework_dir: Path, public_headers_dir: Path, xcframework_info_file: Path, pod_version: str
+    staging_dir: Path, xcframework_dir: Path, public_headers_dir: Path, xcframework_info_file: Path, pod_version: str, catalyst_enabled: bool,
 ):
    staging_dir = staging_dir.resolve()
    xcframework_dir = xcframework_dir.resolve(strict=True)
@ -90,7 +90,7 @@ def assemble_pod_package(
            for key in xcframework_info.keys():
                if key.startswith("iphone") and ios_deployment_target == "":
                    ios_deployment_target = xcframework_info[key]["apple_deployment_target"]
-                if "MacOSX" in key:
+                if "MacOSX" in key and not catalyst_enabled:
                    # Note: For key value for macos, it is directly using the path name from Xcode's MacOS SDK path.
                    macos_deployment_target = xcframework_info[key]["apple_deployment_target"]

@ -130,6 +130,12 @@ def parse_args():
        help="The pod's version.",
    )

+    parser.add_argument(
+        "--mac_catalyst_enabled",
+        action="store_true",
+        help="mac catalyst variants included in pods. Specify this argument when build targets contains catalyst archs. ",
+    )
+
    input_paths_group = parser.add_argument_group(description="Input path arguments.")

    input_paths_group.add_argument(
@ -180,6 +186,7 @@ def main():
        public_headers_dir=args.public_headers_dir,
        xcframework_info_file=args.xcframework_info_file,
        pod_version=args.pod_version,
+        catalyst_enabled=args.mac_catalyst_enabled,
    )


--- a/tools/ios/build_xcframework.py
+++ b/tools/ios/build_xcframework.py
@ -19,7 +19,14 @@ sys.path.insert(0, str(_repo_dir / "tools"))

 from utils import get_logger, run  # noqa

-_supported_platform_archs = {
+_all_supported_platform_archs = {
+    "iphoneos": ["arm64"],
+    "iphonesimulator": ["x86_64", "arm64"],
+    "macosx": ["x86_64", "arm64"],
+    "maccatalyst": ["x86_64", "arm64"],
+}
+
+_default_supported_platform_archs = {
    "iphoneos": ["arm64"],
    "iphonesimulator": ["x86_64", "arm64"],
    "macosx": ["x86_64", "arm64"],
@ -45,6 +52,7 @@ def _rmtree_if_existing(dir: Path):
    except FileNotFoundError:
        pass

+
 def _merge_framework_info_files(files: list[str], output_file: Path):
    merged_data = {}

@ -58,6 +66,7 @@ def _merge_framework_info_files(files: list[str], output_file: Path):
    with open(output_file, "w") as f:
        json.dump(merged_data, f, indent=2)

+
 def build_framework_for_platform_and_arch(
    build_dir: Path,
    platform: str,
@ -68,8 +77,12 @@ def build_framework_for_platform_and_arch(
    macos_deployment_target: str,
    other_build_args: list[str],
 ):
-    build_cmd = (
-        [
+    if platform == "maccatalyst":
+        apple_sysroot = "macosx"
+    else:
+        apple_sysroot = platform
+
+    build_cmd = [
        sys.executable,
        str(_repo_dir / "tools" / "build.py"),
        f"--build_dir={build_dir}",
@ -79,14 +92,13 @@ def build_framework_for_platform_and_arch(
        "--parallel",
        "--test",
        "--build_apple_framework",
-            f"--apple_sysroot={platform}",
+        f"--apple_sysroot={apple_sysroot}",
        f"--apple_arch={arch}",
    ]
-    )

    cmake_defines = []

-    if platform != "macosx":
+    if platform != "macosx" and platform != "maccatalyst":  #ios simulator or iphoneos platform
        cmake_defines += [
            # required by OpenCV CMake toolchain file
            # https://github.com/opencv/opencv/blob/4223495e6cd67011f86b8ecd9be1fa105018f3b1/platforms/ios/cmake/Toolchains/common-ios-toolchain.cmake#L64-L66
@ -107,11 +119,18 @@ def build_framework_for_platform_and_arch(
            f"--ios_toolchain_file={_get_opencv_toolchain_file(platform, opencv_dir)}",
            f"--apple_deploy_target={ios_deployment_target}",
        ]
-    else:
+    elif platform == "macosx":
        build_cmd += [
            # macOS options
+            "--macos=MacOSX",
            f"--apple_deploy_target={macos_deployment_target}",
        ]
+    else:
+        build_cmd += [
+            # mac catalyst options
+            "--macos=Catalyst",
+            f"--apple_deploy_target={ios_deployment_target}",
+        ]
    build_cmd += [f"--one_cmake_extra_define={cmake_define}" for cmake_define in cmake_defines]
    build_cmd += other_build_args

@ -297,16 +316,16 @@ def parse_args():
    # convert from [[platform1, arch1], [platform1, arch2], ...] to {platform1: [arch1, arch2, ...], ...}
    def platform_archs_from_args(platform_archs_arg: list[list[str]] | None) -> dict[str, list[str]]:
        if not platform_archs_arg:
-            return _supported_platform_archs.copy()
+            return _default_supported_platform_archs.copy()

        platform_archs = {}
        for platform, arch in platform_archs_arg:
            assert (
-                platform in _supported_platform_archs.keys()
-            ), f"Unsupported platform: '{platform}'. Valid values are {list(_supported_platform_archs.keys())}"
-            assert arch in _supported_platform_archs[platform], (
+                platform in _all_supported_platform_archs.keys()
+            ), f"Unsupported platform: '{platform}'. Valid values are {list(_all_supported_platform_archs.keys())}"
+            assert arch in _all_supported_platform_archs[platform], (
                f"Unsupported arch for platform '{platform}': '{arch}'. "
-                f"Valid values are {_supported_platform_archs[platform]}"
+                f"Valid values are {_all_supported_platform_archs[platform]}"
            )

            archs = platform_archs.setdefault(platform, [])
--- a/tutorials/demo4dotnet/ClipTokenizerTest/ClipTokenizerTestCS.csproj
+++ b/tutorials/demo4dotnet/ClipTokenizerTest/ClipTokenizerTestCS.csproj
@ -2,7 +2,7 @@

  <PropertyGroup>
    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <Platforms>x64</Platforms>