Add build.py to make it easier for developers to build different variants (#318)

* Add python based build infrastructure to simplify developer builds for various platforms. Majority was copied from the ORT build script so usage is consistent with that.

Left the existing build.bat/build.sh but ideally the CI can be updated to use the new infrastructure so things are more consistent.

Updated gradle to 7.5.1 and Android gradle tools to 7.3.0.

Validated Windows and cross-compiling Android on Windows including builds with explicitly selected ops.
WASM and iOS builds aren't tested yet and might need minor tweaks.

* Update build.py to require Python 3.7, remove git submodule sync, reorder options.
* Use 'cmake -E remove' to remove file.
* Enable specifying the ORT version to fetch
* Add ability to enable Java bindings.


Co-authored-by: Wenbing Li <10278425+wenbingl@users.noreply.github.com>
Co-authored-by: edgchen1 <18449977+edgchen1@users.noreply.github.com>

CMakeLists.txt

@@ -91,8 +91,11 @@ if(NOT OCOS_BUILD_PYTHON AND OCOS_ENABLE_PYTHON)
 endif()
 
 if(OCOS_BUILD_ANDROID)
-  if(NOT ANDROID_SDK_ROOT OR NOT ANDROID_NDK)
+  if(NOT CMAKE_TOOLCHAIN_FILE MATCHES "android.toolchain.cmake")
-    message("Cannot the find Android SDK/NDK")
+    message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE must be set to build/cmake/android.toolchain.cmake from the Android NDK.")
+  endif()
+  if(NOT ANDROID_NDK_VERSION OR NOT ANDROID_PLATFORM OR NOT ANDROID_ABI)
+    message(FATAL_ERROR "The Android platform, ABI and NDK version must be specified")
   endif()
 
   set(OCOS_BUILD_JAVA ON CACHE INTERNAL "")
@@ -459,16 +462,10 @@ endif()
 
 # clean up the requirements.txt files from 3rd party project folder to suppress the code security false alarms
 file(GLOB_RECURSE NO_USE_FILES ${CMAKE_BINARY_DIR}/_deps/*requirements.txt)
-message(STATUS "Found the follow requirements.txt: ${NO_USE_FILES}")
+message(STATUS "Found the following requirements.txt: ${NO_USE_FILES}")
 
 foreach(nf ${NO_USE_FILES})
-  file(TO_NATIVE_PATH ${nf} nf_native)
+  execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${nf})
-
-  if(CMAKE_SYSTEM_NAME MATCHES "Windows")
-    execute_process(COMMAND cmd /c "del ${nf_native}")
-  else()
-    execute_process(COMMAND bash -c "rm ${nf_native}")
-  endif()
 endforeach()
 
 # test section

build.android

@@ -39,7 +39,6 @@ pushd ${target_dir}
 #       the great change of file system permission in Android 7
 cmake "$@" \
     -DCMAKE_TOOLCHAIN_FILE=${NDK_ROOT}/build/cmake/android.toolchain.cmake  \
-    -DANDROID_NDK=${NDK_ROOT}                                               \
     -DANDROID_ABI=${abi_name}                                               \
     -DANDROID_PLATFORM=android-24                                           \
     -DANDROID_NDK_VERSION=${CURRENT_NDK_VERSION}                            \

build_lib.bat

@@ -0,0 +1,4 @@
+:: Copyright (c) Microsoft Corporation. All rights reserved.
+:: Licensed under the MIT License.
+
+python %~dp0\tools\build.py %*

build_lib.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Get directory this script is in
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+OS=$(uname -s)
+
+python3 $DIR/tools/build.py "$@"

cmake/ext_ortlib.cmake

@@ -2,6 +2,7 @@ if(_ONNXRUNTIME_EMBEDDED)
   set(ONNXRUNTIME_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/../include/onnxruntime/core/session)
   set(ONNXRUNTIME_LIB_DIR "")
 else()
+  # default to 1.10.0 if not specified
   set(ONNXRUNTIME_VER "1.10.0" CACHE STRING "ONNX Runtime version")
 
   if(CMAKE_HOST_APPLE)
@@ -31,6 +32,7 @@ else()
     onnxruntime
     URL https://github.com/microsoft/onnxruntime/releases/download/${ONNXRUNTIME_URL}
   )
+
   FetchContent_makeAvailable(onnxruntime)
   set(ONNXRUNTIME_INCLUDE_DIR ${onnxruntime_SOURCE_DIR}/include)
   set(ONNXRUNTIME_LIB_DIR ${onnxruntime_SOURCE_DIR}/lib)

cmake/ortext_ios.toolchain.cmake

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+set(CMAKE_SYSTEM_NAME iOS)
+if (NOT DEFINED CMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM AND NOT DEFINED CMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY)
+  set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED NO)
+endif()
+
+SET(CMAKE_XCODE_ATTRIBUTE_CLANG_ENABLE_MODULES "YES")
+SET(CMAKE_XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC "YES")

java/build-android.gradle

@@ -28,13 +28,14 @@ project.group = "com.microsoft.onnxruntime"
 def mavenArtifactId = project.name + '-android'
 def defaultDescription = 'ONNXRuntime-Extensions is an extension of onnxruntime to support pre- and post-processing.'
 
+
 buildscript {
 	repositories {
 		google()
 		mavenCentral()
 	}
 	dependencies {
-		classpath 'com.android.tools.build:gradle:4.0.1'
+		classpath 'com.android.tools.build:gradle:7.3.0'
 
 		// NOTE: Do not place your application dependencies here; they belong
 		// in the individual module build.gradle files

java/build.gradle

@@ -7,6 +7,7 @@ plugins {
 }
 
 allprojects {
+
 	repositories {
 		mavenCentral()
 	}
@@ -59,7 +60,7 @@ task javadocJar(type: Jar, dependsOn: javadoc) {
 }
 
 wrapper {
-	gradleVersion = '6.1.1'
+	gradleVersion = '7.5.1'
 }
 
 spotless {

java/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

java/gradlew

@@ -1,7 +1,7 @@
-#!/usr/bin/env sh
+#!/bin/sh
 
 #
-# Copyright 2015 the original author or authors.
+# Copyright © 2015-2021 the original authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,67 +17,101 @@
 #
 
 ##############################################################################
-##
+#
-##  Gradle start up script for UN*X
+#   Gradle start up script for POSIX generated by Gradle.
-##
+#
+#   Important for running:
+#
+#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+#       noncompliant, but you have some other compliant shell such as ksh or
+#       bash, then to run this script, type that shell name before the whole
+#       command line, like:
+#
+#           ksh Gradle
+#
+#       Busybox and similar reduced shells will NOT work, because this script
+#       requires all of these POSIX shell features:
+#         * functions;
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#         * compound commands having a testable exit status, especially «case»;
+#         * various built-in commands including «command», «set», and «ulimit».
+#
+#   Important for patching:
+#
+#   (2) This script targets any POSIX shell, so it avoids extensions provided
+#       by Bash, Ksh, etc; in particular arrays are avoided.
+#
+#       The "traditional" practice of packing multiple parameters into a
+#       space-separated string is a well documented source of bugs and security
+#       problems, so this is (mostly) avoided, by progressively accumulating
+#       options in "$@", and eventually passing that to Java.
+#
+#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+#       see the in-line comments for details.
+#
+#       There are tweaks for specific operating systems such as AIX, CygWin,
+#       Darwin, MinGW, and NonStop.
+#
+#   (3) This script is generated from the Groovy template
+#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       within the Gradle project.
+#
+#       You can find Gradle at https://github.com/gradle/gradle/.
+#
 ##############################################################################
 
 # Attempt to set APP_HOME
+
 # Resolve links: $0 may be a link
-PRG="$0"
+app_path=$0
-# Need this for relative symlinks.
+
-while [ -h "$PRG" ] ; do
+# Need this for daisy-chained symlinks.
-    ls=`ls -ld "$PRG"`
+while
-    link=`expr "$ls" : '.*-> \(.*\)$'`
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
-    if expr "$link" : '/.*' > /dev/null; then
+    [ -h "$app_path" ]
-        PRG="$link"
+do
-    else
+    ls=$( ls -ld "$app_path" )
-        PRG=`dirname "$PRG"`"/$link"
+    link=${ls#*' -> '}
-    fi
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
 done
-SAVED="`pwd`"
+
-cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
+APP_BASE_NAME=${0##*/}
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
+MAX_FD=maximum
 
 warn () {
     echo "$*"
-}
+} >&2
 
 die () {
     echo
     echo "$*"
     echo
     exit 1
-}
+} >&2
 
 # OS specific support (must be 'true' or 'false').
 cygwin=false
 msys=false
 darwin=false
 nonstop=false
-case "`uname`" in
+case "$( uname )" in                #(
-  CYGWIN* )
+  CYGWIN* )         cygwin=true  ;; #(
-    cygwin=true
+  Darwin* )         darwin=true  ;; #(
-    ;;
+  MSYS* | MINGW* )  msys=true    ;; #(
-  Darwin* )
+  NONSTOP* )        nonstop=true ;;
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-  NONSTOP* )
-    nonstop=true
-    ;;
 esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
@@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
         # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
+        JAVACMD=$JAVA_HOME/jre/sh/java
     else
-        JAVACMD="$JAVA_HOME/bin/java"
+        JAVACMD=$JAVA_HOME/bin/java
     fi
     if [ ! -x "$JAVACMD" ] ; then
         die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
@@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
     fi
 else
-    JAVACMD="java"
+    JAVACMD=java
     which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
@@ -106,80 +140,101 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
+    case $MAX_FD in #(
-    if [ $? -eq 0 ] ; then
+      max*)
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+        MAX_FD=$( ulimit -H -n ) ||
-            MAX_FD="$MAX_FD_LIMIT"
+            warn "Could not query maximum file descriptor limit"
-        fi
+    esac
-        ulimit -n $MAX_FD
+    case $MAX_FD in  #(
-        if [ $? -ne 0 ] ; then
+      '' | soft) :;; #(
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
+      *)
-        fi
+        ulimit -n "$MAX_FD" ||
-    else
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
-    JAVACMD=`cygpath --unix "$JAVACMD"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=`expr $i + 1`
-    done
-    case $i in
-        0) set -- ;;
-        1) set -- "$args0" ;;
-        2) set -- "$args0" "$args1" ;;
-        3) set -- "$args0" "$args1" "$args2" ;;
-        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
     esac
 fi
 
-# Escape application args
+# Collect all arguments for the java command, stacking in reverse order:
-save () {
+#   * args from the command line
-    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+#   * the main class name
-    echo " "
+#   * -classpath
-}
+#   * -D...appname settings
-APP_ARGS=`save "$@"`
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
 
-# Collect all arguments for the java command, following the shell quoting and substitution rules
+# For Cygwin or MSYS, switch paths to Windows format before running java
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+# Collect all arguments for the java command;
+#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+#     shell script including quotes and variable substitutions, so put them in
+#     double quotes to make sure that they get re-expanded; and
+#   * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
 
 exec "$JAVACMD" "$@"

test/NuGet.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+  <packageSources>
+    <clear />
+    <add key="NuGet Official" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+</configuration>

test/codeconv.runsettings

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- File name extension must be .runsettings -->
+<RunSettings>
+<GoogleTestAdapterSettings>
+		<SolutionSettings>
+			<Settings>
+			<OutputMode>Verbose</OutputMode>
+			<TestDiscoveryTimeoutInSeconds>120</TestDiscoveryTimeoutInSeconds>
+			</Settings>
+		</SolutionSettings>
+	</GoogleTestAdapterSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0" assemblyQualifiedName="Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+        <Configuration>
+          <CodeCoverage>
+            <!-- Match fully qualified names of functions: -->
+            <!-- (Use "\." to delimit namespaces in C# or Visual Basic, "::" in C++.)  -->
+            <Functions>
+              <Exclude>
+                <Function>^std::.*</Function>
+                <Function>^ATL::.*</Function>
+                <Function>^gsl::.*</Function>
+                <Function>^google::.*</Function>
+                <Function>^onnx::.*</Function>
+                <Function>^nlohmann::.*</Function>
+                <Function>^Microsoft::Featurizer.*</Function>
+                <Function>^Eigen::.*</Function>
+                <Function>^onnxruntime::test::.*</Function>
+                <Function>.*::__GetTestMethodInfo.*</Function>
+                <Function>^Microsoft::VisualStudio::CppCodeCoverageFramework::.*</Function>
+                <Function>^Microsoft::VisualStudio::CppUnitTestFramework::.*</Function>
+              </Exclude>
+            </Functions>
+            <!-- Match the path of the source files in which each method is defined: -->
+            <Sources>
+              <Include>
+                <Source>.*\\includes\\.*</Source>
+                <Source>.*\\shared\\.*</Source>
+                <Source>.*\\operators\\.*</Source>
+              </Include>
+              <Exclude>
+                <Source>.*\\atlmfc\\.*</Source>
+                <Source>.*\\vctools\\.*</Source>
+                <Source>.*\\public\\sdk\\.*</Source>
+                <Source>.*\\microsoft sdks\\.*</Source>
+                <Source>.*\\vc\\include\\.*</Source>
+                <Source>.*\\vc\\tools\\msvc\\.*</Source>
+                <Source>.*\\cmake\\.*</Source>
+              </Exclude>
+            </Sources>
+          </CodeCoverage>
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>

test/packages.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="GoogleTestAdapter" version="0.18.0" targetFramework="net46" />
+</packages>

test/shared_test/test_main.cc

@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+
+#include <filesystem>
+#include <fstream>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#define TEST_MAIN main
+
+#if defined(__APPLE__)
+  #include <TargetConditionals.h>
+  #if TARGET_OS_SIMULATOR || TARGET_OS_IOS
+    #undef TEST_MAIN
+    #define TEST_MAIN main_no_link_  // there is a UI test app for iOS.
+  #endif
+#endif
+
+// currently this is the only place with a try/catch. Move the macros to common code if that changes.
+#ifdef OCOS_NO_EXCEPTIONS
+#define OCOS_TRY if (true)
+#define OCOS_CATCH(x) else if (false)
+#define OCOS_RETHROW
+// In order to ignore the catch statement when a specific exception (not ... ) is caught and referred
+// in the body of the catch statements, it is necessary to wrap the body of the catch statement into
+// a lambda function. otherwise the exception referred will be undefined and cause build break
+#define OCOS_HANDLE_EXCEPTION(func)
+#else
+#define OCOS_TRY try
+#define OCOS_CATCH(x) catch (x)
+#define OCOS_RETHROW throw;
+#define OCOS_HANDLE_EXCEPTION(func) func()
+#endif
+
+
+namespace {
+void FixCurrentDir() {
+  // adjust for the Google Test Adapter in Visual Studio not setting the current path to $(ProjectDir),
+  // which results in us being 2 levels below where the `data` folder is copied to and where the extensions
+  // library is
+  auto cur = std::filesystem::current_path();
+
+  do {
+    auto data_dir = cur / "data";
+
+    if (std::filesystem::exists(data_dir) && std::filesystem::is_directory(data_dir)) {
+      break;
+    }
+
+    cur = cur.parent_path();
+    ASSERT_NE(cur, cur.root_path()) << "Reached root directory without finding 'data' directory.";
+  } while (true);
+
+  // set current path as the extensions library is also loaded from that directory by TestInference
+  std::filesystem::current_path(cur);
+}
+}  // namespace
+
+int TEST_MAIN(int argc, char** argv) {
+  int status = 0;
+
+  OCOS_TRY {
+    ::testing::InitGoogleTest(&argc, argv);
+
+    FixCurrentDir();
+    status = RUN_ALL_TESTS();
+  }
+  OCOS_CATCH(const std::exception& ex) {
+    OCOS_HANDLE_EXCEPTION([&]() {
+      std::cerr << ex.what();
+      status = -1;
+    });
+  }
+
+  return status;
+}

test/shared_test/test_ortops_vision.cc

@@ -4,12 +4,13 @@
 #include <filesystem>
 #include <fstream>
 #include <vector>
+
 #include "gtest/gtest.h"
+#include "opencv2/imgcodecs.hpp"
+
 #include "ocos.h"
 #include "test_kernel.hpp"
 
-#include <opencv2/imgcodecs.hpp>
-
 namespace {
 std::vector<uint8_t> LoadBytesFromFile(const std::filesystem::path& filename) {
   using namespace std;
@@ -23,34 +24,11 @@ std::vector<uint8_t> LoadBytesFromFile(const std::filesystem::path& filename) {
 
   return input_bytes;
 }
-
-void FixCurrentDir() {
-  // adjust for the Google Test Adapter in Visual Studio not setting the current path to $(ProjectDir),
-  // which results in us being 2 levels below where the `data` folder is copied to and where the extensions
-  // library is
-  auto cur = std::filesystem::current_path();
-
-  do {
-    auto data_dir = cur / "data";
-
-    if (std::filesystem::exists(data_dir) && std::filesystem::is_directory(data_dir)) {
-      break;
-    }
-
-    cur = cur.parent_path();
-    ASSERT_NE(cur, cur.root_path()) << "Reached root directory without finding 'data' directory.";
-  } while (true);
-
-  // set current path as the extensions library is also loaded from that directory by TestInference
-  std::filesystem::current_path(cur);
-}
 }  // namespace
 
 // Test DecodeImage and EncodeImage by providing a jpg image. Model will decode to BGR, encode to PNG and decode
 // again to BGR. We validate that the BGR output from that matches the original image.
 TEST(VisionOps, image_decode_encode) {
-  FixCurrentDir();
-
   std::string ort_version{OrtGetApiBase()->GetVersionString()};
 
   // the test model requires ONNX opset 16, which requires ORT version 1.11 or later.

tools/build.py

@@ -0,0 +1,652 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import os
+import platform
+import shlex
+import shutil
+import sys
+
+from pathlib import Path
+from typing import List, Set
+
+SCRIPT_DIR = Path(__file__).parent
+REPO_DIR = SCRIPT_DIR.parent
+sys.path.insert(0, str(SCRIPT_DIR / "utils"))
+
+from utils import get_logger, is_linux, is_macOS, is_windows, run  # noqa: E402
+
+log = get_logger("build")
+
+
+class UsageError(Exception):
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
+def _check_python_version():
+    if (sys.version_info.major, sys.version_info.minor) < (3, 7):
+        raise UsageError("Invalid Python version. At least Python 3.7 is required. "
+                         f"Actual Python version: {sys.version}")
+
+
+_check_python_version()
+
+
+def _parse_arguments():
+    class Parser(argparse.ArgumentParser):
+        # override argument file line parsing behavior - allow multiple arguments per line and handle quotes
+        def convert_arg_line_to_args(self, arg_line):
+            return shlex.split(arg_line)
+
+    parser = Parser(
+        description="ONNXRuntime Extensions Shared Library build driver.",
+        usage="""
+        There are 3 phases which can be individually selected.
+
+        The Update (--update) phase will run CMake to generate makefiles.
+        The Build (--build) phase will build all projects.
+        The Test (--test) phase will run all unit tests.
+
+        Default behavior is --update --build --test for native architecture builds.
+        Default behavior is --update --build for cross-compiled builds.
+
+        If phases are explicitly specified only those phases will be run.
+          e.g. run with `--build` to rebuild without running the update or test phases
+        """,
+
+        # files containing arguments can be specified on the command line with "@<filename>" and the arguments within
+        # will be included at that point
+        fromfile_prefix_chars="@",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    # Main arguments
+    parser.add_argument("--build_dir", type=Path,
+                        # We set the default programmatically as it needs to take into account whether we're
+                        # cross-compiling
+                        help="Path to the build directory. Defaults to 'build/<target platform>'")
+    parser.add_argument("--config", nargs="+", default=["Debug"],
+                        choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"],
+                        help="Configuration(s) to build.")
+
+    # Build phases
+    parser.add_argument("--update", action="store_true", help="Update makefiles.")
+
+    parser.add_argument("--build", action="store_true", help="Build.")
+
+    parser.add_argument("--test", action="store_true", help="Run tests.")
+    parser.add_argument("--skip_tests", action="store_true", help="Skip all tests. Overrides --test.")
+
+    parser.add_argument("--clean", action="store_true",
+                        help="Run 'cmake --build --target clean' for the selected config/s.")
+    # Build phases end
+
+    parser.add_argument("--parallel", nargs="?", const="0", default="1", type=int,
+                        help="Use parallel build. The optional value specifies the maximum number of parallel jobs. "
+                             "If the optional value is 0 or unspecified, it is interpreted as the number of CPUs.")
+
+    parser.add_argument("--cmake_extra_defines", nargs="+", action="append",
+                        help="Extra definitions to pass to CMake during build system generation. "
+                             "These are essentially CMake -D options without the leading -D. "
+                             "Multiple name=value defines can be specified, with each separated by a space. "
+                             "Quote the name and value if the value contains spaces. "
+                             "The cmake_extra_defines can also be specified multiple times. "
+                             "  e.g. --cmake_extra_defines \"Name1=the value\" Name2=value2")
+
+    # Test options
+    parser.add_argument("--enable_cxx_tests", action="store_true", help="Enable the C++ unit tests.")
+    parser.add_argument("--cxx_code_coverage", action="store_true",
+                        help="Run C++ unit tests using vstest.exe to produce code coverage output. Windows only.")
+
+    parser.add_argument("--onnxruntime_version", type=str,
+                        help="ONNX Runtime version to fetch for headers and library. Default is 1.10.0.")
+    parser.add_argument("--onnxruntime_lib_dir", type=Path,
+                        help="Path to directory containing the pre-built ONNX Runtime library if you do not want to "
+                             "use the library from the ONNX Runtime release package that is fetched by default.")
+    # Build for ARM
+    parser.add_argument("--arm", action="store_true",
+                        help="[cross-compiling] Create ARM makefiles. Requires --update and no existing cache "
+                             "CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm64", action="store_true",
+                        help="[cross-compiling] Create ARM64 makefiles. Requires --update and no existing cache "
+                             "CMake setup. Delete CMakeCache.txt if needed")
+    parser.add_argument("--arm64ec", action="store_true",
+                        help="[cross-compiling] Create ARM64EC makefiles. Requires --update and no existing cache "
+                             "CMake setup. Delete CMakeCache.txt if needed")
+
+    # Android options
+    parser.add_argument("--android", action="store_true", help="Build for Android")
+    parser.add_argument("--android_abi", default="arm64-v8a", choices=["armeabi-v7a", "arm64-v8a", "x86", "x86_64"],
+                        help="Specify the target Android Application Binary Interface (ABI)")
+    parser.add_argument("--android_api", type=int, default=27, help="Android API Level, e.g. 21")
+    parser.add_argument("--android_home", type=Path, default=os.environ.get("ANDROID_HOME"),
+                        help="Path to the Android SDK.")
+    parser.add_argument("--android_ndk_path", type=Path, default=os.environ.get("ANDROID_NDK_HOME"),
+                        help="Path to the Android NDK. Typically `<Android SDK>/ndk/<ndk_version>")
+
+    # macOS/iOS options
+    parser.add_argument("--build_apple_framework", action="store_true",
+                        help="Build a macOS/iOS framework for the ONNXRuntime.")
+    parser.add_argument("--ios", action="store_true", help="build for iOS")
+    parser.add_argument("--ios_sysroot", default="",
+                        help="Specify the name of the platform SDK to be used. e.g. iphoneos, iphonesimulator")
+    parser.add_argument("--ios_toolchain_file", default=f"{REPO_DIR}/cmake/ortext_ios.toolchain.cmake", type=Path,
+                        help="Path to ios toolchain file. Default is <repo>/cmake/ortext_ios.toolchain.cmake")
+    parser.add_argument("--xcode_code_signing_team_id", default="",
+                        help="The development team ID used for code signing in Xcode")
+    parser.add_argument("--xcode_code_signing_identity", default="",
+                        help="The development identity used for code signing in Xcode")
+    parser.add_argument("--osx_arch", default="arm64" if platform.machine() == "arm64" else "x86_64",
+                        choices=["arm64", "arm64e", "x86_64"],
+                        help="Specify the Target specific architectures for macOS and iOS. "
+                             "This is only supported on macOS")
+    parser.add_argument("--apple_deploy_target", type=str,
+                        help="Specify the minimum version of the target platform (e.g. macOS or iOS). "
+                             "This is only supported on macOS")
+
+    # WebAssembly options
+    parser.add_argument("--wasm", action="store_true", help="Build for WebAssembly")
+    parser.add_argument("--emsdk_path", type=Path,
+                        help="Specify path to emscripten SDK. Setup manually with: "                         
+                             "  git clone https://github.com/emscripten-core/emsdk")
+    parser.add_argument("--emsdk_version", default="3.1.26", help="Specify version of emsdk")
+
+    # x86 args
+    parser.add_argument("--x86", action="store_true",
+                        help="[cross-compiling] Create Windows x86 makefiles. Requires --update and no existing cache "
+                             "CMake setup. Delete CMakeCache.txt if needed")
+
+    # Arguments needed by CI
+    parser.add_argument("--cmake_path", default="cmake", type=Path, help="Path to the CMake program.")
+    parser.add_argument("--ctest_path", default="ctest", type=Path, help="Path to the CTest program.")
+
+    parser.add_argument("--cmake_generator",
+                        choices=["Visual Studio 16 2019", "Visual Studio 17 2022", "Ninja", "Unix Makefiles", "Xcode"],
+                        default="Visual Studio 17 2022" if is_windows() else "Xcode" if is_macOS() else None,
+                        help="Specify the generator that CMake invokes to override the default.")
+
+    # Binary size reduction options
+    parser.add_argument("--include_ops_by_config", type=Path,
+                        help="Only include ops specified in the build that are listed in this config file. "
+                             "Format of config file is `domain;opset;op1,op2,... "
+                             "  e.g. com.microsoft.extensions;1;ImageDecode,ImageEncode")
+
+    parser.add_argument("--disable_exceptions", action="store_true",
+                        help="Disable exceptions to reduce binary size.")
+
+    # Language bindings
+    parser.add_argument("--build_java", action="store_true", help="Build Java bindings.")
+
+    args = parser.parse_args()
+
+    if not args.build_dir:
+        target_sys = platform.system()
+
+        # override if we're cross-compiling
+        if args.android:
+            target_sys = "Android"
+        elif args.ios:
+            target_sys = "iOS"
+        elif args.arm:
+            target_sys = "arm"
+        elif args.arm64:
+            target_sys = "arm64"
+        elif args.arm64ec:
+            target_sys = "arm64ec"
+        elif platform.system() == "Darwin":
+            # also tweak name for mac builds
+            target_sys = "macOS"
+        elif args.wasm:
+            target_sys = "wasm"
+
+        args.build_dir = Path("build/" + target_sys)
+
+    return args
+
+
+def _is_reduced_ops_build(args):
+    return args.include_ops_by_config is not None
+
+
+def _resolve_executable_path(command_or_path: Path):
+    """Returns the absolute path of an executable."""
+
+    exe_path = None
+    if command_or_path:
+        executable_path = shutil.which(str(command_or_path))
+        if executable_path is None:
+            raise UsageError(f"Failed to resolve executable path for '{command_or_path}'.")
+
+        exe_path = Path(executable_path)
+
+    return exe_path
+
+
+def _get_build_config_dir(build_dir: Path, config: str):
+    # build directory per configuration
+    return build_dir / config
+
+
+def _run_subprocess(args: List[str], cwd: Path = None, capture_stdout=False, shell=False, env=None,
+                    python_path: Path = None):
+
+    if isinstance(args, str):
+        raise ValueError("args should be a sequence of strings, not a string")
+
+    if env is None:
+        env = {}
+
+    my_env = os.environ.copy()
+
+    if python_path:
+        python_path = str(python_path.resolve())
+        if "PYTHONPATH" in my_env:
+            my_env["PYTHONPATH"] += os.pathsep + python_path
+        else:
+            my_env["PYTHONPATH"] = python_path
+
+    my_env.update(env)
+
+    return run(*args, cwd=cwd, capture_stdout=capture_stdout, shell=shell, env=my_env)
+
+
+def _flatten_arg_list(nested_list: List[List[str]]):
+    return [i for j in nested_list for i in j] if nested_list else []
+
+
+def _is_cross_compiling_on_apple(args):
+    if is_macOS():
+        return args.ios or args.osx_arch != platform.machine()
+
+    return False
+
+
+def _validate_cxx_test_args(args):
+    ort_lib_dir = None
+    if args.onnxruntime_lib_dir:
+        ort_lib_dir = args.onnxruntime_lib_dir.resolve(strict=True)
+        if not ort_lib_dir.is_dir():
+            raise UsageError("onnxruntime_lib_dir must be a directory")
+
+    return ort_lib_dir
+
+
+def _generate_selected_ops_config(config_file: Path):
+    config_file.resolve(strict=True)
+    script = REPO_DIR / "tools" / "gen_selectedops.py"
+    _run_subprocess([sys.executable, str(script), str(config_file)])
+
+
+def _setup_emscripten(args):
+    if not args.emsdk_path:
+        raise UsageError("emsdk_path must be specified for wasm build")
+
+    emsdk_file = str((args.emsdk_path / ("emsdk.bat" if is_windows() else "emsdk")).resolve(strict=True))
+
+    log.info("Installing emsdk...")
+    _run_subprocess([emsdk_file, "install", args.emsdk_version], cwd=args.emsdk_path)
+    log.info("Activating emsdk...")
+    _run_subprocess([emsdk_file, "activate", args.emsdk_version], cwd=args.emsdk_path)
+
+
+def _generate_build_tree(cmake_path: Path,
+                         source_dir: Path,
+                         build_dir: Path,
+                         configs: Set[str],
+                         cmake_extra_defines: List[str],
+                         args,
+                         cmake_extra_args: List[str]
+                         ):
+    log.info("Generating CMake build tree")
+
+    cmake_args = [
+        str(cmake_path),
+        str(source_dir),
+        # Define Python_EXECUTABLE so find_package(python3 ...) will use the same version of python being used to
+        # run this script
+        "-DPython_EXECUTABLE=" + sys.executable,
+        "-DOCOS_ENABLE_SELECTED_OPLIST=" + ("ON" if _is_reduced_ops_build(args) else "OFF"),
+    ]
+
+    if args.onnxruntime_version:
+        cmake_args.append(f"-DONNXRUNTIME_VER={args.onnxruntime_version}")
+
+    if args.enable_cxx_tests:
+        cmake_args.append("-DOCOS_ENABLE_CTEST=ON")
+        ort_lib_dir = _validate_cxx_test_args(args)
+        if ort_lib_dir:
+            cmake_args.append(f"-DONNXRUNTIME_LIB_DIR={str(ort_lib_dir)}")
+
+    if args.android:
+        if not args.android_ndk_path:
+            raise UsageError("android_ndk_path is required to build for Android")
+        if not args.android_home:
+            raise UsageError("android_home is required to build for Android")
+
+        android_home = args.android_home.resolve(strict=True)
+        android_ndk_path = args.android_ndk_path.resolve(strict=True)
+
+        if not android_home.is_dir() or not android_ndk_path.is_dir():
+            raise UsageError("Android home and NDK paths must be directories.")
+
+        ndk_version = android_ndk_path.name  # NDK version is inferred from the folder name
+
+        cmake_args += [
+            "-DOCOS_BUILD_ANDROID=ON",
+            "-DCMAKE_TOOLCHAIN_FILE="
+            + str((args.android_ndk_path / "build" / "cmake" / "android.toolchain.cmake").resolve(strict=True)),
+            "-DANDROID_NDK_VERSION=" + str(ndk_version),
+            "-DANDROID_PLATFORM=android-" + str(args.android_api),
+            "-DANDROID_ABI=" + str(args.android_abi)
+        ]
+
+    if is_macOS():
+        cmake_args.append("-DOCOS_BUILD_APPLE_FRAMEWORK=" + ("ON" if args.build_apple_framework else "OFF"))
+
+    if args.ios:
+        required_args = [
+            args.ios_sysroot,
+            args.apple_deploy_target,
+        ]
+
+        arg_names = [
+            "--ios_sysroot          " + "<the location or name of the macOS platform SDK>",
+            "--apple_deploy_target  " + "<the minimum version of the target platform>",
+        ]
+
+        if not all(required_args):
+            raise UsageError("iOS build on MacOS canceled due to missing required arguments: "
+                             + ", ".join(val for val, cond in zip(arg_names, required_args) if not cond))
+
+        cmake_args += [
+            "-DCMAKE_SYSTEM_NAME=iOS",
+            "-DCMAKE_OSX_SYSROOT=" + args.ios_sysroot,
+            "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + args.apple_deploy_target,
+            "-DCMAKE_TOOLCHAIN_FILE=" + str(args.ios_toolchain_file.resolve(strict=True)),
+        ]
+
+    if args.wasm:
+        emsdk_toolchain = (args.emsdk_path / "upstream" / "emscripten" / "cmake" / "Modules" / "Platform" /
+                          "Emscripten.cmake").resolve()
+        if not emsdk_toolchain.exists():
+            raise UsageError(f"Emscripten toolchain file was not found at {str(emsdk_toolchain)}")
+
+        # some things aren't currently supported with wasm so disable
+        # TODO: Might be cleaner to do a selected ops build and enable/disable things via that.
+        #       For now replicating the config from .az/mshost.yaml for the WebAssembly job.
+        cmake_args += [
+            "-DCMAKE_TOOLCHAIN_FILE=" + str(emsdk_toolchain),
+            "-DOCOS_ENABLE_SPM_TOKENIZER=ON",
+            "-DOCOS_BUILD_PYTHON=OFF",
+            "-DOCOS_ENABLE_CV2=OFF",
+            "-DOCOS_ENABLE_VISION=OFF"
+        ]
+
+    if args.disable_exceptions:
+        cmake_args.append("-DOCOS_ENABLE_CPP_EXCEPTIONS=OFF")
+
+    if args.build_java:
+        cmake_args.append("-DOCOS_BUILD_JAVA=ON")
+
+    cmake_args += ["-D{}".format(define) for define in cmake_extra_defines]
+    cmake_args += cmake_extra_args
+
+    for config in configs:
+        config_build_dir = _get_build_config_dir(build_dir, config)
+        _run_subprocess(cmake_args + [f"-DCMAKE_BUILD_TYPE={config}"], cwd=config_build_dir)
+
+
+def clean_targets(cmake_path, build_dir: Path, configs: Set[str]):
+    for config in configs:
+        log.info("Cleaning targets for %s configuration", config)
+        build_dir2 = _get_build_config_dir(build_dir, config)
+        cmd_args = [cmake_path, "--build", build_dir2, "--config", config, "--target", "clean"]
+
+        _run_subprocess(cmd_args)
+
+
+def build_targets(args, cmake_path: Path, build_dir: Path, configs: Set[str], num_parallel_jobs: int):
+    env = {}
+    if args.android:
+        env["ANDROID_HOME"] = str(args.android_home)
+        env["ANDROID_NDK_HOME"] = str(args.android_ndk_path)
+
+    for config in configs:
+        log.info("Building targets for %s configuration", config)
+        build_dir2 = _get_build_config_dir(build_dir, config)
+        cmd_args = [str(cmake_path), "--build", str(build_dir2), "--config", config]
+
+        build_tool_args = []
+        if num_parallel_jobs != 1:
+            if is_windows() and args.cmake_generator != "Ninja" and not args.wasm:
+                build_tool_args += [
+                    "/maxcpucount:{}".format(num_parallel_jobs),
+                    # if nodeReuse is true, msbuild processes will stay around for a bit after the build completes
+                    "/nodeReuse:False",
+                ]
+            elif is_macOS() and args.use_xcode:
+                # CMake will generate correct build tool args for Xcode
+                cmd_args += ["--parallel", str(num_parallel_jobs)]
+            else:
+                build_tool_args += ["-j{}".format(num_parallel_jobs)]
+
+        if build_tool_args:
+            cmd_args += ["--"]
+            cmd_args += build_tool_args
+
+        _run_subprocess(cmd_args, env=env)
+
+
+def _run_python_tests():
+    # TODO: Run the python tests in /python
+    pass
+
+
+def _run_android_tests(args, build_dir: Path, config: str, cwd: Path):
+    # TODO: Setup running tests using Android simulator and adb. See ORT build.py for example.
+    source_dir = REPO_DIR
+    pass
+
+
+def _run_ios_tests(args, config: str, cwd: Path):
+    # TODO: Setup running tests using xcode an iPhone simulator. See ORT build.py for example.
+    source_dir = REPO_DIR
+    pass
+
+
+def _run_cxx_tests(args, build_dir: Path, configs: Set[str]):
+    ctest_path = args.ctest_path
+    code_coverage_using_vstest = is_windows() and args.cxx_code_coverage
+
+    if not code_coverage_using_vstest:
+        ctest_path = _resolve_executable_path(ctest_path)
+        if not ctest_path:
+            raise UsageError(f"ctest was not found. Looked for '{args.ctest_path}'. "
+                             "Specify using `--ctest_path` if necessary.")
+
+    for config in configs:
+        log.info("Running tests for %s configuration", config)
+
+        cwd = _get_build_config_dir(build_dir, config)
+
+        if args.android:
+            _run_android_tests(args, build_dir, config, cwd)
+            continue
+        elif args.ios:
+            _run_ios_tests(args, config, cwd)
+            continue
+
+        if code_coverage_using_vstest:
+            # Get the "Google Test Adapter" for vstest.
+            if not (cwd / "GoogleTestAdapter.0.18.0").is_dir():
+                _run_subprocess(
+                    [
+                        "nuget.exe",
+                        "restore",
+                        str(REPO_DIR / "test" / "packages.config"),
+                        "-ConfigFile",
+                        str(REPO_DIR / "test" / "NuGet.config"),
+                        "-PackagesDirectory",
+                        str(cwd),
+                    ]
+                )
+
+            # test exes are in the bin/<config> subdirectory of the build output dir
+            # call resolve() to get the full path as we're going to execute in build_dir not cwd
+            test_dir = (cwd / "bin" / config).resolve()
+            adapter = (cwd / 'GoogleTestAdapter.0.18.0' / 'build' / '_common').resolve()
+
+            executables = [
+                str(test_dir / "extensions_test.exe"),
+                str(test_dir / "ocos_test.exe")
+            ]
+
+            # run this script from a VS dev shell so vstest.console.exe is found via PATH
+            vstest_exe = _resolve_executable_path("vstest.console.exe")
+            _run_subprocess(
+                [
+                    vstest_exe,
+                    "--parallel",
+                    f"--TestAdapterPath:{str(adapter)}",
+                    "/Logger:trx",
+                    "/Enablecodecoverage",
+                    "/Platform:x64",
+                    f"/Settings:{str(REPO_DIR / 'test' / 'codeconv.runsettings')}",
+                ]
+                + executables,
+                cwd=build_dir,
+            )
+        else:
+            ctest_cmd = [str(ctest_path), "--build-config", config, "--verbose", "--timeout", "10800"]
+            _run_subprocess(ctest_cmd, cwd=cwd)
+
+
+def main():
+    log.debug("Command line arguments:\n  {}".format(" ".join(shlex.quote(arg) for arg in sys.argv[1:])))
+
+    args = _parse_arguments()
+    cmake_extra_defines = _flatten_arg_list(args.cmake_extra_defines)
+    cross_compiling = args.arm or args.arm64 or args.arm64ec or args.android or args.wasm
+
+    # If there was no explicit argument saying what to do, default
+    # to update, build and test (for native builds).
+    if not (args.update or args.clean or args.build or args.test):
+        log.debug("Defaulting to running update, build [and test for native builds].")
+        args.update = True
+        args.build = True
+        if cross_compiling:
+            args.test = args.android_abi == "x86_64" or args.android_abi == "arm64-v8a"
+        else:
+            args.test = True
+
+    if args.skip_tests:
+        args.test = False
+
+    if args.android and is_windows():
+        if args.cmake_generator != "Ninja":
+            log.info("Setting cmake_generator to Ninja, which is required when cross-compiling Android on Windows.")
+            args.cmake_generator = "Ninja"
+
+    configs = set(args.config)
+
+    # setup paths and directories
+    # cmake_path can be None. For example, if a person only wants to run the tests, they don't need cmake.
+    cmake_path = _resolve_executable_path(args.cmake_path)
+    build_dir = args.build_dir
+
+    if args.update or args.build:
+        for config in configs:
+            os.makedirs(_get_build_config_dir(build_dir, config), exist_ok=True)
+
+    if args.wasm:
+        _setup_emscripten(args)
+
+    log.info("Build started")
+
+    if args.update:
+        if _is_reduced_ops_build(args):
+            log.info("Generating config for selected ops")
+            _generate_selected_ops_config(args.include_ops_by_config)
+
+        cmake_extra_args = []
+
+        if is_windows():
+            cpu_arch = platform.architecture()[0]
+            if args.wasm:
+                cmake_extra_args = ["-G", "Ninja"]
+            elif args.cmake_generator == "Ninja":
+                if cpu_arch == "32bit" or args.arm or args.arm64 or args.arm64ec:
+                    raise UsageError(
+                        "To cross-compile with Ninja, load the toolset environment for the target processor "
+                        "(e.g. Cross Tools Command Prompt for VS)")
+                cmake_extra_args = ["-G", args.cmake_generator]
+            elif args.arm or args.arm64 or args.arm64ec:
+                # Cross-compiling for ARM(64) architecture
+                if args.arm:
+                    cmake_extra_args = ["-A", "ARM"]
+                elif args.arm64:
+                    cmake_extra_args = ["-A", "ARM64"]
+                elif args.arm64ec:
+                    cmake_extra_args = ["-A", "ARM64EC"]
+
+                cmake_extra_args += ["-G", args.cmake_generator]
+
+                # Cannot test on host build machine for cross-compiled
+                # builds (Override any user-defined behaviour for test if any)
+                if args.test:
+                    log.warning("Cannot test on host build machine for cross-compiled ARM(64) builds. "
+                                "Will skip test running after build.")
+                    args.test = False
+            elif cpu_arch == "32bit" or args.x86:
+                cmake_extra_args = ["-A", "Win32", "-T", "host=x64", "-G", args.cmake_generator]
+            else:
+                toolset = "host=x64"
+                # TODO: Do we need the ability to specify the toolset? If so need to add the msvc_toolset arg back in
+                # if args.msvc_toolset:
+                #     toolset += f",version={args.msvc_toolset}"
+                cmake_extra_args = ["-A", "x64", "-T", toolset, "-G", args.cmake_generator]
+        elif args.cmake_generator is not None:
+            cmake_extra_args += ["-G", args.cmake_generator]
+
+        if is_macOS():
+            if not args.ios and not args.android and args.osx_arch == "arm64" and platform.machine() == "x86_64":
+                if args.test:
+                    log.warning("Cannot test ARM64 build on X86_64. Will skip test running after build.")
+                    args.test = False
+
+        _generate_build_tree(
+            cmake_path,
+            REPO_DIR,
+            build_dir,
+            configs,
+            cmake_extra_defines,
+            args,
+            cmake_extra_args)
+
+    if args.clean:
+        clean_targets(cmake_path, build_dir, configs)
+
+    if args.build:
+        if args.parallel < 0:
+            raise UsageError("Invalid parallel job count: {}".format(args.parallel))
+        num_parallel_jobs = os.cpu_count() if args.parallel == 0 else args.parallel
+        build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs)
+
+    if args.test:
+        _run_python_tests()
+        if args.enable_cxx_tests:
+            _validate_cxx_test_args(args)
+            _run_cxx_tests(args, build_dir, configs)
+
+    log.info("Build complete")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except UsageError as e:
+        log.error(str(e))
+        sys.exit(1)

tools/utils/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from .logger import get_logger
+from .platform_helpers import is_linux, is_macOS, is_windows
+from .run import run
+from .android import SdkToolPaths, create_virtual_device, get_sdk_tool_paths, start_emulator, stop_emulator

tools/utils/android.py

@@ -0,0 +1,159 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import collections
+import contextlib
+import logging
+import os
+import shutil
+import signal
+import subprocess
+import time
+import typing
+
+from .platform_helpers import is_windows
+from .run import run
+
+_log = logging.getLogger("util.android")
+
+
+SdkToolPaths = collections.namedtuple("SdkToolPaths", ["emulator", "adb", "sdkmanager", "avdmanager"])
+
+
+def get_sdk_tool_paths(sdk_root: str):
+    def filename(name, windows_extension):
+        if is_windows():
+            return "{}.{}".format(name, windows_extension)
+        else:
+            return name
+
+    def resolve_path(dirnames, basename):
+        dirnames.insert(0, "")
+        for dirname in dirnames:
+            path = shutil.which(os.path.join(dirname, basename))
+            if path is not None:
+                path = os.path.realpath(path)
+                _log.debug("Found {} at {}".format(basename, path))
+                return path
+        _log.warning("Failed to resolve path for {}".format(basename))
+        return None
+
+    return SdkToolPaths(
+        emulator=resolve_path([os.path.join(sdk_root, "emulator")], filename("emulator", "exe")),
+        adb=resolve_path([os.path.join(sdk_root, "platform-tools")], filename("adb", "exe")),
+        sdkmanager=resolve_path(
+            [os.path.join(sdk_root, "tools", "bin"), os.path.join(sdk_root, "cmdline-tools", "tools", "bin")],
+            filename("sdkmanager", "bat"),
+        ),
+        avdmanager=resolve_path(
+            [os.path.join(sdk_root, "tools", "bin"), os.path.join(sdk_root, "cmdline-tools", "tools", "bin")],
+            filename("avdmanager", "bat"),
+        ),
+    )
+
+
+def create_virtual_device(sdk_tool_paths: SdkToolPaths, system_image_package_name: str, avd_name: str):
+    run(sdk_tool_paths.sdkmanager, "--install", system_image_package_name, input=b"y")
+
+    run(
+        sdk_tool_paths.avdmanager,
+        "create",
+        "avd",
+        "--name",
+        avd_name,
+        "--package",
+        system_image_package_name,
+        "--force",
+        input=b"no",
+    )
+
+
+_process_creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if is_windows() else 0
+
+
+def _start_process(*args) -> subprocess.Popen:
+    _log.debug("Starting process - args: {}".format([*args]))
+    return subprocess.Popen([*args], creationflags=_process_creationflags)
+
+
+_stop_signal = signal.CTRL_BREAK_EVENT if is_windows() else signal.SIGTERM
+
+
+def _stop_process(proc: subprocess.Popen):
+    _log.debug("Stopping process - args: {}".format(proc.args))
+    proc.send_signal(_stop_signal)
+
+    try:
+        proc.wait(30)
+    except subprocess.TimeoutExpired:
+        _log.warning("Timeout expired, forcibly stopping process...")
+        proc.kill()
+
+
+def _stop_process_with_pid(pid: int):
+    # not attempting anything fancier than just sending _stop_signal for now
+    _log.debug("Stopping process - pid: {}".format(pid))
+    os.kill(pid, _stop_signal)
+
+
+def start_emulator(
+    sdk_tool_paths: SdkToolPaths, avd_name: str, extra_args: typing.Optional[typing.Sequence[str]] = None
+) -> subprocess.Popen:
+    with contextlib.ExitStack() as emulator_stack, contextlib.ExitStack() as waiter_stack:
+        emulator_args = [
+            sdk_tool_paths.emulator,
+            "-avd",
+            avd_name,
+            "-memory",
+            "4096",
+            "-timezone",
+            "America/Los_Angeles",
+            "-no-snapshot",
+            "-no-audio",
+            "-no-boot-anim",
+            "-no-window",
+        ]
+        if extra_args is not None:
+            emulator_args += extra_args
+
+        emulator_process = emulator_stack.enter_context(_start_process(*emulator_args))
+        emulator_stack.callback(_stop_process, emulator_process)
+
+        waiter_process = waiter_stack.enter_context(
+            _start_process(
+                sdk_tool_paths.adb,
+                "wait-for-device",
+                "shell",
+                "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done; input keyevent 82",
+            )
+        )
+        waiter_stack.callback(_stop_process, waiter_process)
+
+        # poll subprocesses
+        sleep_interval_seconds = 1
+        while True:
+            waiter_ret, emulator_ret = waiter_process.poll(), emulator_process.poll()
+
+            if emulator_ret is not None:
+                # emulator exited early
+                raise RuntimeError("Emulator exited early with return code: {}".format(emulator_ret))
+
+            if waiter_ret is not None:
+                if waiter_ret == 0:
+                    break
+                raise RuntimeError("Waiter process exited with return code: {}".format(waiter_ret))
+
+            time.sleep(sleep_interval_seconds)
+
+        # emulator is ready now
+        emulator_stack.pop_all()
+        return emulator_process
+
+
+def stop_emulator(emulator_proc_or_pid: typing.Union[subprocess.Popen, int]):
+    if isinstance(emulator_proc_or_pid, subprocess.Popen):
+        _stop_process(emulator_proc_or_pid)
+    elif isinstance(emulator_proc_or_pid, int):
+        _stop_process_with_pid(emulator_proc_or_pid)
+    else:
+        raise ValueError("Expected either a PID or subprocess.Popen instance.")

tools/utils/logger.py

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import logging
+
+
+def get_logger(name):
+    logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG)
+
+    return logging.getLogger(name)

tools/utils/platform_helpers.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import sys
+
+
+def is_windows():
+    return sys.platform.startswith("win")
+
+
+def is_macOS():
+    return sys.platform.startswith("darwin")
+
+
+def is_linux():
+    return sys.platform.startswith("linux")

tools/utils/run.py

@@ -0,0 +1,62 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import logging
+import os
+import shlex
+import subprocess
+
+_log = logging.getLogger("util.run")
+
+
+def run(
+    *args,
+    cwd=None,
+    input=None,
+    capture_stdout=False,
+    capture_stderr=False,
+    shell=False,
+    env=None,
+    check=True,
+    quiet=False,
+):
+    """Runs a subprocess.
+
+    Args:
+        *args: The subprocess arguments.
+        cwd: The working directory. If None, specifies the current directory.
+        input: The optional input byte sequence.
+        capture_stdout: Whether to capture stdout.
+        capture_stderr: Whether to capture stderr.
+        shell: Whether to run using the shell.
+        env: The environment variables as a dict. If None, inherits the current
+            environment.
+        check: Whether to raise an error if the return code is not zero.
+        quiet: If true, do not print output from the subprocess.
+
+    Returns:
+        A subprocess.CompletedProcess instance.
+    """
+    cmd = [*args]
+
+    _log.info(
+        "Running subprocess in '{0}'\n  {1}".format(cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in cmd]))
+    )
+
+    def output(is_stream_captured):
+        return subprocess.PIPE if is_stream_captured else (subprocess.DEVNULL if quiet else None)
+
+    completed_process = subprocess.run(
+        cmd,
+        cwd=cwd,
+        check=check,
+        input=input,
+        stdout=output(capture_stdout),
+        stderr=output(capture_stderr),
+        env=env,
+        shell=shell,
+    )
+
+    _log.debug("Subprocess completed. Return code: {}".format(completed_process.returncode))
+
+    return completed_process