Android package custom build script update (#14403)

Update Android package custom build script.
- Use later version of various dependencies (CMake, JDK, Android command line tools, Android NDK, Ubuntu). The CMake version was too old for the current ORT code.
- Do in-container build in a directory that is not shared with the host. Resolves some file permission issues and speeds up file access.

Add a nightly build to make sure the script works with the latest ORT.

tools/android_custom_build/.dockerignore

@@ -1,2 +1,3 @@
-# ignore all files
+# ignore all files except for the /scripts directory
 *
+!/scripts/*

tools/android_custom_build/Dockerfile

@@ -4,7 +4,7 @@
 # --------------------------------------------------------------
 # Dockerfile for ONNX Runtime Android package build environment
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -14,16 +14,16 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \
   unzip
 
 # install Java
-RUN apt-get install --yes --no-install-recommends openjdk-8-jdk-headless
+RUN apt-get install --yes --no-install-recommends openjdk-11-jdk-headless
 
 ENV ANDROID_HOME=/opt/android-sdk
-ENV NDK_VERSION=24.0.8215888
+ENV NDK_VERSION=25.0.8775105
 ENV ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION}
 
 # install Android command line tools
 RUN aria2c -q -d /tmp -o cmdline-tools.zip \
-  --checksum=sha-256=d71f75333d79c9c6ef5c39d3456c6c58c613de30e6a751ea0dbd433e8f8b9cbf \
-  https://dl.google.com/android/repository/commandlinetools-linux-8092744_latest.zip && \
+  --checksum=sha-256=0bebf59339eaa534f4217f8aa0972d14dc49e7207be225511073c661ae01da0a \
+  https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip && \
   unzip /tmp/cmdline-tools.zip -d /tmp/cmdline-tools && \
   mkdir -p ${ANDROID_HOME}/cmdline-tools && \
   mv /tmp/cmdline-tools/cmdline-tools ${ANDROID_HOME}/cmdline-tools/latest
@@ -46,9 +46,9 @@ RUN apt-get install --yes --no-install-recommends \
   python3-wheel
 
 # cmake
-RUN CMAKE_VERSION=3.21.0 && \
+RUN CMAKE_VERSION=3.25.2 && \
   aria2c -q -d /tmp -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz \
-  --checksum=sha-256=d54ef6909f519740bc85cec07ff54574cd1e061f9f17357d9ace69f61c6291ce \
+  --checksum=sha-256=783da74f132fd1fea91b8236d267efa4df5b91c5eec1dea0a87f0cf233748d99 \
   https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \
   tar -zxf /tmp/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz --strip=1 -C /usr
 
@@ -71,3 +71,6 @@ ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git
 ARG ONNXRUNTIME_BRANCH_OR_TAG=main
 RUN git clone --single-branch --branch=${ONNXRUNTIME_BRANCH_OR_TAG} --recurse-submodules ${ONNXRUNTIME_REPO} \
   /workspace/onnxruntime
+
+# add scripts
+COPY scripts /workspace/scripts

tools/android_custom_build/build_custom_android_package.py

@@ -4,14 +4,22 @@
 
 import argparse
 import pathlib
+import shlex
 import shutil
 import subprocess
+import sys
 
 SCRIPT_DIR = pathlib.Path(__file__).parent.resolve()
 DEFAULT_OPS_CONFIG_RELATIVE_PATH = "tools/ci_build/github/android/mobile_package.required_operators.config"
 DEFAULT_BUILD_SETTINGS_RELATIVE_PATH = "tools/ci_build/github/android/default_mobile_aar_build_settings.json"
 
 
+def run(cmd_arg_list, **kwargs):
+    print(f"Running command:\n  {shlex.join(cmd_arg_list)}")
+    kwargs.update({"check": True})
+    return subprocess.run(cmd_arg_list, **kwargs)
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description="""Builds a custom ONNX Runtime Android package.
@@ -67,6 +75,13 @@ def parse_args():
         help="The tag for the Docker image. " f"The default is {default_docker_image_tag}.",
     )
 
+    parser.add_argument(
+        "--docker_container_name",
+        help="The name of the Docker container that is run (given to the --name option of `docker run`). "
+        "If unspecified, the container will be automatically removed. "
+        "Keeping the container may be useful for debugging.",
+    )
+
     parser.add_argument(
         "--docker_path",
         default=shutil.which("docker"),
@@ -84,13 +99,13 @@ def parse_args():
 def main():
     args = parse_args()
 
-    docker_build_args = []
+    docker_build_image_args = []
     if args.onnxruntime_branch_or_tag:
-        docker_build_args += ["--build-arg", f"ONNXRUNTIME_BRANCH_OR_TAG={args.onnxruntime_branch_or_tag}"]
+        docker_build_image_args += ["--build-arg", f"ONNXRUNTIME_BRANCH_OR_TAG={args.onnxruntime_branch_or_tag}"]
     if args.onnxruntime_repo_url:
-        docker_build_args += ["--build-arg", f"ONNXRUNTIME_REPO={args.onnxruntime_repo_url}"]
+        docker_build_image_args += ["--build-arg", f"ONNXRUNTIME_REPO={args.onnxruntime_repo_url}"]
 
-    docker_build_cmd = (
+    docker_build_image_cmd = (
         [
             args.docker_path,
             "build",
@@ -99,11 +114,11 @@ def main():
             "--file",
             str(SCRIPT_DIR / "Dockerfile"),
         ]
-        + docker_build_args
+        + docker_build_image_args
         + [str(SCRIPT_DIR)]
     )
 
-    subprocess.run(docker_build_cmd, check=True)
+    run(docker_build_image_cmd)
 
     working_dir = args.working_dir
     working_dir.mkdir(parents=True, exist_ok=True)
@@ -132,23 +147,29 @@ def main():
         else f"/workspace/onnxruntime/{DEFAULT_BUILD_SETTINGS_RELATIVE_PATH}"
     )
 
-    docker_run_cmd = [
-        args.docker_path,
-        "run",
-        "--rm",
-        "-it",
-        f"--volume={str(working_dir)}:/workspace/shared",
-        args.docker_image_tag,
-        "/usr/bin/env",
-        "python3",
-        "/workspace/onnxruntime/tools/ci_build/github/android/build_aar_package.py",
-        "--build_dir=/workspace/shared/output",
-        f"--config={args.config}",
-        f"--include_ops_by_config={container_ops_config_file}",
-        container_build_settings_file,
-    ]
+    # enable use of Ctrl-C to stop when running interactively
+    docker_run_interactive_args = ["-it"] if sys.stdin.isatty() else []
 
-    subprocess.run(docker_run_cmd, check=True)
+    docker_container_build_cmd = (
+        [
+            args.docker_path,
+            "run",
+        ]
+        + docker_run_interactive_args
+        + [
+            f"--name={args.docker_container_name}" if args.docker_container_name is not None else "--rm",
+            f"--volume={working_dir}:/workspace/shared",
+            args.docker_image_tag,
+            "/bin/bash",
+            "/workspace/scripts/build.sh",
+            args.config,
+            container_ops_config_file,
+            container_build_settings_file,
+            "/workspace/shared/output",
+        ]
+    )
+
+    run(docker_container_build_cmd)
 
     print("Finished building Android package at '{}'.".format(output_dir / "aar_out"))
 

tools/android_custom_build/readme.md

@@ -3,7 +3,7 @@ This directory contains helper files for building a custom ONNX Runtime Android
 Run the ./build_custom_android_package.py script. Use the --help option for more information.
 
 Prerequisites:
-- Python 3.6+
+- Python 3.8+
 - Docker
 
 See https://onnxruntime.ai/docs/build/custom.html for more information about creating and using custom builds.

tools/android_custom_build/scripts/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Note: This script is intended to be run by ../build_custom_android_package.py in a Docker container.
+
+set -e -x
+
+USAGE_TEXT="Usage: ${0} <build config> <ops config file> <build settings file> <output directory>"
+
+BUILD_CONFIG=${1:?${USAGE_TEXT}}
+OPS_CONFIG_FILE=${2:?${USAGE_TEXT}}
+BUILD_SETTINGS_FILE=${3:?${USAGE_TEXT}}
+OUTPUT_DIR=${4:?${USAGE_TEXT}}
+
+BUILD_DIR=/workspace/build
+
+# build ORT AAR
+python3 /workspace/onnxruntime/tools/ci_build/github/android/build_aar_package.py \
+  --build_dir="${BUILD_DIR}" \
+  --config="${BUILD_CONFIG}" \
+  --include_ops_by_config="${OPS_CONFIG_FILE}" \
+  "${BUILD_SETTINGS_FILE}"
+
+# copy AAR to output directory
+cp -r "${BUILD_DIR}/aar_out" "${OUTPUT_DIR}"

tools/ci_build/github/azure-pipelines/post-merge-jobs.yml

@@ -156,3 +156,38 @@ stages:
         Destination: AzureBlob
         storage: onnxruntime
         ContainerName: '$web'
+
+- stage: AndroidCustomBuildScript
+  dependsOn: []
+  jobs:
+  - job: AndroidCustomBuildScript
+    workspace:
+      clean: all
+    pool: 'Linux-CPU'
+    variables:
+      dockerImageTag: onnxruntime-android-custom-build
+    steps:
+    - checkout: self
+      submodules: false
+
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.8'
+        addToPath: true
+        architecture: x64
+
+    - task: CmdLine@2
+      displayName: 'Run build_custom_android_package.py'
+      inputs:
+        script: |
+          "$(Build.SourcesDirectory)/tools/android_custom_build/build_custom_android_package.py" \
+            --docker_image_tag=$(dockerImageTag) \
+            "$(Build.BinariesDirectory)/custom_android_package"
+        workingDirectory: '$(Build.BinariesDirectory)'
+
+    - task: CmdLine@2
+      displayName: 'Clean up docker image'
+      inputs:
+        script: docker image rm $(dockerImageTag)
+        workingDirectory: '$(Build.BinariesDirectory)'
+      condition: succeededOrFailed()