Build Hermes from Source (#33396)

Summary:
Pull Request resolved: https://github.com/facebook/react-native/pull/33396

This commit fully unplugs the `ReactAndroid` from using hermes from the NPM package and plugs the usage of Hermes via the `packages/hermes-engine` Gradle build.

I've used prefab to share the .so between the two builds, so we don't need any extra machinery to make this possible.

Moreover, I've added a `buildHermesFromSource` property, which defaults to false when RN is imported, but is set to true when RN is opened for local development. This should allow us to distribute the `react-native` NPM package and users could potentially toggle which source to use (but see below).

Changelog:
[Android] [Changed] - Build Hermes from Source

Reviewed By: hramos

Differential Revision: D34389875

fbshipit-source-id: 107cbe3686daf7607a1f0f75202f24cd80ce64bb

.circleci/Dockerfiles/Dockerfile.android

@@ -14,7 +14,7 @@
 # and build a Android application that can be used to run the
 # tests specified in the scripts/ directory.
 #
-FROM reactnativecommunity/react-native-android:5.2
+FROM reactnativecommunity/react-native-android:5.4
 
 LABEL Description="React Native Android Test Image"
 LABEL maintainer="Héctor Ramos <hector@fb.com>"
@@ -53,6 +53,4 @@ ADD . /app
 
 RUN yarn
 
-RUN ./gradlew :ReactAndroid:downloadBoost :ReactAndroid:downloadDoubleConversion :ReactAndroid:downloadFolly :ReactAndroid:downloadGlog
-
-RUN ./gradlew :ReactAndroid:packageReactNdkLibsForBuck -Pjobs=1
+RUN ./gradlew :ReactAndroid:assembleDebug

.circleci/config.yml

@@ -67,7 +67,7 @@ executors:
   reactnativeandroid:
     <<: *defaults
     docker:
-      - image: reactnativecommunity/react-native-android:5.2
+      - image: reactnativecommunity/react-native-android:5.4
     resource_class: "large"
     environment:
       - TERM: "dumb"

ReactAndroid/build.gradle

@@ -45,6 +45,10 @@ def boostPath = dependenciesPath ?: System.getenv("REACT_NATIVE_BOOST_PATH")
 // Setup build type for NDK, supported values: {debug, release}
 def nativeBuildType = System.getenv("NATIVE_BUILD_TYPE") ?: "release"
 
+// We put the publishing version from gradle.properties inside ext. so other
+// subprojects can access it as well.
+ext.publishing_version = VERSION_NAME
+
 task createNativeDepsDirectories {
     downloadsDir.mkdirs()
     thirdPartyNdkDir.mkdirs()
@@ -127,24 +131,6 @@ final def prepareLibevent = tasks.register("prepareLibevent", PrepareLibeventTas
     it.outputDir.set(new File(thirdPartyNdkDir, "libevent"))
 }
 
-task prepareHermes(dependsOn: createNativeDepsDirectories, type: Copy) {
-    def hermesPackagePath = findNodeModulePath(projectDir, "hermes-engine")
-    if (!hermesPackagePath) {
-        throw new GradleScriptException("Could not find the hermes-engine npm package", null)
-    }
-
-    def hermesAAR = file("$hermesPackagePath/android/hermes-debug.aar")
-    if (!hermesAAR.exists()) {
-        throw new GradleScriptException("The hermes-engine npm package is missing \"android/hermes-debug.aar\"", null)
-    }
-
-    def soFiles = zipTree(hermesAAR).matching({ it.include "**/*.so" })
-
-    from soFiles
-    from "src/main/jni/first-party/hermes/Android.mk"
-    into "$thirdPartyNdkDir/hermes"
-}
-
 task downloadGlog(dependsOn: createNativeDepsDirectories, type: Download) {
     src("https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz")
     onlyIfNewer(true)
@@ -259,18 +245,11 @@ final def extractNativeDependencies = tasks.register('extractNativeDependencies'
     it.extractHeadersConfiguration.setFrom(configurations.extractHeaders)
     it.extractJniConfiguration.setFrom(configurations.extractJNI)
     it.baseOutputDir = project.file("src/main/jni/first-party/")
-    // Sadly this task as an output folder path that is directly dependent on
-    // the task input (i.e. src/main/jni/first-party/<package-name>/...
-    // This means that this task is using the parent folder (first-party/) as
-    // @OutputFolder. The `prepareHermes` task will also output inside that
-    // folder and if the two tasks happen to be inside the same run, we want
-    // `extractNativeDependencies` to run after `prepareHermes` to do not
-    // invalidate the input/output calculation for this task.
-    it.mustRunAfter(prepareHermes)
 }
 
 task installArchives {
     dependsOn("publishReleasePublicationToNpmRepository")
+    dependsOn(":ReactAndroid:hermes-engine:installArchives")
 }
 
 // Creating sources with comments
@@ -314,6 +293,7 @@ android {
                         "REACT_COMMON_DIR=$projectDir/../ReactCommon",
                         "REACT_GENERATED_SRC_DIR=$buildDir/generated/source",
                         "REACT_SRC_DIR=$projectDir/src/main/java/com/facebook/react",
+                        "APP_STL=c++_shared",
                         "-j${ndkBuildJobs()}"
 
                 if (Os.isFamily(Os.FAMILY_MAC)) {
@@ -333,7 +313,7 @@ android {
         }
     }
 
-    preBuild.dependsOn(prepareJSC, prepareHermes, prepareBoost, prepareDoubleConversion, prepareFmt, prepareFolly, prepareGlog, prepareLibevent, extractNativeDependencies)
+    preBuild.dependsOn(prepareJSC, prepareBoost, prepareDoubleConversion, prepareFmt, prepareFolly, prepareGlog, prepareLibevent, extractNativeDependencies)
     preBuild.dependsOn("generateCodegenArtifactsFromSchema")
 
     sourceSets.main {
@@ -366,6 +346,10 @@ android {
         extractJNI
         javadocDeps.extendsFrom api
     }
+
+    buildFeatures {
+        prefab true
+    }
 }
 
 dependencies {
@@ -388,6 +372,15 @@ dependencies {
     extractHeaders("com.facebook.fbjni:fbjni:0.2.2:headers")
     extractJNI("com.facebook.fbjni:fbjni:0.2.2")
 
+    // It's up to the consumer to decide if hermes should be included or not.
+    // Therefore hermes-engine is a compileOnly dependency.
+    // Moreover, you can toggle where to get the dependency with `buildHermesFromSource`.
+    if (project.getProperties().getOrDefault("buildHermesFromSource", "false").toBoolean()) {
+        compileOnly(project(":ReactAndroid:hermes-engine"))
+    } else {
+        compileOnly("com.facebook.react:hermes-engine:${VERSION_NAME}")
+    }
+
     javadocDeps("com.squareup:javapoet:1.13.0")
 
     testImplementation("junit:junit:${JUNIT_VERSION}")

ReactAndroid/hermes-engine/.npmignore

@@ -0,0 +1,3 @@
+# Make sure we never publish the build folders to npm.
+build/
+.cxx/

ReactAndroid/hermes-engine/build.gradle

@@ -22,6 +22,7 @@ if (hermesVersionFile.exists()) {
     hermesVersion = hermesVersionFile.text
 }
 def ndkBuildJobs = Runtime.runtime.availableProcessors().toString()
+def prefabHeadersDir = new File("$buildDir/prefab-headers")
 
 // We inject the JSI directory used inside the Hermes build with the -DJSI_DIR config.
 def jsiDir = rootProject.file("ReactCommon/jsi")
@@ -42,6 +43,10 @@ task downloadNdkBuildDependencies() {
     dependsOn(downloadHermes)
 }
 
+task installArchives {
+    dependsOn("publishAllPublicationsToNpmRepository")
+}
+
 task unzipHermes(dependsOn: downloadHermes, type: Copy) {
     from(tarTree(downloadHermes.dest)) {
         eachFile { file ->
@@ -56,7 +61,7 @@ task unzipHermes(dependsOn: downloadHermes, type: Copy) {
 }
 
 
-task configureNinjaForHermes(dependsOn: unzipHermes, type: org.gradle.api.tasks.Exec) {
+task configureNinjaForHermes(dependsOn: unzipHermes, type: Exec) {
     workingDir(hermesDir)
     commandLine(windowsAwareCommandLine(
         "python3",
@@ -67,11 +72,19 @@ task configureNinjaForHermes(dependsOn: unzipHermes, type: org.gradle.api.tasks.
     ))
 }
 
-task buildNinjaForHermes(dependsOn: configureNinjaForHermes, type: org.gradle.api.tasks.Exec) {
+task buildNinjaForHermes(dependsOn: configureNinjaForHermes, type: Exec) {
     workingDir(hermesDir)
     commandLine(windowsAwareCommandLine("cmake", "--build", "./ninja_build", "--target", "hermesc", "-j", ndkBuildJobs))
 }
 
+task prepareHeadersForPrefab(dependsOn: unzipHermes, type: Copy) {
+    from("$hermesDir/API")
+    from("$hermesDir/public")
+    include("**/*.h")
+    exclude("jsi/**")
+    into(prefabHeadersDir)
+}
+
 static def windowsAwareCommandLine(String... commands) {
     def newCommands = []
     if (Os.isFamily(Os.FAMILY_WINDOWS)) {
@@ -88,6 +101,7 @@ def reactNativeArchitectures() {
 
 android {
     compileSdkVersion 31
+    buildToolsVersion = "31.0.0"
 
     // Used to override the NDK path & version on internal CI
     if (System.getenv("ANDROID_NDK") != null && System.getenv("LOCAL_ANDROID_NDK_VERSION") != null) {
@@ -180,12 +194,28 @@ android {
             allVariants()
         }
     }
+
+    buildFeatures {
+        prefabPublishing true
+    }
+
+    prefab {
+        libhermes {
+            headers prefabHeadersDir.absolutePath
+            libraryName "libhermes"
+        }
+    }
 }
 
 afterEvaluate {
     preBuild.dependsOn(buildNinjaForHermes)
+    preBuild.dependsOn(prepareHeadersForPrefab)
     // Needed as some of the native sources needs to be downloaded
     // before configureCMakeRelease/configureCMakeMinSizeRel could be executed.
+    reactNativeArchitectures().each { architecture ->
+        tasks.named("configureCMakeMinSizeRel[${architecture}]") { dependsOn(preBuild) }
+        tasks.named("configureCMakeRelease[${architecture}]") { dependsOn(preBuild) }
+    }
     configureCMakeRelease.dependsOn(preBuild)
     configureCMakeMinSizeRel.dependsOn(preBuild)
 
@@ -206,4 +236,4 @@ afterEvaluate {
 }
 
 group = "com.facebook.react"
-version = VERSION_NAME
+version = parent.publishing_version

ReactAndroid/hermes-engine/gradle.properties

@@ -1,2 +1 @@
-VERSION_NAME=1000.0.0-main
-android.disableAutomaticComponentCreation=true
+android.disableAutomaticComponentCreation=true

ReactAndroid/src/main/java/com/facebook/hermes/instrumentation/Android.mk

@@ -7,14 +7,13 @@
 LOCAL_PATH := $(call my-dir)
 REACT_NATIVE := $(LOCAL_PATH)/../../../../../../../..
 
-include $(REACT_NATIVE)/ReactCommon/common.mk
 include $(CLEAR_VARS)
 
 LOCAL_MODULE := jsijniprofiler
 
 LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
+LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi
 
 LOCAL_CPP_FEATURES := exceptions
 
@@ -29,5 +28,3 @@ LOCAL_SHARED_LIBRARIES := \
 
 include $(BUILD_SHARED_LIBRARY)
 
-
-include $(CLEAR_VARS)

ReactAndroid/src/main/java/com/facebook/hermes/reactexecutor/Android.mk

@@ -6,47 +6,47 @@
 LOCAL_PATH := $(call my-dir)
 REACT_NATIVE := $(LOCAL_PATH)/../../../../../../../..
 
-include $(REACT_NATIVE)/ReactCommon/common.mk
-include $(CLEAR_VARS)
+ifeq ($(APP_OPTIM),debug)
+  include $(CLEAR_VARS)
 
-LOCAL_MODULE := hermes-executor-release
+  LOCAL_MODULE := hermes-executor-debug
+  LOCAL_CFLAGS := -DHERMES_ENABLE_DEBUGGER=1
 
-LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
+  LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
+  LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi
 
-LOCAL_CPP_FEATURES := exceptions
+  LOCAL_CPP_FEATURES := exceptions
 
-LOCAL_STATIC_LIBRARIES := libjsireact libhermes-executor-common-release
-LOCAL_SHARED_LIBRARIES := \
-  libfb \
-  libfbjni \
-  libfolly_json \
-  libhermes \
-  libjsi \
-  libreactnativejni
+  LOCAL_STATIC_LIBRARIES := libjsireact libhermes-executor-common-debug
+  LOCAL_SHARED_LIBRARIES := \
+    libfb \
+    libfbjni \
+    libfolly_json \
+    libhermes \
+    libjsi \
+    libreactnativejni
 
-include $(BUILD_SHARED_LIBRARY)
+  include $(BUILD_SHARED_LIBRARY)
+else
+  include $(CLEAR_VARS)
 
+  LOCAL_MODULE := hermes-executor-release
 
-include $(CLEAR_VARS)
+  LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
 
-LOCAL_MODULE := hermes-executor-debug
-LOCAL_CFLAGS := -DHERMES_ENABLE_DEBUGGER=1
+  LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi
 
-LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
+  LOCAL_CPP_FEATURES := exceptions
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
+  LOCAL_STATIC_LIBRARIES := libjsireact libhermes-executor-common-release
+  LOCAL_SHARED_LIBRARIES := \
+    libfb \
+    libfbjni \
+    libfolly_json \
+    libhermes \
+    libjsi \
+    libreactnativejni
 
-LOCAL_CPP_FEATURES := exceptions
-
-LOCAL_STATIC_LIBRARIES := libjsireact libhermes-executor-common-debug
-LOCAL_SHARED_LIBRARIES := \
-  libfb \
-  libfbjni \
-  libfolly_json \
-  libhermes \
-  libjsi \
-  libreactnativejni
-
-include $(BUILD_SHARED_LIBRARY)
+  include $(BUILD_SHARED_LIBRARY)
+endif

ReactAndroid/src/main/jni/first-party/hermes/Android.mk

@@ -1,10 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-LOCAL_PATH:= $(call my-dir)
-include $(CLEAR_VARS)
-LOCAL_MODULE:= hermes
-LOCAL_SRC_FILES := jni/$(TARGET_ARCH_ABI)/libhermes.so
-include $(PREBUILT_SHARED_LIBRARY)

ReactAndroid/src/main/jni/react/jni/Android.mk

@@ -143,11 +143,18 @@ $(call import-module,jsiexecutor)
 $(call import-module,logger)
 $(call import-module,callinvoker)
 $(call import-module,reactperflogger)
-$(call import-module,hermes)
 $(call import-module,runtimeexecutor)
 $(call import-module,react/renderer/runtimescheduler)
 $(call import-module,react/nativemodule/core)
 
+# This block is needed only because we build the project on NDK r17 internally.
+ifneq ($(call ndk-major-at-least,21),true)
+    $(call import-add-path,$(NDK_GRADLE_INJECTED_IMPORT_PATH))
+endif
+
+$(call import-module,prefab/hermes-engine)
+
+
 include $(REACT_SRC_DIR)/reactperflogger/jni/Android.mk
 # TODO (T48588859): Restructure this target to align with dir structure: "react/nativemodule/..."
 # Note: Update this only when ready to minimize breaking changes.

ReactCommon/common.mk

@@ -1,29 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-##
-# Returns the absolute path to the specified npm package, searching from the
-# given base directory. This function uses Node's module resolution algorithm
-# searching "node_modules" in the base directory and its ancestors. If no
-# matching package is found, this function returns an empty string.
-#
-# The first argument to this function is the base directory from which to begin
-# searching. The second argument is the name of the npm package.
-#
-# Ex: $(call find-node-module,$(LOCAL_PATH),hermes-engine)
-###
-define find-node-module
-$(strip \
-	$(eval _base := $(strip $(1))) \
-	$(eval _package := $(strip $(2))) \
-	$(eval _candidate := $(abspath $(_base)/node_modules/$(_package))) \
-	$(if $(realpath $(_candidate)), \
-		$(_candidate), \
-		$(if $(_base), \
-				$(call find-node-module,$(patsubst %/,%,$(dir $(_base))),$(_package)) \
-		) \
-	) \
-)
-endef

ReactCommon/cxxreact/Android.mk

@@ -42,6 +42,9 @@ $(call import-module,jsc)
 $(call import-module,glog)
 $(call import-module,jsi)
 $(call import-module,jsinspector)
-$(call import-module,hermes/inspector)
 $(call import-module,hermes/executor)
 $(call import-module,logger)
+
+ifeq ($(APP_OPTIM),debug)
+  $(call import-module,hermes/inspector)
+endif

ReactCommon/hermes/executor/Android.mk

@@ -6,32 +6,33 @@
 LOCAL_PATH := $(call my-dir)
 REACT_NATIVE := $(LOCAL_PATH)/../../..
 
-include $(REACT_NATIVE)/ReactCommon/common.mk
-include $(CLEAR_VARS)
+ifeq ($(APP_OPTIM),debug)
+  include $(CLEAR_VARS)
 
-LOCAL_MODULE := hermes-executor-common-release
+  LOCAL_MODULE := hermes-executor-common-debug
+  LOCAL_CFLAGS := -DHERMES_ENABLE_DEBUGGER=1
 
-LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
+  LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
-LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
+  LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi
+  LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
 
-LOCAL_STATIC_LIBRARIES := libjsireact
-LOCAL_SHARED_LIBRARIES := libhermes libjsi
+  LOCAL_STATIC_LIBRARIES := libjsireact libhermes-inspector
+  LOCAL_SHARED_LIBRARIES := libhermes libjsi
 
-include $(BUILD_STATIC_LIBRARY)
+  include $(BUILD_STATIC_LIBRARY)
+else
+  include $(CLEAR_VARS)
 
-include $(CLEAR_VARS)
+  LOCAL_MODULE := hermes-executor-common-release
 
-LOCAL_MODULE := hermes-executor-common-debug
-LOCAL_CFLAGS := -DHERMES_ENABLE_DEBUGGER=1
+  LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
 
-LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp)
+  LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi
+  LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
 
-LOCAL_C_INCLUDES := $(LOCAL_PATH) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
-LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)
+  LOCAL_STATIC_LIBRARIES := libjsireact
+  LOCAL_SHARED_LIBRARIES := libhermes libjsi
 
-LOCAL_STATIC_LIBRARIES := libjsireact libhermes-inspector
-LOCAL_SHARED_LIBRARIES := libhermes libjsi
-
-include $(BUILD_STATIC_LIBRARY)
+  include $(BUILD_STATIC_LIBRARY)
+endif

ReactCommon/hermes/inspector/Android.mk

@@ -6,7 +6,6 @@
 LOCAL_PATH := $(call my-dir)
 REACT_NATIVE := $(LOCAL_PATH)/../../..
 
-include $(REACT_NATIVE)/ReactCommon/common.mk
 include $(CLEAR_VARS)
 
 LOCAL_MODULE := hermes-inspector
@@ -16,7 +15,7 @@ LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/*.cpp $(LOCAL_PATH)/detail/*.cpp $(L
 LOCAL_C_ROOT := $(LOCAL_PATH)/../..
 
 LOCAL_CFLAGS := -DHERMES_ENABLE_DEBUGGER=1 -DHERMES_INSPECTOR_FOLLY_KLUDGE=1
-LOCAL_C_INCLUDES := $(LOCAL_C_ROOT) $(REACT_NATIVE)/ReactCommon/jsi $(call find-node-module,$(LOCAL_PATH),hermes-engine)/android/include
+LOCAL_C_INCLUDES := $(LOCAL_C_ROOT) $(REACT_NATIVE)/ReactCommon/jsi
 LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_ROOT)
 
 LOCAL_CPP_FEATURES := exceptions

build.gradle.kts

@@ -25,6 +25,9 @@ allprojects {
         maven {
             url = uri("$rootDir/node_modules/jsc-android/dist")
         }
+        maven {
+            url = uri("$rootDir/android")
+        }
         google()
         mavenCentral {
             // We don't want to fetch react-native from Maven Central as there are

gradle.properties

@@ -11,3 +11,8 @@ kotlin_version=1.6.10
 # You can also override it from the CLI using
 # ./gradlew <task> -PreactNativeArchitectures=x86_64
 reactNativeArchitectures=armeabi-v7a,arm64-v8a,x86,x86_64
+
+# Use this property if hermes should be built from source or not.
+# If set to true, ReactAndroid will depend on :packages:hermes-engine and will build it from source.
+# If set to false, ReactAndroid will depend a hermes .aar which should be placed inside ./android folder.
+buildHermesFromSource=true