build: use m1 hosts to build apple silicon binaries (#38258)

* build: try m1 speed test

* yolo?

* build: fix gn binary on m1

* build: remove arm64 snapshot logic

* build: strip x64 on arm64 hosts...

* if is fi

* build: no more GENERATE_CROSS_ARCH_SNAPSHOT

* build: chromedriver inline on arm64 darwin

* build: no copy chromedriver

* build: use m1 for publish

* medium?

.circleci/config/base.yml

@@ -52,7 +52,7 @@ executors:
       size:
         description: "macOS executor size"
         type: enum
-        enum: ["macos.x86.medium.gen2"]
+        enum: ["macos.x86.medium.gen2", "macos.m1.medium.gen1"]
       version:
         description: "xcode version"
         type: enum
@@ -119,9 +119,8 @@ env-arm: &env-arm
   TARGET_ARCH: arm
 
 env-apple-silicon: &env-apple-silicon
-  GN_EXTRA_ARGS: 'target_cpu = "arm64" use_prebuilt_v8_context_snapshot = true'
+  GN_EXTRA_ARGS: 'target_cpu = "arm64"'
   TARGET_ARCH: arm64
-  USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1
   npm_config_arch: arm64
 
 env-runner: &env-runner
@@ -138,10 +137,9 @@ env-mas: &env-mas
   MAS_BUILD: 'true'
 
 env-mas-apple-silicon: &env-mas-apple-silicon
-  GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true use_prebuilt_v8_context_snapshot = true'
+  GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true'
   MAS_BUILD: 'true'
   TARGET_ARCH: arm64
-  USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1
   npm_config_arch: arm64
 
 env-send-slack-notifications: &env-send-slack-notifications
@@ -398,18 +396,24 @@ step-get-more-space-on-mac: &step-get-more-space-on-mac
           fi
         }
 
-        strip_arm_deep() {
+        strip_universal_deep() {
           opwd=$(pwd)
           cd $1
           f=$(find . -perm +111 -type f)
           for fp in $f
           do
             if [[ $(file "$fp") == *"universal binary"* ]]; then
-              if [[ $(file "$fp") == *"arm64e)"* ]]; then
-                sudo lipo -remove arm64e "$fp" -o "$fp" || true
-              fi
-              if [[ $(file "$fp") == *"arm64)"* ]]; then
-                sudo lipo -remove arm64 "$fp" -o "$fp" || true
+              if [ "`arch`" == "arm64" ]; then
+                if [[ $(file "$fp") == *"x86_64"* ]]; then
+                  sudo lipo -remove x86_64 "$fp" -o "$fp" || true
+                fi
+              else
+                if [[ $(file "$fp") == *"arm64e)"* ]]; then
+                  sudo lipo -remove arm64e "$fp" -o "$fp" || true
+                fi
+                if [[ $(file "$fp") == *"arm64)"* ]]; then
+                  sudo lipo -remove arm64 "$fp" -o "$fp" || true
+                fi
               fi
             fi
           done
@@ -465,7 +469,7 @@ step-get-more-space-on-mac: &step-get-more-space-on-mac
         sudo rm -rf ~/project/src/third_party/angle/third_party/VK-GL-CTS
 
         # lipo off some huge binaries arm64 versions to save space
-        strip_arm_deep $(xcode-select -p)/../SharedFrameworks
+        strip_universal_deep $(xcode-select -p)/../SharedFrameworks
         # strip_arm_deep /System/Volumes/Data/Library/Developer/CommandLineTools/usr
       fi
     background: true
@@ -524,6 +528,12 @@ step-fix-sync: &step-fix-sync
         # Remove extra output from calling gclient getdep which always calls update_depot_tools
         sed -i '' "s/Updating depot_tools... //g" esbuild_ensure_file
         cipd ensure --root src/third_party/devtools-frontend/src/third_party/esbuild -ensure-file esbuild_ensure_file
+
+        # Fix gn (wrong binary)
+        echo 'gn/gn/mac-${arch}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/mac:gn/gn/mac-${arch}'` > gn_ensure_file
+        # Remove extra output from calling gclient getdep which always calls update_depot_tools
+        sed -i '' "s/Updating depot_tools... //g" gn_ensure_file
+        cipd ensure --root src/buildtools/mac -ensure-file gn_ensure_file
       fi
 
       # Make sure we are using the right ninja
@@ -604,7 +614,7 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build
     name: Build chromedriver.zip
     command: |
       cd src
-      if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then
+      if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then
         gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
         export CHROMEDRIVER_DIR="out/chromedriver"
       else
@@ -615,7 +625,7 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build
         electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver
       fi
       ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip
-      if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then
+      if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then
         cp out/chromedriver/chromedriver.zip out/Default
       fi
 
@@ -692,7 +702,7 @@ step-verify-mksnapshot: &step-verify-mksnapshot
     command: |
       if [ "$IS_ASAN" != "1" ]; then
         cd src
-        if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then
+        if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then
           python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --snapshot-files-dir $PWD/cross-arch-snapshots
         else
           python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default
@@ -735,17 +745,15 @@ step-mksnapshot-build: &step-mksnapshot-build
     no_output_timeout: 30m
     command: |
       cd src
-      if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ]; then
-        ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
-        gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
-        # Remove unused args from mksnapshot_args
-        SEDOPTION="-i"
-        if [ "`uname`" == "Darwin" ]; then
-          SEDOPTION="-i ''"
-        fi
-        sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
-        sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
+      ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
+      gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
+      # Remove unused args from mksnapshot_args
+      SEDOPTION="-i"
+      if [ "`uname`" == "Darwin" ]; then
+        SEDOPTION="-i ''"
       fi
+      sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
+      sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
       if [ "`uname`" != "Darwin" ]; then
         if [ "$TARGET_ARCH" == "arm" ]; then
           electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
@@ -758,7 +766,7 @@ step-mksnapshot-build: &step-mksnapshot-build
           electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator
         fi
       fi
-      if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ] && [ "$SKIP_DIST_ZIP" != "1" ]; then
+      if [ "$SKIP_DIST_ZIP" != "1" ]; then
         ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
         (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S)
       fi
@@ -1221,32 +1229,6 @@ commands:
           no_output_timeout: 60m
           command: |
             cd src
-
-            # Lets generate a snapshot and mksnapshot and then delete all the x-compiled generated files to save space
-            if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" == "1" ]; then
-              ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
-              ninja -C out/Default tools/v8_context_snapshot -j $NUMBER_OF_NINJA_PROCESSES
-              gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
-              # Remove unused args from mksnapshot_args
-              SEDOPTION="-i"
-              if [ "`uname`" == "Darwin" ]; then
-                SEDOPTION="-i ''"
-              fi
-              sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
-              sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
-              (cd out/Default; zip mksnapshot.zip mksnapshot_args clang_x64_v8_arm64/gen/v8/embedded.S)
-              if [ "<< parameters.clean-prebuilt-snapshot >>" == "true" ]; then
-                rm -rf out/Default/clang_x64_v8_arm64/gen
-                rm -rf out/Default/clang_x64_v8_arm64/obj
-                rm -rf out/Default/clang_x64_v8_arm64/thinlto-cache
-                rm -rf out/Default/clang_x64/obj
-                # Regenerate because we just deleted some ninja files
-                gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
-              fi
-              # For x-compiles this will be built to the wrong arch after the context snapshot build
-              # so we wipe it before re-linking it below
-              rm -rf out/Default/libffmpeg.dylib
-            fi
             NINJA_SUMMARIZE_BUILD=1 autoninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES
             cp out/Default/.ninja_log out/electron_ninja_log
             node electron/script/check-symlinks.js
@@ -1665,7 +1647,7 @@ jobs:
           artifact-key: 'nil'
           build-type: 'nil'
 
-  mac-make-src-cache:
+  mac-make-src-cache-x64:
     executor:
       name: linux-docker
       size: xlarge
@@ -1684,6 +1666,25 @@ jobs:
           artifact-key: 'nil'
           build-type: 'nil'
 
+  mac-make-src-cache-arm64:
+    executor:
+      name: linux-docker
+      size: xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      <<: *env-testing-build
+      <<: *env-macos-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac --custom-var=host_cpu=arm64'
+    steps:
+      - electron-build:
+          persist: false
+          build: false
+          checkout: true
+          save-git-cache: true
+          checkout-to-create-src-cache: true
+          artifact-key: 'nil'
+          build-type: 'nil'
+
   # Layer 2: Builds.
   linux-x64-testing:
     executor:
@@ -1941,7 +1942,7 @@ jobs:
   osx-publish-arm64:
     executor:
       name: macos
-      size: macos.x86.medium.gen2
+      size: macos.m1.medium.gen1
     environment:
       <<: *env-mac-large-release
       <<: *env-release-build
@@ -1964,7 +1965,7 @@ jobs:
   osx-testing-arm64:
     executor:
       name: macos
-      size: macos.x86.medium.gen2
+      size: macos.m1.medium.gen1
     environment:
       <<: *env-mac-large
       <<: *env-testing-build
@@ -1972,7 +1973,6 @@ jobs:
       <<: *env-macos-build
       <<: *env-apple-silicon
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
-      GENERATE_CROSS_ARCH_SNAPSHOT: true
     steps:
       - electron-build:
           persist: true
@@ -2023,7 +2023,7 @@ jobs:
   mas-publish-arm64:
     executor:
       name: macos
-      size: macos.x86.medium.gen2
+      size: macos.m1.medium.gen1
     environment:
       <<: *env-mac-large-release
       <<: *env-mas-apple-silicon
@@ -2271,19 +2271,20 @@ workflows:
         - equal: [false, << pipeline.parameters.run-linux-publish >>]
         - equal: [true, << pipeline.parameters.run-build-mac >>]
     jobs:
-      - mac-make-src-cache
+      - mac-make-src-cache-x64
+      - mac-make-src-cache-arm64
       - osx-testing-x64:
           requires:
-            - mac-make-src-cache
+            - mac-make-src-cache-x64
       - osx-testing-x64-gn-check:
           requires:
-            - mac-make-src-cache
+            - mac-make-src-cache-x64
       - darwin-testing-x64-tests:
           requires:
             - osx-testing-x64
       - osx-testing-arm64:
           requires:
-            - mac-make-src-cache
+            - mac-make-src-cache-arm64
       - darwin-testing-arm64-tests:
           filters:
             branches:

BUILD.gn

@@ -99,10 +99,6 @@ if (is_linux) {
   }
 }
 
-declare_args() {
-  use_prebuilt_v8_context_snapshot = false
-}
-
 branding = read_file("shell/app/BRANDING.json", "json")
 electron_project_name = branding.project_name
 electron_product_name = branding.product_name
@@ -797,15 +793,6 @@ if (is_mac) {
     sources = [ "shell/common/resources/mac/MainMenu.xib" ]
   }
 
-  action("fake_v8_context_snapshot_generator") {
-    script = "build/fake_v8_context_snapshot_generator.py"
-    args = [
-      rebase_path("$root_out_dir/$v8_context_snapshot_filename"),
-      rebase_path("$root_out_dir/fake/$v8_context_snapshot_filename"),
-    ]
-    outputs = [ "$root_out_dir/fake/$v8_context_snapshot_filename" ]
-  }
-
   bundle_data("electron_framework_resources") {
     public_deps = [ ":packed_resources" ]
     sources = []
@@ -816,13 +803,8 @@ if (is_mac) {
     if (v8_use_external_startup_data) {
       public_deps += [ "//v8" ]
       if (use_v8_context_snapshot) {
-        if (use_prebuilt_v8_context_snapshot) {
-          sources += [ "$root_out_dir/fake/$v8_context_snapshot_filename" ]
-          public_deps += [ ":fake_v8_context_snapshot_generator" ]
-        } else {
-          sources += [ "$root_out_dir/$v8_context_snapshot_filename" ]
-          public_deps += [ "//tools/v8_context_snapshot" ]
-        }
+        sources += [ "$root_out_dir/$v8_context_snapshot_filename" ]
+        public_deps += [ "//tools/v8_context_snapshot" ]
       } else {
         sources += [ "$root_out_dir/snapshot_blob.bin" ]
       }

build/fake_v8_context_snapshot_generator.py

@@ -1,8 +0,0 @@
-import os
-import shutil
-import sys
-
-if os.path.exists(sys.argv[2]):
-  os.remove(sys.argv[2])
-
-shutil.copy(sys.argv[1], sys.argv[2])