From 08ab45535df4548504befcfb9987b57baf65b29a Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Wed, 7 Jun 2023 16:34:02 -0700 Subject: [PATCH] build: use m1 hosts to build apple silicon binaries (#38258) * build: try m1 speed test * yolo? * build: fix gn binary on m1 * build: remove arm64 snapshot logic * build: strip x64 on arm64 hosts... * if is fi * build: no more GENERATE_CROSS_ARCH_SNAPSHOT * build: chromedriver inline on arm64 darwin * build: no copy chromedriver * build: use m1 for publish * medium? --- .circleci/config/base.yml | 123 ++++++++++---------- BUILD.gn | 22 +--- build/fake_v8_context_snapshot_generator.py | 8 -- 3 files changed, 64 insertions(+), 89 deletions(-) delete mode 100644 build/fake_v8_context_snapshot_generator.py diff --git a/.circleci/config/base.yml b/.circleci/config/base.yml index f9ab0243b5..74304cb1c3 100644 --- a/.circleci/config/base.yml +++ b/.circleci/config/base.yml @@ -52,7 +52,7 @@ executors: size: description: "macOS executor size" type: enum - enum: ["macos.x86.medium.gen2"] + enum: ["macos.x86.medium.gen2", "macos.m1.medium.gen1"] version: description: "xcode version" type: enum @@ -119,9 +119,8 @@ env-arm: &env-arm TARGET_ARCH: arm env-apple-silicon: &env-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64" use_prebuilt_v8_context_snapshot = true' + GN_EXTRA_ARGS: 'target_cpu = "arm64"' TARGET_ARCH: arm64 - USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 npm_config_arch: arm64 env-runner: &env-runner @@ -138,10 +137,9 @@ env-mas: &env-mas MAS_BUILD: 'true' env-mas-apple-silicon: &env-mas-apple-silicon - GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true use_prebuilt_v8_context_snapshot = true' + GN_EXTRA_ARGS: 'target_cpu = "arm64" is_mas_build = true' MAS_BUILD: 'true' TARGET_ARCH: arm64 - USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 npm_config_arch: arm64 env-send-slack-notifications: &env-send-slack-notifications @@ -398,18 +396,24 @@ step-get-more-space-on-mac: &step-get-more-space-on-mac fi } - strip_arm_deep() { + strip_universal_deep() { opwd=$(pwd) cd $1 f=$(find . -perm +111 -type f) for fp in $f do if [[ $(file "$fp") == *"universal binary"* ]]; then - if [[ $(file "$fp") == *"arm64e)"* ]]; then - sudo lipo -remove arm64e "$fp" -o "$fp" || true - fi - if [[ $(file "$fp") == *"arm64)"* ]]; then - sudo lipo -remove arm64 "$fp" -o "$fp" || true + if [ "`arch`" == "arm64" ]; then + if [[ $(file "$fp") == *"x86_64"* ]]; then + sudo lipo -remove x86_64 "$fp" -o "$fp" || true + fi + else + if [[ $(file "$fp") == *"arm64e)"* ]]; then + sudo lipo -remove arm64e "$fp" -o "$fp" || true + fi + if [[ $(file "$fp") == *"arm64)"* ]]; then + sudo lipo -remove arm64 "$fp" -o "$fp" || true + fi fi fi done @@ -465,7 +469,7 @@ step-get-more-space-on-mac: &step-get-more-space-on-mac sudo rm -rf ~/project/src/third_party/angle/third_party/VK-GL-CTS # lipo off some huge binaries arm64 versions to save space - strip_arm_deep $(xcode-select -p)/../SharedFrameworks + strip_universal_deep $(xcode-select -p)/../SharedFrameworks # strip_arm_deep /System/Volumes/Data/Library/Developer/CommandLineTools/usr fi background: true @@ -524,6 +528,12 @@ step-fix-sync: &step-fix-sync # Remove extra output from calling gclient getdep which always calls update_depot_tools sed -i '' "s/Updating depot_tools... //g" esbuild_ensure_file cipd ensure --root src/third_party/devtools-frontend/src/third_party/esbuild -ensure-file esbuild_ensure_file + + # Fix gn (wrong binary) + echo 'gn/gn/mac-${arch}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/mac:gn/gn/mac-${arch}'` > gn_ensure_file + # Remove extra output from calling gclient getdep which always calls update_depot_tools + sed -i '' "s/Updating depot_tools... //g" gn_ensure_file + cipd ensure --root src/buildtools/mac -ensure-file gn_ensure_file fi # Make sure we are using the right ninja @@ -604,7 +614,7 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build name: Build chromedriver.zip command: | cd src - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" export CHROMEDRIVER_DIR="out/chromedriver" else @@ -615,7 +625,7 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver fi ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then cp out/chromedriver/chromedriver.zip out/Default fi @@ -692,7 +702,7 @@ step-verify-mksnapshot: &step-verify-mksnapshot command: | if [ "$IS_ASAN" != "1" ]; then cd src - if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then + if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --snapshot-files-dir $PWD/cross-arch-snapshots else python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default @@ -735,17 +745,15 @@ step-mksnapshot-build: &step-mksnapshot-build no_output_timeout: 30m command: | cd src - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ]; then - ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES - gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args - # Remove unused args from mksnapshot_args - SEDOPTION="-i" - if [ "`uname`" == "Darwin" ]; then - SEDOPTION="-i ''" - fi - sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args - sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args + ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + # Remove unused args from mksnapshot_args + SEDOPTION="-i" + if [ "`uname`" == "Darwin" ]; then + SEDOPTION="-i ''" fi + sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args + sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args if [ "`uname`" != "Darwin" ]; then if [ "$TARGET_ARCH" == "arm" ]; then electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot @@ -758,7 +766,7 @@ step-mksnapshot-build: &step-mksnapshot-build electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator fi fi - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" != "1" ] && [ "$SKIP_DIST_ZIP" != "1" ]; then + if [ "$SKIP_DIST_ZIP" != "1" ]; then ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) fi @@ -1221,32 +1229,6 @@ commands: no_output_timeout: 60m command: | cd src - - # Lets generate a snapshot and mksnapshot and then delete all the x-compiled generated files to save space - if [ "$USE_PREBUILT_V8_CONTEXT_SNAPSHOT" == "1" ]; then - ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES - ninja -C out/Default tools/v8_context_snapshot -j $NUMBER_OF_NINJA_PROCESSES - gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args - # Remove unused args from mksnapshot_args - SEDOPTION="-i" - if [ "`uname`" == "Darwin" ]; then - SEDOPTION="-i ''" - fi - sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args - sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args - (cd out/Default; zip mksnapshot.zip mksnapshot_args clang_x64_v8_arm64/gen/v8/embedded.S) - if [ "<< parameters.clean-prebuilt-snapshot >>" == "true" ]; then - rm -rf out/Default/clang_x64_v8_arm64/gen - rm -rf out/Default/clang_x64_v8_arm64/obj - rm -rf out/Default/clang_x64_v8_arm64/thinlto-cache - rm -rf out/Default/clang_x64/obj - # Regenerate because we just deleted some ninja files - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - fi - # For x-compiles this will be built to the wrong arch after the context snapshot build - # so we wipe it before re-linking it below - rm -rf out/Default/libffmpeg.dylib - fi NINJA_SUMMARIZE_BUILD=1 autoninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES cp out/Default/.ninja_log out/electron_ninja_log node electron/script/check-symlinks.js @@ -1665,7 +1647,7 @@ jobs: artifact-key: 'nil' build-type: 'nil' - mac-make-src-cache: + mac-make-src-cache-x64: executor: name: linux-docker size: xlarge @@ -1684,6 +1666,25 @@ jobs: artifact-key: 'nil' build-type: 'nil' + mac-make-src-cache-arm64: + executor: + name: linux-docker + size: xlarge + environment: + <<: *env-linux-2xlarge + <<: *env-testing-build + <<: *env-macos-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac --custom-var=host_cpu=arm64' + steps: + - electron-build: + persist: false + build: false + checkout: true + save-git-cache: true + checkout-to-create-src-cache: true + artifact-key: 'nil' + build-type: 'nil' + # Layer 2: Builds. linux-x64-testing: executor: @@ -1941,7 +1942,7 @@ jobs: osx-publish-arm64: executor: name: macos - size: macos.x86.medium.gen2 + size: macos.m1.medium.gen1 environment: <<: *env-mac-large-release <<: *env-release-build @@ -1964,7 +1965,7 @@ jobs: osx-testing-arm64: executor: name: macos - size: macos.x86.medium.gen2 + size: macos.m1.medium.gen1 environment: <<: *env-mac-large <<: *env-testing-build @@ -1972,7 +1973,6 @@ jobs: <<: *env-macos-build <<: *env-apple-silicon GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - GENERATE_CROSS_ARCH_SNAPSHOT: true steps: - electron-build: persist: true @@ -2023,7 +2023,7 @@ jobs: mas-publish-arm64: executor: name: macos - size: macos.x86.medium.gen2 + size: macos.m1.medium.gen1 environment: <<: *env-mac-large-release <<: *env-mas-apple-silicon @@ -2271,19 +2271,20 @@ workflows: - equal: [false, << pipeline.parameters.run-linux-publish >>] - equal: [true, << pipeline.parameters.run-build-mac >>] jobs: - - mac-make-src-cache + - mac-make-src-cache-x64 + - mac-make-src-cache-arm64 - osx-testing-x64: requires: - - mac-make-src-cache + - mac-make-src-cache-x64 - osx-testing-x64-gn-check: requires: - - mac-make-src-cache + - mac-make-src-cache-x64 - darwin-testing-x64-tests: requires: - osx-testing-x64 - osx-testing-arm64: requires: - - mac-make-src-cache + - mac-make-src-cache-arm64 - darwin-testing-arm64-tests: filters: branches: diff --git a/BUILD.gn b/BUILD.gn index ccfc20cde6..9d57b9d8e8 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -99,10 +99,6 @@ if (is_linux) { } } -declare_args() { - use_prebuilt_v8_context_snapshot = false -} - branding = read_file("shell/app/BRANDING.json", "json") electron_project_name = branding.project_name electron_product_name = branding.product_name @@ -797,15 +793,6 @@ if (is_mac) { sources = [ "shell/common/resources/mac/MainMenu.xib" ] } - action("fake_v8_context_snapshot_generator") { - script = "build/fake_v8_context_snapshot_generator.py" - args = [ - rebase_path("$root_out_dir/$v8_context_snapshot_filename"), - rebase_path("$root_out_dir/fake/$v8_context_snapshot_filename"), - ] - outputs = [ "$root_out_dir/fake/$v8_context_snapshot_filename" ] - } - bundle_data("electron_framework_resources") { public_deps = [ ":packed_resources" ] sources = [] @@ -816,13 +803,8 @@ if (is_mac) { if (v8_use_external_startup_data) { public_deps += [ "//v8" ] if (use_v8_context_snapshot) { - if (use_prebuilt_v8_context_snapshot) { - sources += [ "$root_out_dir/fake/$v8_context_snapshot_filename" ] - public_deps += [ ":fake_v8_context_snapshot_generator" ] - } else { - sources += [ "$root_out_dir/$v8_context_snapshot_filename" ] - public_deps += [ "//tools/v8_context_snapshot" ] - } + sources += [ "$root_out_dir/$v8_context_snapshot_filename" ] + public_deps += [ "//tools/v8_context_snapshot" ] } else { sources += [ "$root_out_dir/snapshot_blob.bin" ] } diff --git a/build/fake_v8_context_snapshot_generator.py b/build/fake_v8_context_snapshot_generator.py deleted file mode 100644 index 2309b22484..0000000000 --- a/build/fake_v8_context_snapshot_generator.py +++ /dev/null @@ -1,8 +0,0 @@ -import os -import shutil -import sys - -if os.path.exists(sys.argv[2]): - os.remove(sys.argv[2]) - -shutil.copy(sys.argv[1], sys.argv[2])