build: [GHA] cross-compile x64 MacOS jobs on arm64 (#42370)

* build: split x64 mas/darwin to run concurrently * Retry src cache download on failure * build: gate FFMpeg, etc, to release & darwin * build: cross-compile x64 on arm hardware * chore (do not merge): comment out CircleCI config * build: fix FFMpeg conditional but harder * build: add fetch-deps to checkout * build: correctly add target_arch to MAS configs * build: correct target arch * build: consolidate darwin/mas back into single runner per arch * build: re-enable CircleCI * Add missing ELECTRON_OUT_DIR for upload * Add missing ELECTRON_GITHUB_TOKEN to secrets * build: (do not merge) run only darwin * build: remove seperate upload step * build: re-enable mas, remove upload seperate job --------- Co-authored-by: Shelley Vohr <shelley.vohr@gmail.com>
2024-06-05 23:06:25 -07:00 · 2024-06-05 23:06:25 -07:00 · c9349a2590
--- a/.github/workflows/config/release/arm64/evm.mas.json
+++ b/.github/workflows/config/release/arm64/evm.mas.json
@ -0,0 +1,26 @@
+{
+  "root": "/Users/runner/work/electron/electron/",
+  "remotes": {
+    "electron": {
+      "origin": "https://github.com/electron/electron.git"
+    }
+  },
+  "gen": {
+    "args": [
+      "import(\"//electron/build/args/release.gn\")",
+      "use_remoteexec = true",
+      "target_cpu = \"arm64\"",
+      "is_mas_build = true"
+    ],
+    "out": "Default"
+  },
+  "env": {
+    "CHROMIUM_BUILDTOOLS_PATH": "/Users/runner/work/electron/electron/src/buildtools",
+    "GIT_CACHE_PATH": "/Users/runner/work/electron/electron/.git-cache"
+  },
+  "$schema": "file:///home/builduser/.electron_build_tools/evm-config.schema.json",
+  "configValidationLevel": "strict",
+  "reclient": "remote_exec",
+  "goma": "none",
+  "preserveXcode": 5
+}
--- a/.github/workflows/config/release/x64/evm.mas.json
+++ b/.github/workflows/config/release/x64/evm.mas.json
@ -9,6 +9,7 @@
    "args": [
      "import(\"//electron/build/args/release.gn\")",
      "use_remoteexec = true",
+      "target_cpu = \"x64\"",
      "is_mas_build = true"
    ],
    "out": "Default"
--- a/.github/workflows/config/testing/arm64/evm.mas.json
+++ b/.github/workflows/config/testing/arm64/evm.mas.json
--- a/.github/workflows/config/testing/x64/evm.mas.json
+++ b/.github/workflows/config/testing/x64/evm.mas.json
@ -0,0 +1,25 @@
+{
+  "root": "/Users/runner/work/electron/electron/",
+  "remotes": {
+    "electron": {
+      "origin": "https://github.com/electron/electron.git"
+    }
+  },
+  "gen": {
+    "args": [
+      "import(\"//electron/build/args/testing.gn\")",
+      "use_remoteexec = true",
+      "is_mas_build = true"
+    ],
+    "out": "Default"
+  },
+  "env": {
+    "CHROMIUM_BUILDTOOLS_PATH": "/Users/runner/work/electron/electron/src/buildtools",
+    "GIT_CACHE_PATH": "/Users/runner/work/electron/electron/.git-cache"
+  },
+  "$schema": "file:///home/builduser/.electron_build_tools/evm-config.schema.json",
+  "configValidationLevel": "strict",
+  "reclient": "remote_exec",
+  "goma": "none",
+  "preserveXcode": 5
+}
--- a/.github/workflows/macos-build.yml
+++ b/.github/workflows/macos-build.yml
@ -25,7 +25,6 @@ on:
        required: false
        type: string
        default: '0'
-      

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@ -36,12 +35,14 @@ env:
  AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
  AZURE_STORAGE_CONTAINER_NAME: ${{ secrets.AZURE_STORAGE_CONTAINER_NAME }}
  ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
+  ELECTRON_GITHUB_TOKEN: ${{ secrets.ELECTRON_GITHUB_TOKEN }}
  GN_CONFIG: ${{ inputs.GN_CONFIG }}
  # Disable pre-compiled headers to reduce out size - only useful for rebuilds
  GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false'
  GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
  CHECK_DIST_MANIFEST: '1'
  IS_GHA_RELEASE: true
+  ELECTRON_OUT_DIR: Default

 jobs:
  checkout:
@ -51,6 +52,7 @@ jobs:
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
+        fetch-depth: 0
    - name: Install Azure CLI
      run: sudo bash ./src/electron/script/azure_cli_deb_install.sh
    - name: Set GIT_CACHE_PATH to make gclient to use the cache
@ -190,23 +192,24 @@ jobs:
  build:
    strategy:
      fail-fast: false
-      # macos-large is x64, macos-xlarge is arm64
-      # More runner information: https://github.com/actions/runner-images/blob/main/README.md#available-images
      matrix:
-        arch: [ macos-14-large, macos-14-xlarge ]
-    runs-on: ${{ matrix.arch }}
+        build-arch: [ arm64, x64 ]
+    # macos-large is x64, macos-xlarge is arm64
+    # More runner information: https://github.com/actions/runner-images/blob/main/README.md#available-images
+    runs-on: macos-14-xlarge
    needs: checkout
    steps:
    - name: Load Build Tools
      run: |
        export BUILD_TOOLS_SHA=2bb63e2e7877491b52f972532b52adc979a6ec2f
        npm i -g @electron/build-tools
-        e init --root=$(pwd) --out=Default ${{ inputs.GN_BUILD_TYPE }} --import ${{ inputs.GN_BUILD_TYPE }}
+        e init --root=$(pwd) --out=Default ${{ inputs.GN_BUILD_TYPE }} --import ${{ inputs.GN_BUILD_TYPE }} --target-cpu ${{ matrix.build-arch}}
        e use ${{ inputs.GN_BUILD_TYPE }}
    - name: Checkout Electron
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
+        fetch-depth: 0
    - name: Install Azure CLI
      run: |
        brew update && brew install azure-cli
@ -222,16 +225,9 @@ jobs:
        node script/yarn install
    - name: Load Target Arch & CPU
      run: |
-        ARCH=$(uname -m)
-        if [ "$ARCH" == "x86_64" ]; then
-          echo "TARGET_ARCH="x64"" >> $GITHUB_ENV
-          echo "target_cpu="x64"" >> $GITHUB_ENV
-          echo "host_cpu="x64"" >> $GITHUB_ENV
-        else
-          echo "TARGET_ARCH="arm64"" >> $GITHUB_ENV
-          echo "target_cpu="arm64"" >> $GITHUB_ENV
-          echo "host_cpu="arm64"" >> $GITHUB_ENV
-        fi
+        echo "TARGET_ARCH=${{ matrix.build-arch }}" >> $GITHUB_ENV
+        echo "target_cpu=${{ matrix.build-arch }}" >> $GITHUB_ENV
+        echo "host_cpu=${{ matrix.build-arch }}" >> $GITHUB_ENV
    - name: Get Depot Tools
      timeout-minutes: 5
      run: |
@ -258,13 +254,18 @@ jobs:
      # The cache will always exist here as a result of the checkout job
      # Either it was uploaded to Azure in the checkout job for this commit
      # or it was uploaded in the checkout job for a previous commit.
-      run: |
-        az storage blob download \
-          --account-name $AZURE_STORAGE_ACCOUNT \
-          --account-key $AZURE_STORAGE_KEY \
-          --container-name $AZURE_STORAGE_CONTAINER_NAME \
-          --name $DEPSHASH \
-          --file $DEPSHASH.tar \
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+      with:
+        timeout_minutes: 20
+        max_attempts: 3
+        retry_on: error
+        command: |
+          az storage blob download \
+            --account-name $AZURE_STORAGE_ACCOUNT \
+            --account-key $AZURE_STORAGE_KEY \
+            --container-name $AZURE_STORAGE_CONTAINER_NAME \
+            --name $DEPSHASH \
+            --file $DEPSHASH.tar \
    - name: Unzip and Ensure Src Cache
      run: |
        echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)"
@ -508,21 +509,33 @@ jobs:
          electron/script/zip-symbols.py -b $BUILD_PATH
        fi
    - name: Generate FFMpeg
-      if: github.event.inputs.IS_RELEASE == 'true'
+      if: ${{ inputs.IS_RELEASE == true }}
      run: |
        cd src
        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
        autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
    - name: Generate Hunspell Dictionaries
-      if: github.event.inputs.IS_RELEASE == 'true'
+      if: ${{ inputs.IS_RELEASE == true }}
      run: |
        cd src
        autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
    - name: Generate TypeScript Definitions
-      if: github.event.inputs.IS_RELEASE == 'true'
+      if: ${{ inputs.IS_RELEASE == true }}
      run: |
        cd src/electron
        node script/yarn create-typescript-definitions
+    # TODO(vertedinde): These uploads currently point to a different Azure bucket & GitHub Repo
+    - name: Publish Electron Dist
+      run: |
+        rm -rf src/out/Default/obj
+        cd src/electron
+        if [ ${{ inputs.UPLOAD_TO_STORAGE }} == "1" ]; then
+          echo 'Uploading Electron release distribution to Azure'
+          script/release/uploaders/upload.py --verbose --upload_to_storage
+        else
+          echo 'Uploading Electron release distribution to GitHub releases'
+          script/release/uploaders/upload.py --verbose
+        fi
    # The current generated_artifacts_<< artifact.key >> name was taken from CircleCI
    # to ensure we don't break anything, but we may be able to improve that.
    - name: Move all Generated Artifacts to Upload Folder
@ -554,7 +567,7 @@ jobs:
        key: ${{ runner.os }}-build-artifacts-darwin-${{ env.TARGET_ARCH }}-${{ github.sha }}
    - name: Create MAS Config
      run: |
-        mv src/electron/.github/workflows/config/${{ inputs.GN_BUILD_TYPE }}/evm.mas.json $HOME/.electron_build_tools/configs/evm.mas.json
+        mv src/electron/.github/workflows/config/${{ inputs.GN_BUILD_TYPE }}/${{ matrix.build-arch }}/evm.mas.json $HOME/.electron_build_tools/configs/evm.mas.json
        echo "MAS_BUILD=true" >> $GITHUB_ENV
        e use mas
    - name: Build Electron (mas)
@ -615,6 +628,18 @@ jobs:
        else
          electron/script/zip-symbols.py -b $BUILD_PATH
        fi
+    # TODO(vertedinde): These uploads currently point to a different Azure bucket & GitHub Repo
+    - name: Publish Electron Dist
+      run: |
+        rm -rf src/out/Default/obj
+        cd src/electron
+        if [ ${{ inputs.UPLOAD_TO_STORAGE }} == "1" ]; then
+          echo 'Uploading Electron release distribution to Azure'
+          script/release/uploaders/upload.py --verbose --upload_to_storage
+        else
+          echo 'Uploading Electron release distribution to GitHub releases'
+          script/release/uploaders/upload.py --verbose
+        fi
    - name: Move all Generated Artifacts to Upload Folder (mas)
      run: ./src/electron/script/actions/move-artifacts.sh
    - name: Upload Generated Artifacts
@ -644,73 +669,6 @@ jobs:
          src/out/Default/obj/buildtools/third_party
          src/v8/tools/builtins-pgo
        key: ${{ runner.os }}-build-artifacts-mas-${{ env.TARGET_ARCH }}-${{ github.sha }}
-  upload:
-    runs-on: LargeLinuxRunner
-    if: ${{ inputs.IS_RELEASE == true }}
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        build-type: [darwin, mas]
-        arch: [x64, arm64]
-    steps:
-    - name: Checkout Electron
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
-      with:
-        path: src/electron
-    - name: Setup Node.js/npm
-      uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8
-      with:
-        node-version: 20.11.x
-        cache: yarn
-        cache-dependency-path: src/electron/yarn.lock
-    - name: Load Target Arch
-      run: echo "TARGET_ARCH=${{ matrix.arch }}" >> $GITHUB_ENV
-    - name: Install Dependencies
-      run: |
-        cd src/electron
-        node script/yarn install
-    - name: Download Generated Artifacts
-      uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e 
-      with:
-        name: generated_artifacts_${{ matrix.build-type }}_${{ matrix.arch }}
-        path: ./generated_artifacts_${{ matrix.build-type }}_${{ matrix.arch }}
-    - name: Restore Persisted Build Artifacts
-      uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
-      with:
-        path: |
-          src/out/Default/gen/node_headers
-          src/out/Default/overlapped-checker
-          src/out/Default/ffmpeg
-          src/out/Default/hunspell_dictionaries
-          src/electron
-          src/third_party/electron_node
-          src/third_party/nan
-          src/cross-arch-snapshots
-          src/third_party/llvm-build
-          src/build/linux
-          src/buildtools/mac
-          src/buildtools/third_party/libc++
-          src/buildtools/third_party/libc++abi
-          src/third_party/libc++
-          src/third_party/libc++abi
-          src/out/Default/obj/buildtools/third_party
-          src/v8/tools/builtins-pgo
-        key: macOS-build-artifacts-${{ matrix.build-type }}-${{ matrix.arch }}-${{ github.sha }}
-    - name: Restore Generated Artifacts
-      run: ./src/electron/script/actions/restore-artifacts.sh
-    # TODO(vertedinde): These uploads currently point to a different Azure bucket & GitHub Repo
-    - name: Publish Electron Dist
-      run: |
-        rm -rf src/out/Default/obj
-        cd src/electron
-        if [ ${{ inputs.UPLOAD_TO_STORAGE }} == "1" ]; then
-          echo 'Uploading Electron release distribution to Azure'
-          script/release/uploaders/upload.py --verbose --upload_to_storage
-        else
-          echo 'Uploading Electron release distribution to GitHub releases'
-          script/release/uploaders/upload.py --verbose
-        fi
  test:
    if: ${{ inputs.IS_RELEASE == false }}
    runs-on: macos-14-xlarge
@ -718,7 +676,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        build-type: [darwin, mas]
+        build-type: [ darwin, mas ]
    env:
      BUILD_TYPE: ${{ matrix.build-type }}
    steps:
@ -731,6 +689,7 @@ jobs:
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
+        fetch-depth: 0
    - name: Setup Node.js/npm
      uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8
      with: