build: use the linux aks for macos checkout/cache (#42447)

* build: use the linux aks for checkout/cache

* build: use latest container sha

* build: persist the AKS cache (first try)

* build (do not merge): use current branch for workflows

* build: do not check for existing cache

* build: try to move src into /var/portal

* build: add AZURE_AKS_CACHE_SHARE_NAME & AZURE_AKS_CACHE_STORAGE_ACCOUNT

* build: add volumes: /mnt/cross-instance-cache

* build: temporarily skip cache exists checks

* build: reset to aks-linux-large, upload/download tar

* build: pass detailed credentials into download

* build: be better

* build: add skip if cache exists, declare env vars earlier

* build: add quotes

* build: change to connection-string

* build: remove connection string

* build: migrate to azcopy

* build: lol

* build: revert ref to @main

.github/workflows/macos-pipeline.yml

@@ -34,10 +34,17 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  # Old Azure Storage Variables
   AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
   AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
   AZURE_STORAGE_CONTAINER_NAME: ${{ secrets.AZURE_STORAGE_CONTAINER_NAME }}
   ELECTRON_ARTIFACTS_BLOB_STORAGE: ${{ secrets.ELECTRON_ARTIFACTS_BLOB_STORAGE }}
+  # New Azure Storage Variables
+  AZURE_AKS_CACHE_STORAGE_KEY: ${{ secrets.AZURE_AKS_CACHE_STORAGE_KEY }}
+  AZURE_AKS_CACHE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_AKS_CACHE_STORAGE_CONNECTION_STRING }}
+  AZURE_AKS_CACHE_STORAGE_ACCOUNT: ${{ secrets.AZURE_AKS_CACHE_STORAGE_ACCOUNT }}
+  AZURE_AKS_CACHE_SHARE_NAME: ${{ secrets.AZURE_AKS_CACHE_SHARE_NAME }}
+  AZURE_AKS_CACHE_SAS_TOKEN: ${{ secrets.AZURE_AKS_CACHE_SAS_TOKEN }}
   ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
   ELECTRON_GITHUB_TOKEN: ${{ secrets.ELECTRON_GITHUB_TOKEN }}
   GN_CONFIG: ${{ inputs.gn-config }}
@@ -50,7 +57,13 @@ env:
 
 jobs:
   checkout:
-    runs-on: LargeLinuxRunner
+    runs-on: aks-linux-large
+    container:
+      image: ghcr.io/electron/build:latest
+      options: --user root
+      volumes:
+        - /mnt/cross-instance-cache:/mnt/cross-instance-cache
+        - /var/run/sas:/var/run/sas
     steps:
     - name: Checkout Electron
       uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
@@ -60,12 +73,8 @@ jobs:
     - name: Set GIT_CACHE_PATH to make gclient to use the cache
       run: |
         echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV
-    - name: Setup Node.js/npm
-      uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8
-      with:
-        node-version: 20.11.x
-        cache: yarn
-        cache-dependency-path: src/electron/yarn.lock
+    - name: Install Azure CLI
+      run: sudo bash ./src/electron/script/azure_cli_deb_install.sh
     - name: Install Dependencies
       run: |
         cd src/electron
@@ -95,19 +104,14 @@ jobs:
     - name: Check If Cache Exists
       id: check-cache
       run: |
-        exists_json=$(az storage blob exists \
-          --account-name $AZURE_STORAGE_ACCOUNT \
-          --account-key $AZURE_STORAGE_KEY \
-          --container-name $AZURE_STORAGE_CONTAINER_NAME \
-          --name $DEPSHASH)
-
-        cache_exists=$(echo $exists_json | jq -r '.exists')
-        echo "cache_exists=$cache_exists" >> $GITHUB_OUTPUT
-
-        if (test "$cache_exists" = "true"); then
-          echo "Cache Exists for $DEPSHASH"
-        else
+        cache_key=$DEPSHASH
+        cache_path=/mnt/cross-instance-cache/${cache_key}.tar
+        echo "Using cache key: $cache_key"
+        echo "Checking for cache in: $cache_path"
+        if [ ! -f "$cache_path" ]; then
           echo "Cache Does Not Exist for $DEPSHASH"
+        else
+          echo "Cache Already Exists for $DEPSHASH, Skipping.."
         fi
     - name: Gclient Sync
       if: steps.check-cache.outputs.cache_exists == 'false'
@@ -179,16 +183,36 @@ jobs:
         echo "Uncompressed src size: $(du -sh src | cut -f1 -d' ')"
         tar -cvf $DEPSHASH.tar src
         echo "Compressed src to $(du -sh $DEPSHASH.tar | cut -f1 -d' ')"
-    - name: Upload Compressed Src Cache to Azure
+    - name: Move src folder to cross-OS portal
       if: steps.check-cache.outputs.cache_exists == 'false'
       run: |
-        az storage blob upload \
-          --account-name $AZURE_STORAGE_ACCOUNT \
-          --account-key $AZURE_STORAGE_KEY \
-          --container-name $AZURE_STORAGE_CONTAINER_NAME \
-          --file $DEPSHASH.tar \
-          --name $DEPSHASH \
-          --debug
+        cp ./$DEPSHASH.tar /mnt/cross-instance-cache/
+        sudo mkdir -p /var/portal
+        sudo chown -R $(id -u):$(id -g) /var/portal
+        mv ./src /var/portal
+    - name: Persist Src Cache
+      if: steps.check-cache.outputs.cache_exists == 'false'
+      run: |
+        cache_key=$DEPSHASH
+        backup_cache_path=/var/portal
+        final_cache_path=/mnt/cross-instance-cache/${cache_key}.tar
+        echo "Using cache key: $cache_key"
+        echo "Checking path: $final_cache_path"
+        if [ ! -f "$final_cache_path" ]; then
+          echo "Cache key not found, storing tarball"
+          tmp_container=/mnt/cross-instance-cache/tmp/${{ github.sha }}
+          tmp_cache_path=$tmp_container/${cache_key}.tar
+          mkdir -p $tmp_container
+          if [ -f "$backup_cache_path" ]; then
+            tar -cf $tmp_cache_path -C $(dirname $backup_cache_path) ./$(basename $backup_cache_path)
+          else
+            tar -cf $tmp_cache_path -C $backup_cache_path/ ./
+          fi
+          mv -vn $tmp_cache_path $final_cache_path
+          rm -rf $tmp_container
+        else
+          echo "Cache key already exists, skipping.."
+        fi
   build:
     strategy:
       fail-fast: false
@@ -221,6 +245,7 @@ jobs:
       run: |
         cd src/electron
         node script/yarn install
+        brew install azcopy
     - name: Load Target Arch & CPU
       run: |
         echo "TARGET_ARCH=${{ matrix.build-arch }}" >> $GITHUB_ENV
@@ -247,8 +272,10 @@ jobs:
     - name: Generate DEPS Hash
       run: |
         node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
-        echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV
-    - name: Download Src Cache
+        DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')
+        echo "DEPSHASH=$DEPSHASH" >> $GITHUB_ENV
+        echo "CACHE_PATH=$DEPSHASH.tar" >> $GITHUB_ENV
+    - name: Download Src Cache from AKS
       # The cache will always exist here as a result of the checkout job
       # Either it was uploaded to Azure in the checkout job for this commit
       # or it was uploaded in the checkout job for a previous commit.
@@ -258,12 +285,8 @@ jobs:
         max_attempts: 3
         retry_on: error
         command: |
-          az storage blob download \
-            --account-name $AZURE_STORAGE_ACCOUNT \
-            --account-key $AZURE_STORAGE_KEY \
-            --container-name $AZURE_STORAGE_CONTAINER_NAME \
-            --name $DEPSHASH \
-            --file $DEPSHASH.tar \
+          azcopy copy \
+            "https://${AZURE_AKS_CACHE_STORAGE_ACCOUNT}.file.core.windows.net/${AZURE_AKS_CACHE_SHARE_NAME}/${{ env.CACHE_PATH}}?${AZURE_AKS_CACHE_SAS_TOKEN}" $DEPSHASH.tar
     - name: Unzip and Ensure Src Cache
       run: |
         echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)"

script/azure_cli_deb_install.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+#######################################################################################################################
+# This script does three fundamental things:                                                                          #
+#   1. Add Microsoft's GPG Key has a trusted source of apt packages.                                                  #
+#   2. Add Microsoft's repositories as a source for apt packages.                                                     #
+#   3. Installs the Azure CLI from those repositories.                                                                #
+# Given the nature of this script, it must be executed with elevated privileges, i.e. with `sudo`.                    #
+#                                                                                                                     #
+# Copied from https://azurecliprod.blob.core.windows.net/$root/deb_install.sh                                         #
+#######################################################################################################################
+
+set -e
+
+if [[ $# -ge 1 && $1 == "-y" ]]; then
+    global_consent=0
+else
+    global_consent=1
+fi
+
+function assert_consent {
+    if [[ $2 -eq 0 ]]; then
+        return 0
+    fi
+
+    echo -n "$1 [Y/n] "
+    read consent
+    if [[ ! "${consent}" == "y" && ! "${consent}" == "Y" && ! "${consent}" == "" ]]; then
+        echo "'${consent}'"
+        exit 1
+    fi
+}
+
+global_consent=0 # Artificially giving global consent after review-feedback. Remove this line to enable interactive mode
+
+setup() {
+
+    assert_consent "Add packages necessary to modify your apt-package sources?" ${global_consent}
+    set -v
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+    apt-get install -y apt-transport-https lsb-release gnupg curl
+    set +v
+
+    assert_consent "Add Microsoft as a trusted package signer?" ${global_consent}
+    set -v
+    curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg
+    set +v
+
+    assert_consent "Add the Azure CLI Repository to your apt sources?" ${global_consent}
+    set -v
+    # Use env var DIST_CODE for the package dist name if provided
+    if [[ -z $DIST_CODE ]]; then
+        CLI_REPO=$(lsb_release -cs)
+        shopt -s nocasematch
+        ERROR_MSG="Unable to find a package for your system. Please check if an existing package in https://packages.microsoft.com/repos/azure-cli/dists/ can be used in your system and install with the dist name: 'curl -sL https://aka.ms/InstallAzureCLIDeb | sudo DIST_CODE=<dist_code_name> bash'"
+        if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then
+            DIST=$(lsb_release -is)
+            if [[ $DIST =~ "Ubuntu" ]]; then
+                CLI_REPO="jammy"
+            elif [[ $DIST =~ "Debian" ]]; then
+                CLI_REPO="bookworm"
+            elif [[ $DIST =~ "LinuxMint" ]]; then
+                CLI_REPO=$(cat /etc/os-release | grep -Po 'UBUNTU_CODENAME=\K.*') || true
+                if [[ -z $CLI_REPO ]]; then
+                    echo $ERROR_MSG
+                    exit 1
+                fi
+            else
+                echo $ERROR_MSG
+                exit 1
+            fi
+        fi
+    else
+        CLI_REPO=$DIST_CODE
+        if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then
+            echo "Unable to find an azure-cli package with DIST_CODE=$CLI_REPO in https://packages.microsoft.com/repos/azure-cli/dists/."
+            exit 1
+        fi
+    fi
+    echo "deb [arch=$(dpkg --print-architecture)] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \
+        > /etc/apt/sources.list.d/azure-cli.list
+    apt-get update
+    set +v
+
+    assert_consent "Install the Azure CLI?" ${global_consent}
+    apt-get install -y azure-cli
+
+}
+
+setup  # ensure the whole file is downloaded before executing