diff --git a/.github/workflows/macos-pipeline.yml b/.github/workflows/macos-pipeline.yml index 908b386575..cbd2c399e7 100644 --- a/.github/workflows/macos-pipeline.yml +++ b/.github/workflows/macos-pipeline.yml @@ -34,10 +34,17 @@ concurrency: cancel-in-progress: true env: + # Old Azure Storage Variables AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} AZURE_STORAGE_CONTAINER_NAME: ${{ secrets.AZURE_STORAGE_CONTAINER_NAME }} ELECTRON_ARTIFACTS_BLOB_STORAGE: ${{ secrets.ELECTRON_ARTIFACTS_BLOB_STORAGE }} + # New Azure Storage Variables + AZURE_AKS_CACHE_STORAGE_KEY: ${{ secrets.AZURE_AKS_CACHE_STORAGE_KEY }} + AZURE_AKS_CACHE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_AKS_CACHE_STORAGE_CONNECTION_STRING }} + AZURE_AKS_CACHE_STORAGE_ACCOUNT: ${{ secrets.AZURE_AKS_CACHE_STORAGE_ACCOUNT }} + AZURE_AKS_CACHE_SHARE_NAME: ${{ secrets.AZURE_AKS_CACHE_SHARE_NAME }} + AZURE_AKS_CACHE_SAS_TOKEN: ${{ secrets.AZURE_AKS_CACHE_SAS_TOKEN }} ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }} ELECTRON_GITHUB_TOKEN: ${{ secrets.ELECTRON_GITHUB_TOKEN }} GN_CONFIG: ${{ inputs.gn-config }} @@ -50,7 +57,13 @@ env: jobs: checkout: - runs-on: LargeLinuxRunner + runs-on: aks-linux-large + container: + image: ghcr.io/electron/build:latest + options: --user root + volumes: + - /mnt/cross-instance-cache:/mnt/cross-instance-cache + - /var/run/sas:/var/run/sas steps: - name: Checkout Electron uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 @@ -60,12 +73,8 @@ jobs: - name: Set GIT_CACHE_PATH to make gclient to use the cache run: | echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV - - name: Setup Node.js/npm - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 - with: - node-version: 20.11.x - cache: yarn - cache-dependency-path: src/electron/yarn.lock + - name: Install Azure CLI + run: sudo bash ./src/electron/script/azure_cli_deb_install.sh - name: Install Dependencies run: | cd src/electron @@ -95,19 +104,14 @@ jobs: - name: Check If Cache Exists id: check-cache run: | - exists_json=$(az storage blob exists \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --account-key $AZURE_STORAGE_KEY \ - --container-name $AZURE_STORAGE_CONTAINER_NAME \ - --name $DEPSHASH) - - cache_exists=$(echo $exists_json | jq -r '.exists') - echo "cache_exists=$cache_exists" >> $GITHUB_OUTPUT - - if (test "$cache_exists" = "true"); then - echo "Cache Exists for $DEPSHASH" - else + cache_key=$DEPSHASH + cache_path=/mnt/cross-instance-cache/${cache_key}.tar + echo "Using cache key: $cache_key" + echo "Checking for cache in: $cache_path" + if [ ! -f "$cache_path" ]; then echo "Cache Does Not Exist for $DEPSHASH" + else + echo "Cache Already Exists for $DEPSHASH, Skipping.." fi - name: Gclient Sync if: steps.check-cache.outputs.cache_exists == 'false' @@ -179,16 +183,36 @@ jobs: echo "Uncompressed src size: $(du -sh src | cut -f1 -d' ')" tar -cvf $DEPSHASH.tar src echo "Compressed src to $(du -sh $DEPSHASH.tar | cut -f1 -d' ')" - - name: Upload Compressed Src Cache to Azure + - name: Move src folder to cross-OS portal if: steps.check-cache.outputs.cache_exists == 'false' run: | - az storage blob upload \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --account-key $AZURE_STORAGE_KEY \ - --container-name $AZURE_STORAGE_CONTAINER_NAME \ - --file $DEPSHASH.tar \ - --name $DEPSHASH \ - --debug + cp ./$DEPSHASH.tar /mnt/cross-instance-cache/ + sudo mkdir -p /var/portal + sudo chown -R $(id -u):$(id -g) /var/portal + mv ./src /var/portal + - name: Persist Src Cache + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + cache_key=$DEPSHASH + backup_cache_path=/var/portal + final_cache_path=/mnt/cross-instance-cache/${cache_key}.tar + echo "Using cache key: $cache_key" + echo "Checking path: $final_cache_path" + if [ ! -f "$final_cache_path" ]; then + echo "Cache key not found, storing tarball" + tmp_container=/mnt/cross-instance-cache/tmp/${{ github.sha }} + tmp_cache_path=$tmp_container/${cache_key}.tar + mkdir -p $tmp_container + if [ -f "$backup_cache_path" ]; then + tar -cf $tmp_cache_path -C $(dirname $backup_cache_path) ./$(basename $backup_cache_path) + else + tar -cf $tmp_cache_path -C $backup_cache_path/ ./ + fi + mv -vn $tmp_cache_path $final_cache_path + rm -rf $tmp_container + else + echo "Cache key already exists, skipping.." + fi build: strategy: fail-fast: false @@ -221,6 +245,7 @@ jobs: run: | cd src/electron node script/yarn install + brew install azcopy - name: Load Target Arch & CPU run: | echo "TARGET_ARCH=${{ matrix.build-arch }}" >> $GITHUB_ENV @@ -247,8 +272,10 @@ jobs: - name: Generate DEPS Hash run: | node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target - echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV - - name: Download Src Cache + DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ') + echo "DEPSHASH=$DEPSHASH" >> $GITHUB_ENV + echo "CACHE_PATH=$DEPSHASH.tar" >> $GITHUB_ENV + - name: Download Src Cache from AKS # The cache will always exist here as a result of the checkout job # Either it was uploaded to Azure in the checkout job for this commit # or it was uploaded in the checkout job for a previous commit. @@ -258,12 +285,8 @@ jobs: max_attempts: 3 retry_on: error command: | - az storage blob download \ - --account-name $AZURE_STORAGE_ACCOUNT \ - --account-key $AZURE_STORAGE_KEY \ - --container-name $AZURE_STORAGE_CONTAINER_NAME \ - --name $DEPSHASH \ - --file $DEPSHASH.tar \ + azcopy copy \ + "https://${AZURE_AKS_CACHE_STORAGE_ACCOUNT}.file.core.windows.net/${AZURE_AKS_CACHE_SHARE_NAME}/${{ env.CACHE_PATH}}?${AZURE_AKS_CACHE_SAS_TOKEN}" $DEPSHASH.tar - name: Unzip and Ensure Src Cache run: | echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)" diff --git a/script/azure_cli_deb_install.sh b/script/azure_cli_deb_install.sh new file mode 100755 index 0000000000..e2dde20e1f --- /dev/null +++ b/script/azure_cli_deb_install.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +####################################################################################################################### +# This script does three fundamental things: # +# 1. Add Microsoft's GPG Key has a trusted source of apt packages. # +# 2. Add Microsoft's repositories as a source for apt packages. # +# 3. Installs the Azure CLI from those repositories. # +# Given the nature of this script, it must be executed with elevated privileges, i.e. with `sudo`. # +# # +# Copied from https://azurecliprod.blob.core.windows.net/$root/deb_install.sh # +####################################################################################################################### + +set -e + +if [[ $# -ge 1 && $1 == "-y" ]]; then + global_consent=0 +else + global_consent=1 +fi + +function assert_consent { + if [[ $2 -eq 0 ]]; then + return 0 + fi + + echo -n "$1 [Y/n] " + read consent + if [[ ! "${consent}" == "y" && ! "${consent}" == "Y" && ! "${consent}" == "" ]]; then + echo "'${consent}'" + exit 1 + fi +} + +global_consent=0 # Artificially giving global consent after review-feedback. Remove this line to enable interactive mode + +setup() { + + assert_consent "Add packages necessary to modify your apt-package sources?" ${global_consent} + set -v + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y apt-transport-https lsb-release gnupg curl + set +v + + assert_consent "Add Microsoft as a trusted package signer?" ${global_consent} + set -v + curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg + set +v + + assert_consent "Add the Azure CLI Repository to your apt sources?" ${global_consent} + set -v + # Use env var DIST_CODE for the package dist name if provided + if [[ -z $DIST_CODE ]]; then + CLI_REPO=$(lsb_release -cs) + shopt -s nocasematch + ERROR_MSG="Unable to find a package for your system. Please check if an existing package in https://packages.microsoft.com/repos/azure-cli/dists/ can be used in your system and install with the dist name: 'curl -sL https://aka.ms/InstallAzureCLIDeb | sudo DIST_CODE= bash'" + if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then + DIST=$(lsb_release -is) + if [[ $DIST =~ "Ubuntu" ]]; then + CLI_REPO="jammy" + elif [[ $DIST =~ "Debian" ]]; then + CLI_REPO="bookworm" + elif [[ $DIST =~ "LinuxMint" ]]; then + CLI_REPO=$(cat /etc/os-release | grep -Po 'UBUNTU_CODENAME=\K.*') || true + if [[ -z $CLI_REPO ]]; then + echo $ERROR_MSG + exit 1 + fi + else + echo $ERROR_MSG + exit 1 + fi + fi + else + CLI_REPO=$DIST_CODE + if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then + echo "Unable to find an azure-cli package with DIST_CODE=$CLI_REPO in https://packages.microsoft.com/repos/azure-cli/dists/." + exit 1 + fi + fi + echo "deb [arch=$(dpkg --print-architecture)] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \ + > /etc/apt/sources.list.d/azure-cli.list + apt-get update + set +v + + assert_consent "Install the Azure CLI?" ${global_consent} + apt-get install -y azure-cli + +} + +setup # ensure the whole file is downloaded before executing \ No newline at end of file