[CI] Move a number of bash scripts from inline scripts to file based scripts (#18311)

This allow stop minize the expanded size of the yaml.

tools/devops/automation/scripts/bash/build-macios.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -ex
+
+echo "##vso[task.setvariable variable=TESTS_BOT;isOutput=true]$AGENT_NAME"
+options=()
+
+if [[ "$SYSTEM_DEBUG" == "true" ]]; then
+  options=(V=1 -w)
+fi
+
+if test -z "$MAKE_PARALLELISM"; then
+  options=("${options[@]}" -j8)
+else
+  options=("${options[@]}" -j "$MAKE_PARALLELISM")
+fi
+
+# shellcheck disable=SC2046
+time make all "${options[@]}" IGNORE_SIMULATORS=1
+# shellcheck disable=SC2046
+time make install "${options[@]}" IGNORE_SIMULATORS=1

tools/devops/automation/scripts/bash/clean-results-dir.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -ex
+
+rm -Rvf package
+time make -C xamarin-macios/ git-clean-all

tools/devops/automation/scripts/bash/configure-build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -xe
+flags=( "--enable-xamarin" )
+
+if [[ "$ENABLE_DOT_NET" == "True" ]]; then
+  echo "Enabling dotnet builds."
+  flags=("${flags[@]}" --enable-dotnet)
+fi
+
+flags=("${flags[@]}" --enable-install-source)
+./configure "${flags[@]}"

tools/devops/automation/scripts/bash/delete-library-dirs.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+sudo rm -Rf /Developer/MonoTouch
+sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework
+sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework
+ls -R /Library/Frameworks

tools/devops/automation/scripts/bash/ensure-xcode-select.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+XCODE_SELECT=$(xcode-select -p)
+if [[ -d $XCODE_SELECT ]]; then
+  echo "Using Xcode in path $XCODE_SELECT"
+else
+  echo "Setting Xcode to point to the default location."
+  xcode-select -s "/Applications/Xcode.app/Contents/Developer"
+fi

tools/devops/automation/scripts/bash/fix-github-ssh-key.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+
+# ensure that we do remove the old ssh key from github which was leaked: https://blog.gitguardian.com/github-exposed-private-ssh-key/
+
+ssh-keygen -R github.com
+{
+  echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"
+  echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
+  echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg="
+} >> ~/.ssh/known_hosts

tools/devops/automation/scripts/bash/remove-ui-prompt.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+RC=0
+security set-key-partition-list -S apple-tool:,apple: -s -k "$OSX_KEYCHAIN_PASS" login.keychain || RC=$?
+if [ $RC -eq 0 ]; then
+  echo "Security UI-prompt removed."
+else
+  echo "Security UI-prompt could NOT be removed."
+fi

tools/devops/automation/scripts/parse_pr_labels.ps1

@@ -0,0 +1,65 @@
+param (
+    [Parameter(Mandatory)]
+    [String]
+    $XamarinTop,
+
+    [Parameter(Mandatory)]
+    [String]
+    $BuildReason
+)
+
+# we have a number of scripts that require to be executed from the top of the src, rather
+# than keeping track of the location of the script, we create two env vars that can be used to
+# get to the top
+$configVars = @{} # var name/value pair, later print twice, to process and debug
+
+Write-Host "##vso[task.setvariable variable=XAM_TOP]$XamarinTop"
+Write-Host "PrID in env is $Env:PR_ID"
+
+if (-not [string]::IsNullOrEmpty($Env:PR_ID)) {
+    $configVars.Add("PR_ID", $Env:PR_ID)
+} else {
+    $configVars.Add("PR_ID", "")
+}
+
+# decide if we are dealing with a PR or a re-triggered PR or a build from
+# a branch in origin
+
+if ($BuildReason -eq "PullRequest" -or (($BuildReason -eq "Manual" -or $BuildReason -eq "IndividualCI") -and ($BuildSourceBranchName -eq "merge")) ) {
+    $configVars.Add("BuildPkgs", "True")
+
+    # interesting case, we have build-pkg and skip-pkg... if that is the case, we build it, but we set a warning
+    if ($Env:BuildPackage -eq "True" -and $Env:SkipPackages -eq "True") {
+        Write-Host "##vso[task.logissue type=warning]'build-package' and 'skip-packages' are both present. Building packages in case of a doubt."
+        $configVars.Add("BuildPkgs", "True")
+    }
+
+    if ($Env:SkipNugets -eq "True") {
+        $configVars.Add("BuildNugets", "False")
+    } else {
+        $configVars.Add("BuildNugets", "True")
+    }
+
+    if ($Env:SkipSigning -eq "True") {
+        $configVars.Add("SignPkgs", "False")
+    } else {
+        $configVars.Add("SignPkgs", "True")
+    }
+
+    $configVars.Add("RunSampleTests", $Env:RunSampleTests)
+
+} else {
+    # set the defaults, all the things! o/
+    # build pkg, nugets and sign them
+    $configVars.Add("BuildPkgs", "True")
+    $configVars.Add("BuildNugets", "True")
+    $configVars.Add("SignPkgs", "True")
+
+    # tests, run all of them, internal, external, mac but not sample tests
+    $configVars.Add("RunSampleTests", "False")
+}
+# write debugging and process of the vars
+foreach($key in $configVars.Keys) {
+    Write-Host "$key='$($configVars[$key])'"
+    Write-Host "##vso[task.setvariable variable=$key;isOutput=true]$($configVars[$key])"
+}

tools/devops/automation/scripts/show_python_env.ps1

@@ -0,0 +1,6 @@
+Write-Host "Python3 location"
+which python3
+
+Write-Host "Pip3 version"
+pip3 -V
+

tools/devops/automation/templates/build/build.yml

@@ -95,70 +95,13 @@ steps:
 # print some useful logging to allow to know what is going on AND allow make some
 # choices, there are labels that contradict each other (skip-package vs build-packages)
 # we use warnings for those case we are not sure about.
-- pwsh: |
+- task: PowerShell@2
-    # we have a number of scripts that require to be executed from the top of the src, rather
-    # than keeping track of the location of the script, we create two env vars that can be used to
-    # get to the top
-    $configVars = @{} # var name/value pair, later print twice, to process and debug
-
-    $xamTop = "$(Build.SourcesDirectory)/xamarin-macios/"
-    Write-Host "##vso[task.setvariable variable=XAM_TOP]$xamTop"
-
-    $buildReason = "$(Build.Reason)"
-    $buildSourceBranchName = "$(Build.SourceBranchName)"
-
-    Write-Host "PrID in env is $Env:PR_ID"
-    if (-not [string]::IsNullOrEmpty($Env:PR_ID)) {
-      $configVars.Add("PR_ID", $Env:PR_ID)
-    } else {
-      $configVars.Add("PR_ID", "")
-    }
-
-    # decide if we are dealing with a PR or a re-triggered PR or a build from
-    # a branch in origin
-
-    if ($buildReason -eq "PullRequest" -or (($buildReason -eq "Manual" -or $buildReason -eq "IndividualCI") -and ($buildSourceBranchName -eq "merge")) ) {
-      $configVars.Add("BuildPkgs", "True")
-
-      # interesting case, we have build-pkg and skip-pkg... if that is the case, we build it, but we set a warning
-      if ($Env:BuildPackage -eq "True" -and $Env:SkipPackages -eq "True") {
-        Write-Host "##vso[task.logissue type=warning]'build-package' and 'skip-packages' are both present. Building packages in case of a doubt."
-        $configVars.Add("BuildPkgs", "True")
-      }
-
-      if ($Env:SkipNugets -eq "True") {
-        $configVars.Add("BuildNugets", "False")
-      } else {
-        $configVars.Add("BuildNugets", "True")
-      }
-
-      if ($Env:SkipSigning -eq "True") {
-        $configVars.Add("SignPkgs", "False")
-      } else {
-        $configVars.Add("SignPkgs", "True")
-      }
-
-      $configVars.Add("RunSampleTests", $Env:RunSampleTests)
-
-    } else {
-      # set the defaults, all the things! o/
-      # build pkg, nugets and sign them
-      $configVars.Add("BuildPkgs", "True")
-      $configVars.Add("BuildNugets", "True")
-      $configVars.Add("SignPkgs", "True")
-
-      # tests, run all of them, internal, external, mac but not sample tests
-      $configVars.Add("RunSampleTests", "False")
-    }
-    # write debugging and process of the vars
-    foreach($key in $configVars.Keys) {
-      Write-Host "$key='$($configVars[$key])'"
-      Write-Host "##vso[task.setvariable variable=$key;isOutput=true]$($configVars[$key])"
-    }
-
   name: configuration
   displayName: "Parse PR labels"
   timeoutInMinutes: 5
+  inputs:
+    filePath: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/parse_pr_labels.ps1
+    arguments: -XamarinTop "$(Build.SourcesDirectory)/xamarin-macios/" -BuildReason "$(Build.Reason)"
 
 - ${{ if contains(variables['Build.DefinitionName'], 'private') }}:
   - task: PowerShell@2
@@ -179,31 +122,16 @@ steps:
     displayName: "Configure install extra args"
     timeoutInMinutes: 5
 
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/configure-build.sh
-    set -x
-    set -e
-
-    CONFIGURE_FLAGS="--enable-xamarin"
-
-    if [[ "$EnableDotNet" == "True" ]]; then
-      echo "Enabling dotnet builds."
-      CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-dotnet"
-    fi
-
-    CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-install-source"
-    echo "Configuration flags are '$CONFIGURE_FLAGS'"
-
-    cd $(Build.SourcesDirectory)/xamarin-macios/
-    ./configure $CONFIGURE_FLAGS
-    echo $(cat $(Build.SourcesDirectory)/xamarin-macios/configure.inc)
   env:
     ${{ if eq(parameters.isPR, true) }}:
       IsPR: 'True'
     ${{ else }}:
       IsPR: 'False'
     ${{ if eq(parameters.enableDotnet, true) }}:
-      EnableDotNet: 'True'
+      ENABLE_DOT_NET: 'True'
   displayName: "Configure build"
+  workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios"
   timeoutInMinutes: 5
 
 # Make sure we have the right maccore hash checked out before we try to add
@@ -217,15 +145,11 @@ steps:
 
 # We'll need these profiles to build the hot restart prebuilt app during the build
 # (it's built for device, and thus needs a certificate available so that the app can be signed).
-- bash: |
+- bash: ./install-qa-provisioning-profiles.sh -v
-    set -x
-    set -e
-
-    cd "$SOURCES_DIR/maccore/tools/"
-    ./install-qa-provisioning-profiles.sh -v
   displayName: 'Add build provisioning profiles'
   timeoutInMinutes: 30
   continueOnError: true # should not stop the build
+  workingDirectory: $(Build.SourcesDirectory)/maccore/tools
   env:
     AUTH_TOKEN_GITHUB_COM: ${{ parameters.gitHubToken }}
     AUTH_TOKEN_LA_DEV_APPLE_P12: ${{ parameters.xqaCertPass }}
@@ -233,28 +157,15 @@ steps:
     AUTH_TOKEN_LA_MAC_INSTALLER_DISTR_P12: ${{ parameters.xqaCertPass }}
     AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_P12: ${{ parameters.xqaCertPass }}
     AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_2_P12: ${{ parameters.xqaCertPass }}
-    SOURCES_DIR: $(Build.SourcesDirectory)
 
 # Actual build of the project
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/build-macios.sh
-    set -x
-    set -e
-    echo "##vso[task.setvariable variable=TESTS_BOT;isOutput=true]$AGENT_NAME"
-    MAKE_FLAGS=""
-
-    if [[ "$SYSTEM_DEBUG" == "true" ]]; then
-      MAKE_FLAGS="V=1 -w"
-    fi
-
-    if test -z "$makeParallelism"; then
-      makeParallelism=8
-    fi
-
-    time make -C $(Build.SourcesDirectory)/xamarin-macios/ all -j$makeParallelism $MAKE_FLAGS IGNORE_SIMULATORS=1
-    time make -C $(Build.SourcesDirectory)/xamarin-macios/ install -j$makeParallelism $MAKE_FLAGS IGNORE_SIMULATORS=1
   name: build
+  workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios"
   displayName: 'Build'
   timeoutInMinutes: 300
+  env:
+    MAKE_PARALLELISM: ${{ parameters.makeParallelism }}
 
 - ${{ each step in parameters.buildSteps }}:
   - ${{ each pair in step }}:

tools/devops/automation/templates/common/setup.yml

@@ -10,13 +10,7 @@ steps:
   parameters:
     CleanseProcesses: true
 
-- bash: |
+- bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh
-    set -x
-    set -e
-    ssh-keygen -R github.com
-    echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ~/.ssh/known_hosts
-    echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ~/.ssh/known_hosts
-    echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ~/.ssh/known_hosts
   displayName: 'Fix GitHub SSH host key'
 
 - bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf
@@ -30,40 +24,20 @@ steps:
 - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1
   displayName: 'Show Environment'
 
-- powershell: |
+- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_python_env.ps1
-    Write-Host "Python3 location"
-    which python3
-
-    Write-Host "Pip3 version"
-    pip3 -V
   displayName: 'Show Python information'
 
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/delete-library-dirs.sh
-    set -x
-    sudo rm -Rf /Developer/MonoTouch
-    sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework
-    sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework
-    ls -R /Library/Frameworks
   displayName: 'Delete library folders'
   timeoutInMinutes: 5
 
-- bash:
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-results-dir.sh
-    set -x
+  workingDirectory: $(System.DefaultWorkingDirectory)
-    set -e
-    rm -Rvf $(Build.SourcesDirectory)/package
-    time make -C $(Build.SourcesDirectory)/xamarin-macios/ git-clean-all
   displayName: 'Clear results directory'
   timeoutInMinutes: 5
   continueOnError: true
 
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh
-    RC=0
-    security set-key-partition-list -S apple-tool:,apple: -s -k $OSX_KEYCHAIN_PASS login.keychain || RC=$?
-    if [ $RC -eq 0 ]; then
-      echo "Security UI-prompt removed."
-    else
-      echo "Security UI-prompt could NOT be removed."
-    fi
   env:
     OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }}
   displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)'
@@ -71,14 +45,7 @@ steps:
 
 # was the current xcode deleted (has happened due to a bug in the clean step)? then set the default so 
 # that we can call make and generated the provisionator config files.
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/ensure-xcode-select.sh
-    XCODE_SELECT=$(xcode-select -p)
-    if [[ -d $XCODE_SELECT ]]; then
-      echo "Using Xcode in path $XCODE_SELECT"
-    else
-      echo "Setting Xcode to point to the default location."
-      xcode-select -s "/Applications/Xcode.app/Contents/Developer"
-    fi
   displayName: 'Ensure Xcode is selected.'
 
 - pwsh: |

tools/devops/automation/templates/mac/build.yml

@@ -53,16 +53,9 @@ steps:
     GITHUB_TOKEN: $(GitHub.Token)
     SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 
-- bash: |
+- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh
-    RC=0
-    security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASS login.keychain || RC=$?
-    if [ $RC -eq 0 ]; then
-      echo "Security UI-prompt removed."
-    else
-      echo "Security UI-prompt could NOT be removed."
-    fi
   env:
-    KEYCHAIN_PASS: ${{ parameters.keyringPass }}
+    OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }}
   displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)'
   condition: succeededOrFailed() # we do not care about the previous process cleanup
   continueOnError: true