diff --git a/tools/devops/automation/scripts/bash/build-macios.sh b/tools/devops/automation/scripts/bash/build-macios.sh new file mode 100755 index 0000000000..0ebcf809cb --- /dev/null +++ b/tools/devops/automation/scripts/bash/build-macios.sh @@ -0,0 +1,19 @@ +#!/bin/bash -ex + +echo "##vso[task.setvariable variable=TESTS_BOT;isOutput=true]$AGENT_NAME" +options=() + +if [[ "$SYSTEM_DEBUG" == "true" ]]; then + options=(V=1 -w) +fi + +if test -z "$MAKE_PARALLELISM"; then + options=("${options[@]}" -j8) +else + options=("${options[@]}" -j "$MAKE_PARALLELISM") +fi + +# shellcheck disable=SC2046 +time make all "${options[@]}" IGNORE_SIMULATORS=1 +# shellcheck disable=SC2046 +time make install "${options[@]}" IGNORE_SIMULATORS=1 diff --git a/tools/devops/automation/scripts/bash/clean-results-dir.sh b/tools/devops/automation/scripts/bash/clean-results-dir.sh new file mode 100755 index 0000000000..3e1aa2dfb4 --- /dev/null +++ b/tools/devops/automation/scripts/bash/clean-results-dir.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex + +rm -Rvf package +time make -C xamarin-macios/ git-clean-all diff --git a/tools/devops/automation/scripts/bash/configure-build.sh b/tools/devops/automation/scripts/bash/configure-build.sh new file mode 100755 index 0000000000..742096dc3d --- /dev/null +++ b/tools/devops/automation/scripts/bash/configure-build.sh @@ -0,0 +1,10 @@ +#!/bin/bash -xe +flags=( "--enable-xamarin" ) + +if [[ "$ENABLE_DOT_NET" == "True" ]]; then + echo "Enabling dotnet builds." + flags=("${flags[@]}" --enable-dotnet) +fi + +flags=("${flags[@]}" --enable-install-source) +./configure "${flags[@]}" diff --git a/tools/devops/automation/scripts/bash/delete-library-dirs.sh b/tools/devops/automation/scripts/bash/delete-library-dirs.sh new file mode 100755 index 0000000000..3c647824f7 --- /dev/null +++ b/tools/devops/automation/scripts/bash/delete-library-dirs.sh @@ -0,0 +1,6 @@ +#!/bin/bash -ex + +sudo rm -Rf /Developer/MonoTouch +sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework +sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework +ls -R /Library/Frameworks diff --git a/tools/devops/automation/scripts/bash/ensure-xcode-select.sh b/tools/devops/automation/scripts/bash/ensure-xcode-select.sh new file mode 100755 index 0000000000..09534e1d44 --- /dev/null +++ b/tools/devops/automation/scripts/bash/ensure-xcode-select.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ex + +XCODE_SELECT=$(xcode-select -p) +if [[ -d $XCODE_SELECT ]]; then + echo "Using Xcode in path $XCODE_SELECT" +else + echo "Setting Xcode to point to the default location." + xcode-select -s "/Applications/Xcode.app/Contents/Developer" +fi diff --git a/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh b/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh new file mode 100755 index 0000000000..a603c99bc6 --- /dev/null +++ b/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh @@ -0,0 +1,10 @@ +#!/bin/bash -ex + +# ensure that we do remove the old ssh key from github which was leaked: https://blog.gitguardian.com/github-exposed-private-ssh-key/ + +ssh-keygen -R github.com +{ + echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" + echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" +} >> ~/.ssh/known_hosts diff --git a/tools/devops/automation/scripts/bash/remove-ui-prompt.sh b/tools/devops/automation/scripts/bash/remove-ui-prompt.sh new file mode 100755 index 0000000000..b1fed49ba1 --- /dev/null +++ b/tools/devops/automation/scripts/bash/remove-ui-prompt.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ex + +RC=0 +security set-key-partition-list -S apple-tool:,apple: -s -k "$OSX_KEYCHAIN_PASS" login.keychain || RC=$? +if [ $RC -eq 0 ]; then + echo "Security UI-prompt removed." +else + echo "Security UI-prompt could NOT be removed." +fi diff --git a/tools/devops/automation/scripts/parse_pr_labels.ps1 b/tools/devops/automation/scripts/parse_pr_labels.ps1 new file mode 100644 index 0000000000..b637e99e4d --- /dev/null +++ b/tools/devops/automation/scripts/parse_pr_labels.ps1 @@ -0,0 +1,65 @@ +param ( + [Parameter(Mandatory)] + [String] + $XamarinTop, + + [Parameter(Mandatory)] + [String] + $BuildReason +) + +# we have a number of scripts that require to be executed from the top of the src, rather +# than keeping track of the location of the script, we create two env vars that can be used to +# get to the top +$configVars = @{} # var name/value pair, later print twice, to process and debug + +Write-Host "##vso[task.setvariable variable=XAM_TOP]$XamarinTop" +Write-Host "PrID in env is $Env:PR_ID" + +if (-not [string]::IsNullOrEmpty($Env:PR_ID)) { + $configVars.Add("PR_ID", $Env:PR_ID) +} else { + $configVars.Add("PR_ID", "") +} + +# decide if we are dealing with a PR or a re-triggered PR or a build from +# a branch in origin + +if ($BuildReason -eq "PullRequest" -or (($BuildReason -eq "Manual" -or $BuildReason -eq "IndividualCI") -and ($BuildSourceBranchName -eq "merge")) ) { + $configVars.Add("BuildPkgs", "True") + + # interesting case, we have build-pkg and skip-pkg... if that is the case, we build it, but we set a warning + if ($Env:BuildPackage -eq "True" -and $Env:SkipPackages -eq "True") { + Write-Host "##vso[task.logissue type=warning]'build-package' and 'skip-packages' are both present. Building packages in case of a doubt." + $configVars.Add("BuildPkgs", "True") + } + + if ($Env:SkipNugets -eq "True") { + $configVars.Add("BuildNugets", "False") + } else { + $configVars.Add("BuildNugets", "True") + } + + if ($Env:SkipSigning -eq "True") { + $configVars.Add("SignPkgs", "False") + } else { + $configVars.Add("SignPkgs", "True") + } + + $configVars.Add("RunSampleTests", $Env:RunSampleTests) + +} else { + # set the defaults, all the things! o/ + # build pkg, nugets and sign them + $configVars.Add("BuildPkgs", "True") + $configVars.Add("BuildNugets", "True") + $configVars.Add("SignPkgs", "True") + + # tests, run all of them, internal, external, mac but not sample tests + $configVars.Add("RunSampleTests", "False") +} +# write debugging and process of the vars +foreach($key in $configVars.Keys) { + Write-Host "$key='$($configVars[$key])'" + Write-Host "##vso[task.setvariable variable=$key;isOutput=true]$($configVars[$key])" +} diff --git a/tools/devops/automation/scripts/show_python_env.ps1 b/tools/devops/automation/scripts/show_python_env.ps1 new file mode 100644 index 0000000000..79873375cf --- /dev/null +++ b/tools/devops/automation/scripts/show_python_env.ps1 @@ -0,0 +1,6 @@ +Write-Host "Python3 location" +which python3 + +Write-Host "Pip3 version" +pip3 -V + diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index f0fcab2227..cbbf1a0a4e 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -95,70 +95,13 @@ steps: # print some useful logging to allow to know what is going on AND allow make some # choices, there are labels that contradict each other (skip-package vs build-packages) # we use warnings for those case we are not sure about. -- pwsh: | - # we have a number of scripts that require to be executed from the top of the src, rather - # than keeping track of the location of the script, we create two env vars that can be used to - # get to the top - $configVars = @{} # var name/value pair, later print twice, to process and debug - - $xamTop = "$(Build.SourcesDirectory)/xamarin-macios/" - Write-Host "##vso[task.setvariable variable=XAM_TOP]$xamTop" - - $buildReason = "$(Build.Reason)" - $buildSourceBranchName = "$(Build.SourceBranchName)" - - Write-Host "PrID in env is $Env:PR_ID" - if (-not [string]::IsNullOrEmpty($Env:PR_ID)) { - $configVars.Add("PR_ID", $Env:PR_ID) - } else { - $configVars.Add("PR_ID", "") - } - - # decide if we are dealing with a PR or a re-triggered PR or a build from - # a branch in origin - - if ($buildReason -eq "PullRequest" -or (($buildReason -eq "Manual" -or $buildReason -eq "IndividualCI") -and ($buildSourceBranchName -eq "merge")) ) { - $configVars.Add("BuildPkgs", "True") - - # interesting case, we have build-pkg and skip-pkg... if that is the case, we build it, but we set a warning - if ($Env:BuildPackage -eq "True" -and $Env:SkipPackages -eq "True") { - Write-Host "##vso[task.logissue type=warning]'build-package' and 'skip-packages' are both present. Building packages in case of a doubt." - $configVars.Add("BuildPkgs", "True") - } - - if ($Env:SkipNugets -eq "True") { - $configVars.Add("BuildNugets", "False") - } else { - $configVars.Add("BuildNugets", "True") - } - - if ($Env:SkipSigning -eq "True") { - $configVars.Add("SignPkgs", "False") - } else { - $configVars.Add("SignPkgs", "True") - } - - $configVars.Add("RunSampleTests", $Env:RunSampleTests) - - } else { - # set the defaults, all the things! o/ - # build pkg, nugets and sign them - $configVars.Add("BuildPkgs", "True") - $configVars.Add("BuildNugets", "True") - $configVars.Add("SignPkgs", "True") - - # tests, run all of them, internal, external, mac but not sample tests - $configVars.Add("RunSampleTests", "False") - } - # write debugging and process of the vars - foreach($key in $configVars.Keys) { - Write-Host "$key='$($configVars[$key])'" - Write-Host "##vso[task.setvariable variable=$key;isOutput=true]$($configVars[$key])" - } - +- task: PowerShell@2 name: configuration displayName: "Parse PR labels" timeoutInMinutes: 5 + inputs: + filePath: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/parse_pr_labels.ps1 + arguments: -XamarinTop "$(Build.SourcesDirectory)/xamarin-macios/" -BuildReason "$(Build.Reason)" - ${{ if contains(variables['Build.DefinitionName'], 'private') }}: - task: PowerShell@2 @@ -179,31 +122,16 @@ steps: displayName: "Configure install extra args" timeoutInMinutes: 5 -- bash: | - set -x - set -e - - CONFIGURE_FLAGS="--enable-xamarin" - - if [[ "$EnableDotNet" == "True" ]]; then - echo "Enabling dotnet builds." - CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-dotnet" - fi - - CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-install-source" - echo "Configuration flags are '$CONFIGURE_FLAGS'" - - cd $(Build.SourcesDirectory)/xamarin-macios/ - ./configure $CONFIGURE_FLAGS - echo $(cat $(Build.SourcesDirectory)/xamarin-macios/configure.inc) +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/configure-build.sh env: ${{ if eq(parameters.isPR, true) }}: IsPR: 'True' ${{ else }}: IsPR: 'False' ${{ if eq(parameters.enableDotnet, true) }}: - EnableDotNet: 'True' + ENABLE_DOT_NET: 'True' displayName: "Configure build" + workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios" timeoutInMinutes: 5 # Make sure we have the right maccore hash checked out before we try to add @@ -217,15 +145,11 @@ steps: # We'll need these profiles to build the hot restart prebuilt app during the build # (it's built for device, and thus needs a certificate available so that the app can be signed). -- bash: | - set -x - set -e - - cd "$SOURCES_DIR/maccore/tools/" - ./install-qa-provisioning-profiles.sh -v +- bash: ./install-qa-provisioning-profiles.sh -v displayName: 'Add build provisioning profiles' timeoutInMinutes: 30 continueOnError: true # should not stop the build + workingDirectory: $(Build.SourcesDirectory)/maccore/tools env: AUTH_TOKEN_GITHUB_COM: ${{ parameters.gitHubToken }} AUTH_TOKEN_LA_DEV_APPLE_P12: ${{ parameters.xqaCertPass }} @@ -233,28 +157,15 @@ steps: AUTH_TOKEN_LA_MAC_INSTALLER_DISTR_P12: ${{ parameters.xqaCertPass }} AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_P12: ${{ parameters.xqaCertPass }} AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_2_P12: ${{ parameters.xqaCertPass }} - SOURCES_DIR: $(Build.SourcesDirectory) # Actual build of the project -- bash: | - set -x - set -e - echo "##vso[task.setvariable variable=TESTS_BOT;isOutput=true]$AGENT_NAME" - MAKE_FLAGS="" - - if [[ "$SYSTEM_DEBUG" == "true" ]]; then - MAKE_FLAGS="V=1 -w" - fi - - if test -z "$makeParallelism"; then - makeParallelism=8 - fi - - time make -C $(Build.SourcesDirectory)/xamarin-macios/ all -j$makeParallelism $MAKE_FLAGS IGNORE_SIMULATORS=1 - time make -C $(Build.SourcesDirectory)/xamarin-macios/ install -j$makeParallelism $MAKE_FLAGS IGNORE_SIMULATORS=1 +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/build-macios.sh name: build + workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios" displayName: 'Build' timeoutInMinutes: 300 + env: + MAKE_PARALLELISM: ${{ parameters.makeParallelism }} - ${{ each step in parameters.buildSteps }}: - ${{ each pair in step }}: diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index fdf52dc522..9c88f049f5 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -10,13 +10,7 @@ steps: parameters: CleanseProcesses: true -- bash: | - set -x - set -e - ssh-keygen -R github.com - echo "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl" >> ~/.ssh/known_hosts - echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ~/.ssh/known_hosts - echo "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=" >> ~/.ssh/known_hosts +- bash: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/fix-github-ssh-key.sh displayName: 'Fix GitHub SSH host key' - bash: cd $(System.DefaultWorkingDirectory)/xamarin-macios/ && git clean -xdf @@ -30,40 +24,20 @@ steps: - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 displayName: 'Show Environment' -- powershell: | - Write-Host "Python3 location" - which python3 - - Write-Host "Pip3 version" - pip3 -V +- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_python_env.ps1 displayName: 'Show Python information' -- bash: | - set -x - sudo rm -Rf /Developer/MonoTouch - sudo rm -Rf /Library/Frameworks/Xamarin.iOS.framework - sudo rm -Rf /Library/Frameworks/Xamarin.Mac.framework - ls -R /Library/Frameworks +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/delete-library-dirs.sh displayName: 'Delete library folders' timeoutInMinutes: 5 -- bash: - set -x - set -e - rm -Rvf $(Build.SourcesDirectory)/package - time make -C $(Build.SourcesDirectory)/xamarin-macios/ git-clean-all +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/clean-results-dir.sh + workingDirectory: $(System.DefaultWorkingDirectory) displayName: 'Clear results directory' timeoutInMinutes: 5 continueOnError: true -- bash: | - RC=0 - security set-key-partition-list -S apple-tool:,apple: -s -k $OSX_KEYCHAIN_PASS login.keychain || RC=$? - if [ $RC -eq 0 ]; then - echo "Security UI-prompt removed." - else - echo "Security UI-prompt could NOT be removed." - fi +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh env: OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }} displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)' @@ -71,14 +45,7 @@ steps: # was the current xcode deleted (has happened due to a bug in the clean step)? then set the default so # that we can call make and generated the provisionator config files. -- bash: | - XCODE_SELECT=$(xcode-select -p) - if [[ -d $XCODE_SELECT ]]; then - echo "Using Xcode in path $XCODE_SELECT" - else - echo "Setting Xcode to point to the default location." - xcode-select -s "/Applications/Xcode.app/Contents/Developer" - fi +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/ensure-xcode-select.sh displayName: 'Ensure Xcode is selected.' - pwsh: | diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index 511367ebfb..1576334c02 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -53,16 +53,9 @@ steps: GITHUB_TOKEN: $(GitHub.Token) SYSTEM_ACCESSTOKEN: $(System.AccessToken) -- bash: | - RC=0 - security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASS login.keychain || RC=$? - if [ $RC -eq 0 ]; then - echo "Security UI-prompt removed." - else - echo "Security UI-prompt could NOT be removed." - fi +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/remove-ui-prompt.sh env: - KEYCHAIN_PASS: ${{ parameters.keyringPass }} + OSX_KEYCHAIN_PASS: ${{ parameters.keyringPass }} displayName: 'Remove security UI-prompt (http://stackoverflow.com/a/40039594/183422)' condition: succeededOrFailed() # we do not care about the previous process cleanup continueOnError: true